From 4bc2bc7981f7dee03f06b228ed59d2f0d9fafff1 Mon Sep 17 00:00:00 2001 From: Allegre7tto Date: Fri, 21 Aug 2026 11:42:58 +0800 Subject: [PATCH 1/2] feat(writer): add Date32 support --- src/arrow_writer.rs | 68 +++++++++++++++++++++++++++++--- src/writer/column.rs | 3 +- src/writer/stripe.rs | 6 ++- tests/writer_date/main.rs | 81 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 150 insertions(+), 8 deletions(-) create mode 100644 tests/writer_date/main.rs diff --git a/src/arrow_writer.rs b/src/arrow_writer.rs index 9636ceb..b2fa8ae 100644 --- a/src/arrow_writer.rs +++ b/src/arrow_writer.rs @@ -229,6 +229,10 @@ fn serialize_schema(schema: &SchemaRef) -> Vec { kind: Some(proto::r#type::Kind::Long.into()), ..Default::default() }, + ArrowDataType::Date32 => proto::Type { + kind: Some(proto::r#type::Kind::Date.into()), + ..Default::default() + }, ArrowDataType::Utf8 | ArrowDataType::LargeUtf8 => proto::Type { kind: Some(proto::r#type::Kind::String.into()), ..Default::default() @@ -302,9 +306,9 @@ mod tests { use arrow::{ array::{ - Array, BinaryArray, BooleanArray, Float32Array, Float64Array, Int16Array, Int32Array, - Int64Array, Int8Array, LargeBinaryArray, LargeStringArray, RecordBatchReader, - StringArray, + Array, BinaryArray, BooleanArray, Date32Array, Float32Array, Float64Array, Int16Array, + Int32Array, Int64Array, Int8Array, LargeBinaryArray, LargeStringArray, + RecordBatchReader, StringArray, }, buffer::NullBuffer, compute::concat_batches, @@ -315,13 +319,14 @@ mod tests { use crate::{ compression::{CompressionType, Decompressor}, + schema::DataType as OrcDataType, stripe::Stripe, ArrowReaderBuilder, }; use super::*; - fn roundtrip(batches: &[RecordBatch]) -> Vec { + fn encode(batches: &[RecordBatch]) -> Bytes { let mut f = vec![]; let mut writer = ArrowWriterBuilder::new(&mut f, batches[0].schema()) .try_build() @@ -331,7 +336,11 @@ mod tests { } writer.close().unwrap(); - let f = Bytes::from(f); + Bytes::from(f) + } + + fn roundtrip(batches: &[RecordBatch]) -> Vec { + let f = encode(batches); let reader = ArrowReaderBuilder::try_new(f).unwrap().build(); reader.collect::, _>>().unwrap() } @@ -454,6 +463,55 @@ mod tests { assert_eq!(batch, rows[0]); } + #[test] + fn test_roundtrip_write_date32() { + let schema = Arc::new(Schema::new(vec![Field::new( + "date", + ArrowDataType::Date32, + true, + )])); + + let batch_without_nulls = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Date32Array::from(vec![-1, 0, 1, 11_016]))], + ) + .unwrap(); + let batch_with_nulls = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Date32Array::from(vec![ + Some(18_321), + None, + Some(19_782), + ]))], + ) + .unwrap(); + + let file = encode(&[batch_without_nulls, batch_with_nulls]); + let builder = ArrowReaderBuilder::try_new(file).unwrap(); + assert!(matches!( + builder.file_metadata().root_data_type().children()[0].data_type(), + OrcDataType::Date { .. } + )); + + let rows = builder.build().collect::, _>>().unwrap(); + let actual = concat_batches(&schema, rows.iter()).unwrap(); + let expected = RecordBatch::try_new( + schema, + vec![Arc::new(Date32Array::from(vec![ + Some(-1), + Some(0), + Some(1), + Some(11_016), + Some(18_321), + None, + Some(19_782), + ]))], + ) + .unwrap(); + + assert_eq!(expected, actual); + } + #[test] fn test_write_small_stripes() { // Set small stripe size to ensure writing across multiple stripes works diff --git a/src/writer/column.rs b/src/writer/column.rs index f1ec72f..352b805 100644 --- a/src/writer/column.rs +++ b/src/writer/column.rs @@ -20,7 +20,7 @@ use std::marker::PhantomData; use arrow::{ array::{Array, ArrayRef, AsArray}, datatypes::{ - ArrowPrimitiveType, ByteArrayType, Float32Type, Float64Type, GenericBinaryType, + ArrowPrimitiveType, ByteArrayType, Date32Type, Float32Type, Float64Type, GenericBinaryType, GenericStringType, Int16Type, Int32Type, Int64Type, Int8Type, }, }; @@ -396,6 +396,7 @@ pub type ByteColumnEncoder = PrimitiveColumnEncoder; pub type Int16ColumnEncoder = PrimitiveColumnEncoder>; pub type Int32ColumnEncoder = PrimitiveColumnEncoder>; pub type Int64ColumnEncoder = PrimitiveColumnEncoder>; +pub type DateColumnEncoder = PrimitiveColumnEncoder>; pub type StringColumnEncoder = GenericBinaryColumnEncoder>; pub type LargeStringColumnEncoder = GenericBinaryColumnEncoder>; pub type BinaryColumnEncoder = GenericBinaryColumnEncoder>; diff --git a/src/writer/stripe.rs b/src/writer/stripe.rs index 7de50ae..e449941 100644 --- a/src/writer/stripe.rs +++ b/src/writer/stripe.rs @@ -29,8 +29,9 @@ use crate::proto; use super::column::{ BinaryColumnEncoder, BooleanColumnEncoder, ByteColumnEncoder, ColumnStripeEncoder, - DoubleColumnEncoder, FloatColumnEncoder, Int16ColumnEncoder, Int32ColumnEncoder, - Int64ColumnEncoder, LargeBinaryColumnEncoder, LargeStringColumnEncoder, StringColumnEncoder, + DateColumnEncoder, DoubleColumnEncoder, FloatColumnEncoder, Int16ColumnEncoder, + Int32ColumnEncoder, Int64ColumnEncoder, LargeBinaryColumnEncoder, LargeStringColumnEncoder, + StringColumnEncoder, }; use super::{ColumnEncoding, StreamType}; @@ -183,6 +184,7 @@ fn create_encoder(field: &FieldRef) -> Box { ArrowDataType::Int16 => Box::new(Int16ColumnEncoder::new(ColumnEncoding::DirectV2)), ArrowDataType::Int32 => Box::new(Int32ColumnEncoder::new(ColumnEncoding::DirectV2)), ArrowDataType::Int64 => Box::new(Int64ColumnEncoder::new(ColumnEncoding::DirectV2)), + ArrowDataType::Date32 => Box::new(DateColumnEncoder::new(ColumnEncoding::DirectV2)), ArrowDataType::Utf8 => Box::new(StringColumnEncoder::new()), ArrowDataType::LargeUtf8 => Box::new(LargeStringColumnEncoder::new()), ArrowDataType::Binary => Box::new(BinaryColumnEncoder::new()), diff --git a/tests/writer_date/main.rs b/tests/writer_date/main.rs new file mode 100644 index 0000000..9036a2d --- /dev/null +++ b/tests/writer_date/main.rs @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::{fs, process::Command, sync::Arc}; + +use arrow::{ + array::{Date32Array, RecordBatch}, + datatypes::{DataType, Field, Schema}, +}; +use orc_rust::{compression::CompressionType, ArrowWriterBuilder}; + +#[test] +#[ignore = "requires ./venv/bin/python with pyorc installed"] +fn date32_file_is_readable_by_pyorc() { + let schema = Arc::new(Schema::new(vec![Field::new( + "date", + DataType::Date32, + true, + )])); + let batch = RecordBatch::try_new( + schema, + vec![Arc::new(Date32Array::from(vec![ + Some(-1), + Some(0), + Some(1), + Some(11_016), + None, + Some(18_321), + Some(19_782), + ]))], + ) + .unwrap(); + + let path = + std::env::temp_dir().join(format!("orc-rust-writer-date32-{}.orc", std::process::id())); + let file = fs::File::create(&path).unwrap(); + let mut writer = ArrowWriterBuilder::new(file, batch.schema()) + .with_compression(CompressionType::Zstd) + .try_build() + .unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + + let status = Command::new(format!("{}/venv/bin/python", env!("CARGO_MANIFEST_DIR"))) + .arg("-c") + .arg( + "import datetime, pyorc, sys\n\ + with open(sys.argv[1], 'rb') as source:\n\ + \treader = pyorc.Reader(source)\n\ + \tassert str(reader.schema) == 'struct', reader.schema\n\ + \trows = list(reader)\n\ + \texpected = [(datetime.date(1969, 12, 31),),\n\ + \t (datetime.date(1970, 1, 1),),\n\ + \t (datetime.date(1970, 1, 2),),\n\ + \t (datetime.date(2000, 2, 29),),\n\ + \t (None,),\n\ + \t (datetime.date(2020, 2, 29),),\n\ + \t (datetime.date(2024, 2, 29),)]\n\ + \tassert rows == expected, rows", + ) + .arg(&path) + .status() + .unwrap(); + assert!(status.success(), "pyorc failed to read Date32 file"); + + fs::remove_file(path).unwrap(); +} From 48e79877d98067ed0925682ca015cd67118db0f2 Mon Sep 17 00:00:00 2001 From: Allegre7tto Date: Fri, 21 Aug 2026 16:09:57 +0800 Subject: [PATCH 2/2] feat(writer): add timestamp support --- src/arrow_writer.rs | 168 +++++++++++++++++++++++++++++- src/encoding/mod.rs | 4 +- src/encoding/timestamp.rs | 182 ++++++++++++++++++++++++++++++++- src/error.rs | 12 +++ src/writer/column.rs | 95 ++++++++++++++++- src/writer/stripe.rs | 40 +++++++- tests/writer_timestamp/main.rs | 91 +++++++++++++++++ 7 files changed, 578 insertions(+), 14 deletions(-) create mode 100644 tests/writer_timestamp/main.rs diff --git a/src/arrow_writer.rs b/src/arrow_writer.rs index b2fa8ae..c9f93a7 100644 --- a/src/arrow_writer.rs +++ b/src/arrow_writer.rs @@ -86,10 +86,16 @@ impl ArrowWriterBuilder { /// Construct an [`ArrowWriter`] ready to encode [`RecordBatch`]es into /// an ORC file. pub fn try_build(mut self) -> Result> { + let timezone = self + .schema + .fields() + .iter() + .any(|field| matches!(field.data_type(), ArrowDataType::Timestamp(_, None))) + .then(|| "UTC".to_owned()); let compressor = Compressor::new(self.compression, self.compression_block_size)?; // Required magic "ORC" bytes at start of file self.writer.write_all(b"ORC").context(IoSnafu)?; - let writer = StripeWriter::new(self.writer, &self.schema, compressor); + let writer = StripeWriter::new(self.writer, &self.schema, compressor, timezone); Ok(ArrowWriter { writer, schema: self.schema, @@ -233,6 +239,16 @@ fn serialize_schema(schema: &SchemaRef) -> Vec { kind: Some(proto::r#type::Kind::Date.into()), ..Default::default() }, + ArrowDataType::Timestamp(_, None) => proto::Type { + kind: Some(proto::r#type::Kind::Timestamp.into()), + ..Default::default() + }, + ArrowDataType::Timestamp(_, Some(timezone)) if timezone.as_ref() == "UTC" => { + proto::Type { + kind: Some(proto::r#type::Kind::TimestampInstant.into()), + ..Default::default() + } + } ArrowDataType::Utf8 | ArrowDataType::LargeUtf8 => proto::Type { kind: Some(proto::r#type::Kind::String.into()), ..Default::default() @@ -306,13 +322,14 @@ mod tests { use arrow::{ array::{ - Array, BinaryArray, BooleanArray, Date32Array, Float32Array, Float64Array, Int16Array, - Int32Array, Int64Array, Int8Array, LargeBinaryArray, LargeStringArray, - RecordBatchReader, StringArray, + Array, ArrayRef, BinaryArray, BooleanArray, Date32Array, Float32Array, Float64Array, + Int16Array, Int32Array, Int64Array, Int8Array, LargeBinaryArray, LargeStringArray, + RecordBatchReader, StringArray, TimestampMicrosecondArray, TimestampMillisecondArray, + TimestampNanosecondArray, TimestampSecondArray, }, buffer::NullBuffer, compute::concat_batches, - datatypes::{DataType as ArrowDataType, Field, Schema}, + datatypes::{DataType as ArrowDataType, Field, Schema, TimeUnit}, }; use bytes::Bytes; use prost::Message; @@ -512,6 +529,147 @@ mod tests { assert_eq!(expected, actual); } + #[test] + fn test_roundtrip_write_timestamps() { + let second = vec![Some(-2), Some(-1), Some(0), None, Some(1), Some(2)]; + let millisecond = vec![ + Some(-1_001), + Some(-1_000), + Some(-1_999), + None, + Some(1_001), + Some(1_234), + ]; + let microsecond = vec![ + Some(-1_001_001), + Some(-1_000_000), + Some(-999_999), + None, + Some(1_001_001), + Some(1_234_567), + ]; + let nanosecond = vec![ + Some(-1_001_001_001), + Some(-1_000_000_000), + Some(-999_999_999), + None, + Some(1_001_001_001), + Some(1_234_567_890), + ]; + + let arrays: Vec = vec![ + Arc::new(TimestampSecondArray::from(second.clone())), + Arc::new(TimestampMillisecondArray::from(millisecond.clone())), + Arc::new(TimestampMicrosecondArray::from(microsecond.clone())), + Arc::new(TimestampNanosecondArray::from(nanosecond.clone())), + Arc::new(TimestampSecondArray::from(second).with_timezone("UTC")), + Arc::new(TimestampMillisecondArray::from(millisecond).with_timezone("UTC")), + Arc::new(TimestampMicrosecondArray::from(microsecond).with_timezone("UTC")), + Arc::new(TimestampNanosecondArray::from(nanosecond).with_timezone("UTC")), + ]; + let schema = Arc::new(Schema::new( + arrays + .iter() + .enumerate() + .map(|(index, array)| { + Field::new( + format!("timestamp_{index}"), + array.data_type().clone(), + true, + ) + }) + .collect::>(), + )); + let batch = RecordBatch::try_new(schema.clone(), arrays).unwrap(); + + let mut file = Vec::new(); + let mut writer = ArrowWriterBuilder::new(&mut file, schema.clone()) + .try_build() + .unwrap(); + writer.write(&batch.slice(0, 3)).unwrap(); + writer.flush_stripe().unwrap(); + writer.write(&batch.slice(3, 3)).unwrap(); + writer.close().unwrap(); + + let file = Bytes::from(file); + let builder = ArrowReaderBuilder::try_new(file.clone()).unwrap(); + let children = builder.file_metadata().root_data_type().children(); + for child in &children[..4] { + assert!(matches!(child.data_type(), OrcDataType::Timestamp { .. })); + } + for child in &children[4..] { + assert!(matches!( + child.data_type(), + OrcDataType::TimestampWithLocalTimezone { .. } + )); + } + + let stripes = builder.file_metadata().stripe_metadatas(); + assert_eq!(stripes.len(), 2); + for stripe in stripes { + let footer_start = stripe.footer_offset() as usize; + let footer_end = footer_start + stripe.footer_length() as usize; + let mut decoded_footer = Vec::new(); + Decompressor::new( + file.slice(footer_start..footer_end), + builder.file_metadata().compression(), + Vec::new(), + ) + .read_to_end(&mut decoded_footer) + .unwrap(); + let footer = proto::StripeFooter::decode(decoded_footer.as_slice()).unwrap(); + assert_eq!(footer.writer_timezone.as_deref(), Some("UTC")); + assert_eq!( + footer + .streams + .iter() + .filter(|stream| stream.kind() == proto::stream::Kind::Secondary) + .count(), + 8 + ); + } + + let rows = builder + .with_schema(schema.clone()) + .build() + .collect::, _>>() + .unwrap(); + let actual = concat_batches(&schema, rows.iter()).unwrap(); + assert_eq!(batch, actual); + } + + #[test] + fn test_write_timestamps_with_inconsistent_null_buffers() { + let schema = Arc::new(Schema::new(vec![Field::new( + "timestamp", + ArrowDataType::Timestamp(TimeUnit::Nanosecond, None), + true, + )])); + + let without_nulls = Arc::new(TimestampNanosecondArray::from(vec![1, 2, 3])); + let with_nulls = Arc::new(TimestampNanosecondArray::from(vec![None, Some(4), None])); + assert!(without_nulls.nulls().is_none()); + assert!(with_nulls.nulls().is_some()); + + let batch1 = RecordBatch::try_new(schema.clone(), vec![without_nulls]).unwrap(); + let batch2 = RecordBatch::try_new(schema.clone(), vec![with_nulls]).unwrap(); + let expected = RecordBatch::try_new( + schema, + vec![Arc::new(TimestampNanosecondArray::from(vec![ + Some(1), + Some(2), + Some(3), + None, + Some(4), + None, + ]))], + ) + .unwrap(); + + let rows = roundtrip(&[batch1, batch2]); + assert_eq!(expected, rows[0]); + } + #[test] fn test_write_small_stripes() { // Set small stripe size to ensure writing across multiple stripes works diff --git a/src/encoding/mod.rs b/src/encoding/mod.rs index 57633c9..2021cd4 100644 --- a/src/encoding/mod.rs +++ b/src/encoding/mod.rs @@ -34,7 +34,9 @@ mod util; /// Encodes primitive values into an internal buffer, usually with a specialized run length /// encoding for better compression. pub trait PrimitiveValueEncoder: EstimateMemory { - fn new() -> Self; + fn new() -> Self + where + Self: Sized; fn write_one(&mut self, value: V); diff --git a/src/encoding/timestamp.rs b/src/encoding/timestamp.rs index be01bd6..3d59e33 100644 --- a/src/encoding/timestamp.rs +++ b/src/encoding/timestamp.rs @@ -18,15 +18,108 @@ use std::marker::PhantomData; use arrow::datatypes::{ArrowTimestampType, TimeUnit}; +use bytes::Bytes; use snafu::ensure; use crate::{ - encoding::PrimitiveValueDecoder, - error::{DecodeTimestampSnafu, Result}, + encoding::{PrimitiveValueDecoder, PrimitiveValueEncoder}, + error::{DecodeTimestampSnafu, EncodeTimestampSnafu, Result}, + memory::EstimateMemory, }; const NANOSECONDS_IN_SECOND: i64 = 1_000_000_000; +/// Encodes Arrow timestamps into the two integer streams used by ORC timestamps. +pub struct TimestampEncoder { + base_from_epoch: i64, + data: Box>, + secondary: Box>, + _marker: PhantomData, +} + +impl TimestampEncoder { + pub fn new( + base_from_epoch: i64, + data: Box>, + secondary: Box>, + ) -> Self { + Self { + base_from_epoch, + data, + secondary, + _marker: PhantomData, + } + } + + pub fn encode(&mut self, values: &[T::Native]) -> Result<()> { + let (units_per_second, nanoseconds_per_unit) = match T::UNIT { + TimeUnit::Second => (1, NANOSECONDS_IN_SECOND), + TimeUnit::Millisecond => (1_000, 1_000_000), + TimeUnit::Microsecond => (1_000_000, 1_000), + TimeUnit::Nanosecond => (NANOSECONDS_IN_SECOND, 1), + }; + + let mut data = Vec::with_capacity(values.len()); + let mut secondary = Vec::with_capacity(values.len()); + for &value in values { + let mut seconds = value.div_euclid(units_per_second); + let nanoseconds = value.rem_euclid(units_per_second) * nanoseconds_per_unit; + + // ORC-763 makes this interval immediately before the Unix epoch + // impossible to represent without changing the decoded value. + ensure!( + seconds != -1 || nanoseconds <= 999_999, + EncodeTimestampSnafu { + value, + time_unit: T::UNIT, + } + ); + if seconds < 0 && nanoseconds > 999_999 { + seconds += 1; + } + let Some(seconds) = seconds.checked_sub(self.base_from_epoch) else { + return EncodeTimestampSnafu { + value, + time_unit: T::UNIT, + } + .fail(); + }; + + // The last three bits store how many trailing decimal zeros were removed. + let encoded_nanoseconds = if nanoseconds == 0 { + 0 + } else if nanoseconds % 100 != 0 { + nanoseconds << 3 + } else { + let mut nanoseconds = nanoseconds / 100; + let mut trailing = 1; + while nanoseconds % 10 == 0 && trailing < 7 { + nanoseconds /= 10; + trailing += 1; + } + (nanoseconds << 3) | trailing + }; + + data.push(seconds); + secondary.push(encoded_nanoseconds); + } + + self.data.write_slice(&data); + self.secondary.write_slice(&secondary); + Ok(()) + } + + pub fn take_inner(&mut self) -> (Bytes, Bytes) { + (self.data.take_inner(), self.secondary.take_inner()) + } +} + +impl EstimateMemory for TimestampEncoder { + fn estimate_memory_size(&self) -> usize { + self.data.estimate_memory_size() + self.secondary.estimate_memory_size() + } +} + pub struct TimestampDecoder { base_from_epoch: i64, data: Box + Send>, @@ -195,3 +288,88 @@ fn decode_timestamp_as_i128(base: i64, seconds_since_orc_base: i64, nanoseconds: let (nanoseconds_since_epoch, _, _) = decode(base, seconds_since_orc_base, nanoseconds); nanoseconds_since_epoch } + +#[cfg(test)] +mod tests { + use std::io::Cursor; + + use arrow::datatypes::{ + TimestampMicrosecondType, TimestampMillisecondType, TimestampNanosecondType, + TimestampSecondType, + }; + + use crate::encoding::integer::{ + rle_v2::{RleV2Decoder, RleV2Encoder}, + SignedEncoding, UnsignedEncoding, + }; + + use super::*; + + const ORC_EPOCH_SECONDS: i64 = 1_420_070_400; + + fn encoder() -> TimestampEncoder { + TimestampEncoder::new( + ORC_EPOCH_SECONDS, + Box::new(RleV2Encoder::::new()), + Box::new(RleV2Encoder::::new()), + ) + } + + fn roundtrip(values: &[i64]) { + let mut encoder = encoder::(); + encoder.encode(values).unwrap(); + let (data, secondary) = encoder.take_inner(); + + let data = Box::new(RleV2Decoder::::new(Cursor::new( + data, + ))); + let secondary = Box::new(RleV2Decoder::::new(Cursor::new( + secondary, + ))); + let mut decoder = TimestampDecoder::::new(ORC_EPOCH_SECONDS, data, secondary); + let mut actual = vec![0; values.len()]; + decoder.decode(&mut actual).unwrap(); + assert_eq!(values, actual); + } + + #[test] + fn test_timestamp_encoder_roundtrip() { + roundtrip::(&[-2, -1, 0, 1, 2, 1_420_070_400]); + roundtrip::(&[-2_001, -2_000, -1_999, -1_000, 0, 1_001, 1_234]); + roundtrip::(&[ + -2_000_001, -1_001_001, -1_000_000, -999_999, 0, 1_001_001, 1_234_567, + ]); + roundtrip::(&[ + -2_000_000_001, + -1_001_001_001, + -1_000_000_000, + -999_999_999, + 0, + 1_001_001_001, + 1_234_567_890, + ]); + } + + #[test] + fn test_timestamp_encoder_rejects_orc_epoch_overflow() { + let mut encoder = encoder::(); + assert!(encoder.encode(&[i64::MIN]).is_err()); + } + + #[test] + fn test_timestamp_encoder_rejects_orc_763_gap() { + let mut milliseconds = encoder::(); + assert!(milliseconds.encode(&[-1_000]).is_ok()); + assert!(milliseconds.encode(&[-999]).is_err()); + assert!(milliseconds.encode(&[-1]).is_err()); + assert!(milliseconds.encode(&[0]).is_ok()); + + let mut microseconds = encoder::(); + assert!(microseconds.encode(&[-999_001]).is_ok()); + assert!(microseconds.encode(&[-999_000]).is_err()); + + let mut nanoseconds = encoder::(); + assert!(nanoseconds.encode(&[-999_000_001]).is_ok()); + assert!(nanoseconds.encode(&[-999_000_000]).is_err()); + } +} diff --git a/src/error.rs b/src/error.rs index f1705d9..3b5a48f 100644 --- a/src/error.rs +++ b/src/error.rs @@ -70,6 +70,18 @@ pub enum OrcError { to_time_unit: TimeUnit, }, + #[snafu(display( + "Cannot encode timestamp value {} in {:?} as an ORC timestamp", + value, + time_unit, + ))] + EncodeTimestamp { + #[snafu(implicit)] + location: Location, + value: i64, + time_unit: TimeUnit, + }, + #[snafu(display( "String/Binary data size ({} bytes) exceeds maximum offset size ({}) \ with current batch size {}. Please reduce the batch size to avoid offset overflow.", diff --git a/src/writer/column.rs b/src/writer/column.rs index 352b805..37d2b6d 100644 --- a/src/writer/column.rs +++ b/src/writer/column.rs @@ -20,8 +20,9 @@ use std::marker::PhantomData; use arrow::{ array::{Array, ArrayRef, AsArray}, datatypes::{ - ArrowPrimitiveType, ByteArrayType, Date32Type, Float32Type, Float64Type, GenericBinaryType, - GenericStringType, Int16Type, Int32Type, Int64Type, Int8Type, + ArrowPrimitiveType, ArrowTimestampType, ByteArrayType, Date32Type, Float32Type, + Float64Type, GenericBinaryType, GenericStringType, Int16Type, Int32Type, Int64Type, + Int8Type, }, }; use bytes::{BufMut, BytesMut}; @@ -32,6 +33,7 @@ use crate::{ byte::ByteRleEncoder, float::FloatEncoder, integer::{rle_v2::RleV2Encoder, NInt, SignedEncoding, UnsignedEncoding}, + timestamp::TimestampEncoder, PrimitiveValueEncoder, }, error::Result, @@ -164,6 +166,95 @@ impl> ColumnStripeEnc } } +/// Seconds from the ORC epoch of 1 January 2015 to the Unix epoch. +const ORC_EPOCH_UTC_SECONDS_SINCE_UNIX_EPOCH: i64 = 1_420_070_400; + +/// Encodes an Arrow timestamp into the DATA and SECONDARY streams used by ORC. +pub struct TimestampColumnEncoder { + encoder: TimestampEncoder, + present: Option, + encoded_count: usize, +} + +impl TimestampColumnEncoder { + pub fn new() -> Self { + Self { + encoder: TimestampEncoder::new( + ORC_EPOCH_UTC_SECONDS_SINCE_UNIX_EPOCH, + Box::new(RleV2Encoder::::new()), + Box::new(RleV2Encoder::::new()), + ), + present: None, + encoded_count: 0, + } + } +} + +impl EstimateMemory for TimestampColumnEncoder { + fn estimate_memory_size(&self) -> usize { + self.encoder.estimate_memory_size() + + self + .present + .as_ref() + .map(|present| present.estimate_memory_size()) + .unwrap_or(0) + } +} + +impl ColumnStripeEncoder for TimestampColumnEncoder { + fn encode_array(&mut self, array: &ArrayRef) -> Result<()> { + let array = array.as_primitive::(); + if let Some(null_buffer) = array.nulls() { + let values = null_buffer + .valid_indices() + .map(|index| array.value(index)) + .collect::>(); + self.encoder.encode(&values)?; + + let encoded_count = self.encoded_count; + let present = self.present.get_or_insert_with(|| { + let mut present = BooleanEncoder::new(); + present.extend_present(encoded_count); + present + }); + present.extend(null_buffer); + } else { + self.encoder.encode(array.values())?; + if let Some(present) = self.present.as_mut() { + present.extend_present(array.len()); + } + } + self.encoded_count += array.len() - array.null_count(); + Ok(()) + } + + fn column_encoding(&self) -> ColumnEncoding { + ColumnEncoding::DirectV2 + } + + fn finish(&mut self) -> Vec { + let (data, secondary) = self.encoder.take_inner(); + let mut streams = vec![ + Stream { + kind: StreamType::Data, + bytes: data, + }, + Stream { + kind: StreamType::Secondary, + bytes: secondary, + }, + ]; + self.encoded_count = 0; + if let Some(present) = self.present.as_mut() { + streams.push(Stream { + kind: StreamType::Present, + bytes: present.finish(), + }); + } + streams + } +} + pub struct BooleanColumnEncoder { encoder: BooleanEncoder, /// Lazily initialized once we encounter an [`Array`] with a [`NullBuffer`]. diff --git a/src/writer/stripe.rs b/src/writer/stripe.rs index e449941..0197c49 100644 --- a/src/writer/stripe.rs +++ b/src/writer/stripe.rs @@ -18,7 +18,10 @@ use std::io::Write; use arrow::array::RecordBatch; -use arrow::datatypes::{DataType as ArrowDataType, FieldRef, SchemaRef}; +use arrow::datatypes::{ + DataType as ArrowDataType, FieldRef, SchemaRef, TimeUnit, TimestampMicrosecondType, + TimestampMillisecondType, TimestampNanosecondType, TimestampSecondType, +}; use prost::Message; use snafu::ResultExt; @@ -31,7 +34,7 @@ use super::column::{ BinaryColumnEncoder, BooleanColumnEncoder, ByteColumnEncoder, ColumnStripeEncoder, DateColumnEncoder, DoubleColumnEncoder, FloatColumnEncoder, Int16ColumnEncoder, Int32ColumnEncoder, Int64ColumnEncoder, LargeBinaryColumnEncoder, LargeStringColumnEncoder, - StringColumnEncoder, + StringColumnEncoder, TimestampColumnEncoder, }; use super::{ColumnEncoding, StreamType}; @@ -72,6 +75,7 @@ pub struct StripeWriter { columns: Vec>, pub row_count: usize, compressor: Compressor, + timezone: Option, } impl EstimateMemory for StripeWriter { @@ -83,13 +87,19 @@ impl EstimateMemory for StripeWriter { } impl StripeWriter { - pub fn new(writer: W, schema: &SchemaRef, compressor: Compressor) -> Self { + pub fn new( + writer: W, + schema: &SchemaRef, + compressor: Compressor, + timezone: Option, + ) -> Self { let columns = schema.fields().iter().map(create_encoder).collect(); Self { writer, columns, row_count: 0, compressor, + timezone, } } @@ -148,7 +158,7 @@ impl StripeWriter { let stripe_footer = proto::StripeFooter { streams, columns: column_encodings, - writer_timezone: None, + writer_timezone: self.timezone.clone(), encryption: vec![], }; @@ -185,6 +195,28 @@ fn create_encoder(field: &FieldRef) -> Box { ArrowDataType::Int32 => Box::new(Int32ColumnEncoder::new(ColumnEncoding::DirectV2)), ArrowDataType::Int64 => Box::new(Int64ColumnEncoder::new(ColumnEncoding::DirectV2)), ArrowDataType::Date32 => Box::new(DateColumnEncoder::new(ColumnEncoding::DirectV2)), + // TODO: Support non-UTC Arrow timestamp timezones. ORC TIMESTAMP_INSTANT + // does not preserve the original Arrow timezone metadata. + ArrowDataType::Timestamp(TimeUnit::Second, timezone) + if matches!(timezone.as_deref(), None | Some("UTC")) => + { + Box::new(TimestampColumnEncoder::::new()) + } + ArrowDataType::Timestamp(TimeUnit::Millisecond, timezone) + if matches!(timezone.as_deref(), None | Some("UTC")) => + { + Box::new(TimestampColumnEncoder::::new()) + } + ArrowDataType::Timestamp(TimeUnit::Microsecond, timezone) + if matches!(timezone.as_deref(), None | Some("UTC")) => + { + Box::new(TimestampColumnEncoder::::new()) + } + ArrowDataType::Timestamp(TimeUnit::Nanosecond, timezone) + if matches!(timezone.as_deref(), None | Some("UTC")) => + { + Box::new(TimestampColumnEncoder::::new()) + } ArrowDataType::Utf8 => Box::new(StringColumnEncoder::new()), ArrowDataType::LargeUtf8 => Box::new(LargeStringColumnEncoder::new()), ArrowDataType::Binary => Box::new(BinaryColumnEncoder::new()), diff --git a/tests/writer_timestamp/main.rs b/tests/writer_timestamp/main.rs new file mode 100644 index 0000000..4867b3c --- /dev/null +++ b/tests/writer_timestamp/main.rs @@ -0,0 +1,91 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::{fs, process::Command, sync::Arc}; + +use arrow::{ + array::{RecordBatch, TimestampMicrosecondArray}, + datatypes::{DataType, Field, Schema, TimeUnit}, +}; +use orc_rust::{compression::CompressionType, ArrowWriterBuilder}; + +#[test] +#[ignore = "requires ./venv/bin/python with pyorc installed"] +fn timestamp_file_is_readable_by_pyorc() { + let values = vec![ + Some(-2_208_988_800_000_000), + Some(-1_001_000), + Some(0), + None, + Some(1_420_070_400_000_000), + Some(1_709_210_096_123_456), + ]; + let schema = Arc::new(Schema::new(vec![ + Field::new( + "timestamp", + DataType::Timestamp(TimeUnit::Microsecond, None), + true, + ), + Field::new( + "timestamp_instant", + DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())), + true, + ), + ])); + let batch = RecordBatch::try_new( + schema, + vec![ + Arc::new(TimestampMicrosecondArray::from(values.clone())), + Arc::new(TimestampMicrosecondArray::from(values).with_timezone("UTC")), + ], + ) + .unwrap(); + + let path = std::env::temp_dir().join(format!( + "orc-rust-writer-timestamp-{}.orc", + std::process::id() + )); + let file = fs::File::create(&path).unwrap(); + let mut writer = ArrowWriterBuilder::new(file, batch.schema()) + .with_compression(CompressionType::Zstd) + .try_build() + .unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + + let status = Command::new(format!("{}/venv/bin/python", env!("CARGO_MANIFEST_DIR"))) + .arg("-c") + .arg( + "import datetime, pyorc, sys\n\ + with open(sys.argv[1], 'rb') as source:\n\ + \treader = pyorc.Reader(source)\n\ + \tassert str(reader.schema) == 'struct', reader.schema\n\ + \tvalues = [datetime.datetime(1900, 1, 1, tzinfo=datetime.timezone.utc),\n\ + \t datetime.datetime(1969, 12, 31, 23, 59, 58, 999000, tzinfo=datetime.timezone.utc),\n\ + \t datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc),\n\ + \t None,\n\ + \t datetime.datetime(2015, 1, 1, tzinfo=datetime.timezone.utc),\n\ + \t datetime.datetime(2024, 2, 29, 12, 34, 56, 123456, tzinfo=datetime.timezone.utc)]\n\ + \tassert list(reader) == [(value, value) for value in values]", + ) + .arg(&path) + .status() + .unwrap(); + assert!(status.success(), "pyorc failed to read timestamp file"); + + fs::remove_file(path).unwrap(); +}