Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
230 changes: 223 additions & 7 deletions src/arrow_writer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -86,10 +86,16 @@ impl<W: Write> ArrowWriterBuilder<W> {
/// Construct an [`ArrowWriter`] ready to encode [`RecordBatch`]es into
/// an ORC file.
pub fn try_build(mut self) -> Result<ArrowWriter<W>> {
let timezone = self
.schema
.fields()
.iter()
.any(|field| matches!(field.data_type(), ArrowDataType::Timestamp(_, None)))
.then(|| "UTC".to_owned());
let compressor = Compressor::new(self.compression, self.compression_block_size)?;
// Required magic "ORC" bytes at start of file
self.writer.write_all(b"ORC").context(IoSnafu)?;
let writer = StripeWriter::new(self.writer, &self.schema, compressor);
let writer = StripeWriter::new(self.writer, &self.schema, compressor, timezone);
Ok(ArrowWriter {
writer,
schema: self.schema,
Expand Down Expand Up @@ -229,6 +235,20 @@ fn serialize_schema(schema: &SchemaRef) -> Vec<proto::Type> {
kind: Some(proto::r#type::Kind::Long.into()),
..Default::default()
},
ArrowDataType::Date32 => proto::Type {
kind: Some(proto::r#type::Kind::Date.into()),
..Default::default()
},
ArrowDataType::Timestamp(_, None) => proto::Type {
kind: Some(proto::r#type::Kind::Timestamp.into()),
..Default::default()
},
ArrowDataType::Timestamp(_, Some(timezone)) if timezone.as_ref() == "UTC" => {
proto::Type {
kind: Some(proto::r#type::Kind::TimestampInstant.into()),
..Default::default()
}
}
ArrowDataType::Utf8 | ArrowDataType::LargeUtf8 => proto::Type {
kind: Some(proto::r#type::Kind::String.into()),
..Default::default()
Expand Down Expand Up @@ -302,26 +322,28 @@ mod tests {

use arrow::{
array::{
Array, BinaryArray, BooleanArray, Float32Array, Float64Array, Int16Array, Int32Array,
Int64Array, Int8Array, LargeBinaryArray, LargeStringArray, RecordBatchReader,
StringArray,
Array, ArrayRef, BinaryArray, BooleanArray, Date32Array, Float32Array, Float64Array,
Int16Array, Int32Array, Int64Array, Int8Array, LargeBinaryArray, LargeStringArray,
RecordBatchReader, StringArray, TimestampMicrosecondArray, TimestampMillisecondArray,
TimestampNanosecondArray, TimestampSecondArray,
},
buffer::NullBuffer,
compute::concat_batches,
datatypes::{DataType as ArrowDataType, Field, Schema},
datatypes::{DataType as ArrowDataType, Field, Schema, TimeUnit},
};
use bytes::Bytes;
use prost::Message;

use crate::{
compression::{CompressionType, Decompressor},
schema::DataType as OrcDataType,
stripe::Stripe,
ArrowReaderBuilder,
};

use super::*;

fn roundtrip(batches: &[RecordBatch]) -> Vec<RecordBatch> {
fn encode(batches: &[RecordBatch]) -> Bytes {
let mut f = vec![];
let mut writer = ArrowWriterBuilder::new(&mut f, batches[0].schema())
.try_build()
Expand All @@ -331,7 +353,11 @@ mod tests {
}
writer.close().unwrap();

let f = Bytes::from(f);
Bytes::from(f)
}

fn roundtrip(batches: &[RecordBatch]) -> Vec<RecordBatch> {
let f = encode(batches);
let reader = ArrowReaderBuilder::try_new(f).unwrap().build();
reader.collect::<Result<Vec<_>, _>>().unwrap()
}
Expand Down Expand Up @@ -454,6 +480,196 @@ mod tests {
assert_eq!(batch, rows[0]);
}

#[test]
fn test_roundtrip_write_date32() {
let schema = Arc::new(Schema::new(vec![Field::new(
"date",
ArrowDataType::Date32,
true,
)]));

let batch_without_nulls = RecordBatch::try_new(
schema.clone(),
vec![Arc::new(Date32Array::from(vec![-1, 0, 1, 11_016]))],
)
.unwrap();
let batch_with_nulls = RecordBatch::try_new(
schema.clone(),
vec![Arc::new(Date32Array::from(vec![
Some(18_321),
None,
Some(19_782),
]))],
)
.unwrap();

let file = encode(&[batch_without_nulls, batch_with_nulls]);
let builder = ArrowReaderBuilder::try_new(file).unwrap();
assert!(matches!(
builder.file_metadata().root_data_type().children()[0].data_type(),
OrcDataType::Date { .. }
));

let rows = builder.build().collect::<Result<Vec<_>, _>>().unwrap();
let actual = concat_batches(&schema, rows.iter()).unwrap();
let expected = RecordBatch::try_new(
schema,
vec![Arc::new(Date32Array::from(vec![
Some(-1),
Some(0),
Some(1),
Some(11_016),
Some(18_321),
None,
Some(19_782),
]))],
)
.unwrap();

assert_eq!(expected, actual);
}

#[test]
fn test_roundtrip_write_timestamps() {
let second = vec![Some(-2), Some(-1), Some(0), None, Some(1), Some(2)];
let millisecond = vec![
Some(-1_001),
Some(-1_000),
Some(-1_999),
None,
Some(1_001),
Some(1_234),
];
let microsecond = vec![
Some(-1_001_001),
Some(-1_000_000),
Some(-999_999),
None,
Some(1_001_001),
Some(1_234_567),
];
let nanosecond = vec![
Some(-1_001_001_001),
Some(-1_000_000_000),
Some(-999_999_999),
None,
Some(1_001_001_001),
Some(1_234_567_890),
];

let arrays: Vec<ArrayRef> = vec![
Arc::new(TimestampSecondArray::from(second.clone())),
Arc::new(TimestampMillisecondArray::from(millisecond.clone())),
Arc::new(TimestampMicrosecondArray::from(microsecond.clone())),
Arc::new(TimestampNanosecondArray::from(nanosecond.clone())),
Arc::new(TimestampSecondArray::from(second).with_timezone("UTC")),
Arc::new(TimestampMillisecondArray::from(millisecond).with_timezone("UTC")),
Arc::new(TimestampMicrosecondArray::from(microsecond).with_timezone("UTC")),
Arc::new(TimestampNanosecondArray::from(nanosecond).with_timezone("UTC")),
];
let schema = Arc::new(Schema::new(
arrays
.iter()
.enumerate()
.map(|(index, array)| {
Field::new(
format!("timestamp_{index}"),
array.data_type().clone(),
true,
)
})
.collect::<Vec<_>>(),
));
let batch = RecordBatch::try_new(schema.clone(), arrays).unwrap();

let mut file = Vec::new();
let mut writer = ArrowWriterBuilder::new(&mut file, schema.clone())
.try_build()
.unwrap();
writer.write(&batch.slice(0, 3)).unwrap();
writer.flush_stripe().unwrap();
writer.write(&batch.slice(3, 3)).unwrap();
writer.close().unwrap();

let file = Bytes::from(file);
let builder = ArrowReaderBuilder::try_new(file.clone()).unwrap();
let children = builder.file_metadata().root_data_type().children();
for child in &children[..4] {
assert!(matches!(child.data_type(), OrcDataType::Timestamp { .. }));
}
for child in &children[4..] {
assert!(matches!(
child.data_type(),
OrcDataType::TimestampWithLocalTimezone { .. }
));
}

let stripes = builder.file_metadata().stripe_metadatas();
assert_eq!(stripes.len(), 2);
for stripe in stripes {
let footer_start = stripe.footer_offset() as usize;
let footer_end = footer_start + stripe.footer_length() as usize;
let mut decoded_footer = Vec::new();
Decompressor::new(
file.slice(footer_start..footer_end),
builder.file_metadata().compression(),
Vec::new(),
)
.read_to_end(&mut decoded_footer)
.unwrap();
let footer = proto::StripeFooter::decode(decoded_footer.as_slice()).unwrap();
assert_eq!(footer.writer_timezone.as_deref(), Some("UTC"));
assert_eq!(
footer
.streams
.iter()
.filter(|stream| stream.kind() == proto::stream::Kind::Secondary)
.count(),
8
);
}

let rows = builder
.with_schema(schema.clone())
.build()
.collect::<Result<Vec<_>, _>>()
.unwrap();
let actual = concat_batches(&schema, rows.iter()).unwrap();
assert_eq!(batch, actual);
}

#[test]
fn test_write_timestamps_with_inconsistent_null_buffers() {
let schema = Arc::new(Schema::new(vec![Field::new(
"timestamp",
ArrowDataType::Timestamp(TimeUnit::Nanosecond, None),
true,
)]));

let without_nulls = Arc::new(TimestampNanosecondArray::from(vec![1, 2, 3]));
let with_nulls = Arc::new(TimestampNanosecondArray::from(vec![None, Some(4), None]));
assert!(without_nulls.nulls().is_none());
assert!(with_nulls.nulls().is_some());

let batch1 = RecordBatch::try_new(schema.clone(), vec![without_nulls]).unwrap();
let batch2 = RecordBatch::try_new(schema.clone(), vec![with_nulls]).unwrap();
let expected = RecordBatch::try_new(
schema,
vec![Arc::new(TimestampNanosecondArray::from(vec![
Some(1),
Some(2),
Some(3),
None,
Some(4),
None,
]))],
)
.unwrap();

let rows = roundtrip(&[batch1, batch2]);
assert_eq!(expected, rows[0]);
}

#[test]
fn test_write_small_stripes() {
// Set small stripe size to ensure writing across multiple stripes works
Expand Down
4 changes: 3 additions & 1 deletion src/encoding/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,9 @@ mod util;
/// Encodes primitive values into an internal buffer, usually with a specialized run length
/// encoding for better compression.
pub trait PrimitiveValueEncoder<V: Copy>: EstimateMemory {
fn new() -> Self;
fn new() -> Self
where
Self: Sized;

fn write_one(&mut self, value: V);

Expand Down
Loading
Loading