Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ ordered-float = { version = "5.1.0", default-features = false }
rand = { version = "0.9.2", features = ["small_rng"] }
serde = { version = "1.0", features = ["derive"] }
serde_json = { version = "1.0", default-features = false, features = ["std"] }
simdutf8 = "0.1.5"
zmij = "1.0"

[dev-dependencies]
Expand All @@ -57,6 +58,9 @@ sqlite = []
arbitrary_precision = ["serde_json/arbitrary_precision"]
preserve_order = ["serde_json/preserve_order"]

[lints.clippy]
undocumented_unsafe_blocks = "warn"

[[bench]]
name = "parser"
harness = false
Expand Down
2 changes: 1 addition & 1 deletion benches/get_path.rs
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ fn add_benchmark(c: &mut Criterion) {
let bytes = read(&format!("./data/{}.json", test_suite.file));

let val = jsonb::parse_value(&bytes).unwrap();
let jsonb_bytes = val.to_vec();
let jsonb_bytes = val.to_vec().unwrap();

c.bench_function(
&format!(
Expand Down
4 changes: 2 additions & 2 deletions benches/strip_nulls.rs
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ fn strip_nulls_deser(data: &[u8]) {
let mut buf = Vec::new();
let mut json = from_slice(data).unwrap();
strip_value_nulls(&mut json);
json.write_to_vec(&mut buf);
json.write_to_vec(&mut buf).unwrap();
assert!(!buf.is_empty());
}

Expand Down Expand Up @@ -60,7 +60,7 @@ fn add_benchmark(c: &mut Criterion) {
for path in paths {
let file = format!("{}", path.unwrap().path().display());
let bytes = read(&file);
let json = from_slice(&bytes).unwrap().to_vec();
let json = from_slice(&bytes).unwrap().to_vec().unwrap();

c.bench_function(&format!("strip_nulls_deser[{file}]"), |b| {
b.iter(|| strip_nulls_deser(&json));
Expand Down
20 changes: 20 additions & 0 deletions src/constants.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,26 @@
// JSON text constants
pub(crate) const UNICODE_LEN: usize = 4;

/// Maximum nesting depth of arrays and objects when parsing JSON text or
/// walking JSONB recursively. A level takes up to about 1.5 KiB of stack in
/// release builds, so 512 levels fit in tokio's default 2 MiB thread stack.
pub(crate) const MAX_NESTING_DEPTH: usize = 512;

/// Returns the nesting depth after entering one more array or object.
///
/// # Errors
///
/// Returns `Error::RecursionLimitExceeded` if the new depth would exceed
/// [`MAX_NESTING_DEPTH`].
#[inline]
pub(crate) fn enter_container(depth: usize) -> crate::error::Result<usize> {
if depth >= MAX_NESTING_DEPTH {
Err(crate::Error::RecursionLimitExceeded)
} else {
Ok(depth + 1)
}
}

// JSON text escape characters constants
pub(crate) const BS: char = '\x5C'; // \\ Backslash
pub(crate) const QU: char = '\x22'; // \" Double quotation mark
Expand Down
47 changes: 19 additions & 28 deletions src/core/databend/builder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ use byteorder::BigEndian;
use byteorder::WriteBytesExt;

use super::constants::*;
use super::jentry::encode_container_header;
use super::jentry::JEntry;
use crate::core::ExtensionItem;
use crate::core::JsonbItem;
Expand Down Expand Up @@ -60,7 +61,7 @@ impl<'a> ArrayBuilder<'a> {

pub(crate) fn build(self) -> Result<OwnedJsonb> {
let mut buf = Vec::new();
let header = ARRAY_CONTAINER_TAG | self.items.len() as u32;
let header = encode_container_header(ARRAY_CONTAINER_TAG, self.items.len())?;
buf.write_u32::<BigEndian>(header)?;

let mut jentry_index = reserve_jentries(&mut buf, self.items.len() * 4);
Expand Down Expand Up @@ -93,11 +94,6 @@ impl<'a> ArrayDistinctBuilder<'a> {
}
}

pub(crate) fn push_raw_jsonb(&mut self, raw: RawJsonb<'a>) {
let item = JsonbItem::Raw(raw);
self.push_jsonb_item(item);
}

pub(crate) fn pop_jsonb_item(&mut self, item: JsonbItem<'a>) -> Option<()> {
if let Some(cnt) = self.item_map.get_mut(&item) {
if *cnt > 0 {
Expand All @@ -108,14 +104,9 @@ impl<'a> ArrayDistinctBuilder<'a> {
None
}

pub(crate) fn pop_raw_jsonb(&mut self, raw: RawJsonb<'a>) -> Option<()> {
let item = JsonbItem::Raw(raw);
self.pop_jsonb_item(item)
}

pub(crate) fn build(self) -> Result<OwnedJsonb> {
let mut buf = Vec::new();
let header = ARRAY_CONTAINER_TAG | self.items.len() as u32;
let header = encode_container_header(ARRAY_CONTAINER_TAG, self.items.len())?;
buf.write_u32::<BigEndian>(header)?;

let mut jentry_index = reserve_jentries(&mut buf, self.items.len() * 4);
Expand Down Expand Up @@ -163,12 +154,12 @@ impl<'a> ObjectBuilder<'a> {

pub(crate) fn build(self) -> Result<OwnedJsonb> {
let mut buf = Vec::new();
let header = OBJECT_CONTAINER_TAG | self.entries.len() as u32;
let header = encode_container_header(OBJECT_CONTAINER_TAG, self.entries.len())?;
buf.write_u32::<BigEndian>(header)?;

let mut jentry_index = reserve_jentries(&mut buf, self.entries.len() * 8);
for key in self.entries.keys() {
append_object_key(&mut buf, &mut jentry_index, key);
append_object_key(&mut buf, &mut jentry_index, key)?;
}
for (_, item) in self.entries.into_iter() {
append_jsonb_item(&mut buf, &mut jentry_index, item)?;
Expand All @@ -192,24 +183,24 @@ where
K: AsRef<str>,
{
let mut buf = Vec::new();
let header = OBJECT_CONTAINER_TAG | entries.len() as u32;
let header = encode_container_header(OBJECT_CONTAINER_TAG, entries.len())?;
buf.write_u32::<BigEndian>(header)?;

let mut jentry_index = reserve_jentries(&mut buf, entries.len() * 8);
for (key, _) in entries.iter() {
append_object_key(&mut buf, &mut jentry_index, key.as_ref());
append_object_key(&mut buf, &mut jentry_index, key.as_ref())?;
}
for (_, item) in entries.into_iter() {
append_jsonb_item(&mut buf, &mut jentry_index, item)?;
}
Ok(OwnedJsonb::new(buf))
}

fn append_object_key(buf: &mut Vec<u8>, jentry_index: &mut usize, key: &str) {
let key_len = key.len();
fn append_object_key(buf: &mut Vec<u8>, jentry_index: &mut usize, key: &str) -> Result<()> {
let jentry = JEntry::make_string_jentry(key.len())?;
buf.extend_from_slice(key.as_bytes());
let jentry = JEntry::make_string_jentry(key_len);
replace_jentry(buf, jentry, jentry_index)
replace_jentry(buf, jentry, jentry_index);
Ok(())
}

fn append_jsonb_item(buf: &mut Vec<u8>, jentry_index: &mut usize, item: JsonbItem) -> Result<()> {
Expand All @@ -228,30 +219,30 @@ fn append_jsonb_item(buf: &mut Vec<u8>, jentry_index: &mut usize, item: JsonbIte
}
JsonbItem::Number(num) => match num {
NumberItem::Raw(data) => {
let jentry = JEntry::make_number_jentry(data.len());
let jentry = JEntry::make_number_jentry(data.len())?;
replace_jentry(buf, jentry, jentry_index);
buf.extend_from_slice(data);
}
NumberItem::Number(num) => {
let len = num.compact_encode(&mut *buf)?;
let jentry = JEntry::make_number_jentry(len);
let jentry = JEntry::make_number_jentry(len)?;
replace_jentry(buf, jentry, jentry_index);
}
},
JsonbItem::String(data) => {
let jentry = JEntry::make_string_jentry(data.len());
let jentry = JEntry::make_string_jentry(data.len())?;
replace_jentry(buf, jentry, jentry_index);
buf.extend_from_slice(data.as_bytes());
}
JsonbItem::Extension(ext) => match ext {
ExtensionItem::Raw(data) => {
let jentry = JEntry::make_extension_jentry(data.len());
let jentry = JEntry::make_extension_jentry(data.len())?;
replace_jentry(buf, jentry, jentry_index);
buf.extend_from_slice(data);
}
ExtensionItem::Extension(ext) => {
let len = ext.compact_encode(&mut *buf)?;
let jentry = JEntry::make_extension_jentry(len);
let jentry = JEntry::make_extension_jentry(len)?;
replace_jentry(buf, jentry, jentry_index);
}
},
Expand Down Expand Up @@ -282,7 +273,7 @@ fn append_raw_jsonb_data(
replace_jentry(buf, scalar_jentry, jentry_index);
buf.extend_from_slice(data);
} else {
let jentry = JEntry::make_container_jentry(raw_jsonb.len());
let jentry = JEntry::make_container_jentry(raw_jsonb.len())?;
replace_jentry(buf, jentry, jentry_index);
buf.extend_from_slice(raw_jsonb.data);
}
Expand Down Expand Up @@ -330,7 +321,7 @@ mod tests {
let mut from_encoder = Vec::new();
{
let value = init_object(vec![("arr", Value::Array(vec![Value::Bool(false)]))]);
value.write_to_vec(&mut from_encoder);
value.write_to_vec(&mut from_encoder).unwrap();
}
assert_eq!(from_builder, from_encoder);
}
Expand All @@ -355,7 +346,7 @@ mod tests {
"obj",
init_object(vec![("field", Value::Bool(true))]),
)]);
value.write_to_vec(&mut from_encoder);
value.write_to_vec(&mut from_encoder).unwrap();
}
assert_eq!(from_builder, from_encoder);
}
Expand Down
Loading
Loading