From ba5908c3f02b21d668f492b75ec30078e033df89 Mon Sep 17 00:00:00 2001 From: baishen Date: Fri, 13 Nov 2026 21:17:23 +0800 Subject: [PATCH 1/3] fix: Make number arithmetic and JSONPath evaluation exact and bounded Numbers: - Compute integer arithmetic in `i128`, so mixed `Int64`/`UInt64` operands no longer wrap (`u64::MAX + (-1)` returned `-2`), and report overflow from `neg`. - Convert decimals to the nearest `f64` with one rounding, through an exact division, Eisel-Lemire, or parsing `e-`. JSONPath: - Limit parenthesis depth to 64 and `&&`/`||` operators to 256 when parsing (`Error::RecursionLimitExceeded`). - Bound one evaluation to 1,000,000 work units and to copying 32 MiB beyond the document size (`Error::JsonPathTooComplex`), and limit container nesting while selecting. - Parse float literals with a fraction, an exponent, or `inf`/`nan`. - Make slices outside the array select nothing instead of looping. - Quote and escape names in `Display` for JSONPath and key paths, so the output parses back to the same path. --- src/constants.rs | 20 ++ src/eisel_lemire.rs | 425 +++++++++++++++++++++++++ src/error.rs | 8 +- src/jsonpath/mod.rs | 3 + src/jsonpath/parser.rs | 222 +++++++++++--- src/jsonpath/path.rs | 151 +++++++-- src/jsonpath/selector.rs | 235 +++++++++----- src/keypath.rs | 13 +- src/lib.rs | 1 + src/number.rs | 528 ++++++++++++++++++++++---------- tests/allocation.rs | 36 ++- tests/it/functions.rs | 4 + tests/it/jsonpath_parser.rs | 124 ++++++++ tests/it/keypath_parser.rs | 34 ++ tests/it/malformed.rs | 348 +++++++++++++++++++++ tests/it/number.rs | 427 +++++++++++++++++++++++++- tests/it/parser.rs | 56 +++- tests/it/testdata/json_path.txt | 2 +- 18 files changed, 2300 insertions(+), 337 deletions(-) create mode 100644 src/eisel_lemire.rs diff --git a/src/constants.rs b/src/constants.rs index 8d782d7..3241502 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -22,6 +22,26 @@ pub(crate) const UNICODE_LEN: usize = 4; /// `to_value`, `partial_cmp` and others overflow 2 MiB at 512 levels. pub(crate) const MAX_NESTING_DEPTH: usize = 512; +/// Maximum parenthesis depth in a JSONPath expression. The JSONPath parser +/// uses much more stack per level than data traversal. +pub(crate) const MAX_JSON_PATH_DEPTH: usize = 64; + +/// Maximum number of `&&` and `||` operators in a JSONPath expression. Each +/// one adds a level to the parsed tree, which is processed recursively. +/// The parser counts them because unquoted field names may contain `&&`. +pub(crate) const MAX_JSON_PATH_LOGICAL_OPERATORS: usize = 256; + +/// Maximum work units for one JSONPath evaluation. Each selected item, filter +/// value, node visited by `.**` and compared pair costs one unit; without a +/// limit, chained `.**` steps grow combinatorially with the nesting depth. +pub(crate) const MAX_JSON_PATH_ITEMS: usize = 1_000_000; + +/// Bytes one JSONPath evaluation may copy beyond the document size. A value +/// is copied once per selected ancestor, so `$.**.**` on 500 nested arrays +/// around a 1 KiB string copies about 300 MB. The budget is this plus the +/// document size, so any single value can be copied at least once. +pub(crate) const MAX_JSON_PATH_EXTRA_COPY_BYTES: usize = 32 * 1024 * 1024; + /// Returns the nesting depth after entering one more array or object. /// /// # Errors diff --git a/src/eisel_lemire.rs b/src/eisel_lemire.rs new file mode 100644 index 0000000..53bc763 --- /dev/null +++ b/src/eisel_lemire.rs @@ -0,0 +1,425 @@ +// Adapted from fast-float2 0.2.4 (`src/binary.rs` and `src/table.rs`), +// https://github.com/Alexhuszagh/fast-float-rust, by Ivan Smirnov and +// Alex Huszagh, licensed under MIT OR Apache-2.0 and used here under the +// Apache License, Version 2.0. fast-float2 is a port of fast_float by +// Daniel Lemire, https://github.com/fastfloat/fast_float. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! The Eisel-Lemire algorithm: the `f64` nearest to `mantissa / 10^scale`, +//! computed from the integers without formatting them as text. +//! +//! fast-float2 keeps `compute_float` private and only parses text, so it is +//! copied here. Changes from upstream: +//! - specialized to `f64`; +//! - returns `None` where upstream returns `power2 = -1`, so the caller falls +//! back to parsing text; +//! - the power-of-five table keeps only the exponents -255..=0 that a `u8` +//! scale can produce, indexed by the scale so the lookup needs no bounds +//! check. +//! +//! Upstream's `parse_float` also re-runs the algorithm with `mantissa + 1` +//! when the text had more than 19 digits; a decimal's mantissa is exact, so +//! that step does not apply. + +const MANTISSA_EXPLICIT_BITS: usize = 52; +const MIN_EXPONENT_ROUND_TO_EVEN: i64 = -4; +const MAX_EXPONENT_ROUND_TO_EVEN: i64 = 23; +const MINIMUM_EXPONENT: i32 = -1023; +const INFINITE_POWER: i32 = 0x7FF; +const SIGN_INDEX: usize = 63; + +/// Returns the `f64` nearest to `mantissa / 10^scale`, negated if `negative`, +/// or `None` if the 128-bit product cannot decide the rounding. On `None`, +/// parse the decimal text instead. +#[inline] +pub(crate) fn decimal_to_f64(negative: bool, mantissa: u64, scale: u8) -> Option { + let (mantissa, power2) = compute_float(scale, mantissa)?; + let mut word = mantissa; + word |= (power2 as u64) << MANTISSA_EXPLICIT_BITS; + if negative { + word |= 1_u64 << SIGN_INDEX; + } + Some(f64::from_bits(word)) +} + +/// Upstream `compute_float::(q, w)` with `q = -scale`, returning the +/// `(mantissa, power2)` of the adjusted mantissa. +#[inline] +fn compute_float(scale: u8, mut w: u64) -> Option<(u64, i32)> { + let q = -i64::from(scale); + // Upstream also returns zero or infinity for `q` outside -342..=308, + // which a `u8` scale cannot reach. + if w == 0 { + return Some((0, 0)); + } + let lz = w.leading_zeros(); + w <<= lz; + let (lo, hi) = compute_product_approx(scale, w, MANTISSA_EXPLICIT_BITS + 3); + if lo == 0xFFFF_FFFF_FFFF_FFFF { + let inside_safe_exponent = (-27..=55).contains(&q); + if !inside_safe_exponent { + return None; + } + } + let upperbit = (hi >> 63) as i32; + let mut mantissa = hi >> (upperbit + 64 - MANTISSA_EXPLICIT_BITS as i32 - 3); + let mut power2 = power(q as i32) + upperbit - lz as i32 - MINIMUM_EXPONENT; + if power2 <= 0 { + if -power2 + 1 >= 64 { + return Some((0, 0)); + } + mantissa >>= -power2 + 1; + mantissa += mantissa & 1; + mantissa >>= 1; + power2 = (mantissa >= (1_u64 << MANTISSA_EXPLICIT_BITS)) as i32; + return Some((mantissa, power2)); + } + if lo <= 1 + && (MIN_EXPONENT_ROUND_TO_EVEN..=MAX_EXPONENT_ROUND_TO_EVEN).contains(&q) + && mantissa & 3 == 1 + && (mantissa << (upperbit + 64 - MANTISSA_EXPLICIT_BITS as i32 - 3)) == hi + { + mantissa &= !1_u64; + } + mantissa += mantissa & 1; + mantissa >>= 1; + if mantissa >= (2_u64 << MANTISSA_EXPLICIT_BITS) { + mantissa = 1_u64 << MANTISSA_EXPLICIT_BITS; + power2 += 1; + } + mantissa &= !(1_u64 << MANTISSA_EXPLICIT_BITS); + if power2 >= INFINITE_POWER { + return Some((0, INFINITE_POWER)); + } + Some((mantissa, power2)) +} + +#[inline] +fn power(q: i32) -> i32 { + (q.wrapping_mul(152_170 + 65536) >> 16) + 63 +} + +#[inline] +fn full_multiplication(a: u64, b: u64) -> (u64, u64) { + let r = (a as u128) * (b as u128); + (r as u64, (r >> 64) as u64) +} + +// This will compute or rather approximate w * 5**q and return a pair of 64-bit +// words approximating the result, with the "high" part corresponding to the +// most significant bits and the low part corresponding to the least significant +// bits. +#[inline] +fn compute_product_approx(scale: u8, w: u64, precision: usize) -> (u64, u64) { + debug_assert!(precision <= 64); + + let mask = if precision < 64 { + 0xFFFF_FFFF_FFFF_FFFF_u64 >> precision + } else { + 0xFFFF_FFFF_FFFF_FFFF_u64 + }; + // Entry `i` holds 5^(i - 255), so 5^-scale is at `255 - scale`. + let (lo5, hi5) = POWER_OF_FIVE_128[usize::from(u8::MAX - scale)]; + let (mut first_lo, mut first_hi) = full_multiplication(w, lo5); + if first_hi & mask == mask { + let (_, second_hi) = full_multiplication(w, hi5); + first_lo = first_lo.wrapping_add(second_hi); + if second_hi > first_lo { + first_hi += 1; + } + } + (first_lo, first_hi) +} + +/// 128-bit approximations of 5^-255 through 5^0, copied from upstream's +/// `POWER_OF_FIVE_128` (entries 87..=342, upstream starts at 5^-342). +#[allow(clippy::unreadable_literal)] +const POWER_OF_FIVE_128: [(u64, u64); 256] = [ + (0xf03d93eebc589f88, 0x793555ab7eba27ca), + (0x96267c7535b763b5, 0x4bc1558b2f3458de), + (0xbbb01b9283253ca2, 0x9eb1aaedfb016f16), + (0xea9c227723ee8bcb, 0x465e15a979c1cadc), + (0x92a1958a7675175f, 0xbfacd89ec191ec9), + (0xb749faed14125d36, 0xcef980ec671f667b), + (0xe51c79a85916f484, 0x82b7e12780e7401a), + (0x8f31cc0937ae58d2, 0xd1b2ecb8b0908810), + (0xb2fe3f0b8599ef07, 0x861fa7e6dcb4aa15), + (0xdfbdcece67006ac9, 0x67a791e093e1d49a), + (0x8bd6a141006042bd, 0xe0c8bb2c5c6d24e0), + (0xaecc49914078536d, 0x58fae9f773886e18), + (0xda7f5bf590966848, 0xaf39a475506a899e), + (0x888f99797a5e012d, 0x6d8406c952429603), + (0xaab37fd7d8f58178, 0xc8e5087ba6d33b83), + (0xd5605fcdcf32e1d6, 0xfb1e4a9a90880a64), + (0x855c3be0a17fcd26, 0x5cf2eea09a55067f), + (0xa6b34ad8c9dfc06f, 0xf42faa48c0ea481e), + (0xd0601d8efc57b08b, 0xf13b94daf124da26), + (0x823c12795db6ce57, 0x76c53d08d6b70858), + (0xa2cb1717b52481ed, 0x54768c4b0c64ca6e), + (0xcb7ddcdda26da268, 0xa9942f5dcf7dfd09), + (0xfe5d54150b090b02, 0xd3f93b35435d7c4c), + (0x9efa548d26e5a6e1, 0xc47bc5014a1a6daf), + (0xc6b8e9b0709f109a, 0x359ab6419ca1091b), + (0xf867241c8cc6d4c0, 0xc30163d203c94b62), + (0x9b407691d7fc44f8, 0x79e0de63425dcf1d), + (0xc21094364dfb5636, 0x985915fc12f542e4), + (0xf294b943e17a2bc4, 0x3e6f5b7b17b2939d), + (0x979cf3ca6cec5b5a, 0xa705992ceecf9c42), + (0xbd8430bd08277231, 0x50c6ff782a838353), + (0xece53cec4a314ebd, 0xa4f8bf5635246428), + (0x940f4613ae5ed136, 0x871b7795e136be99), + (0xb913179899f68584, 0x28e2557b59846e3f), + (0xe757dd7ec07426e5, 0x331aeada2fe589cf), + (0x9096ea6f3848984f, 0x3ff0d2c85def7621), + (0xb4bca50b065abe63, 0xfed077a756b53a9), + (0xe1ebce4dc7f16dfb, 0xd3e8495912c62894), + (0x8d3360f09cf6e4bd, 0x64712dd7abbbd95c), + (0xb080392cc4349dec, 0xbd8d794d96aacfb3), + (0xdca04777f541c567, 0xecf0d7a0fc5583a0), + (0x89e42caaf9491b60, 0xf41686c49db57244), + (0xac5d37d5b79b6239, 0x311c2875c522ced5), + (0xd77485cb25823ac7, 0x7d633293366b828b), + (0x86a8d39ef77164bc, 0xae5dff9c02033197), + (0xa8530886b54dbdeb, 0xd9f57f830283fdfc), + (0xd267caa862a12d66, 0xd072df63c324fd7b), + (0x8380dea93da4bc60, 0x4247cb9e59f71e6d), + (0xa46116538d0deb78, 0x52d9be85f074e608), + (0xcd795be870516656, 0x67902e276c921f8b), + (0x806bd9714632dff6, 0xba1cd8a3db53b6), + (0xa086cfcd97bf97f3, 0x80e8a40eccd228a4), + (0xc8a883c0fdaf7df0, 0x6122cd128006b2cd), + (0xfad2a4b13d1b5d6c, 0x796b805720085f81), + (0x9cc3a6eec6311a63, 0xcbe3303674053bb0), + (0xc3f490aa77bd60fc, 0xbedbfc4411068a9c), + (0xf4f1b4d515acb93b, 0xee92fb5515482d44), + (0x991711052d8bf3c5, 0x751bdd152d4d1c4a), + (0xbf5cd54678eef0b6, 0xd262d45a78a0635d), + (0xef340a98172aace4, 0x86fb897116c87c34), + (0x9580869f0e7aac0e, 0xd45d35e6ae3d4da0), + (0xbae0a846d2195712, 0x8974836059cca109), + (0xe998d258869facd7, 0x2bd1a438703fc94b), + (0x91ff83775423cc06, 0x7b6306a34627ddcf), + (0xb67f6455292cbf08, 0x1a3bc84c17b1d542), + (0xe41f3d6a7377eeca, 0x20caba5f1d9e4a93), + (0x8e938662882af53e, 0x547eb47b7282ee9c), + (0xb23867fb2a35b28d, 0xe99e619a4f23aa43), + (0xdec681f9f4c31f31, 0x6405fa00e2ec94d4), + (0x8b3c113c38f9f37e, 0xde83bc408dd3dd04), + (0xae0b158b4738705e, 0x9624ab50b148d445), + (0xd98ddaee19068c76, 0x3badd624dd9b0957), + (0x87f8a8d4cfa417c9, 0xe54ca5d70a80e5d6), + (0xa9f6d30a038d1dbc, 0x5e9fcf4ccd211f4c), + (0xd47487cc8470652b, 0x7647c3200069671f), + (0x84c8d4dfd2c63f3b, 0x29ecd9f40041e073), + (0xa5fb0a17c777cf09, 0xf468107100525890), + (0xcf79cc9db955c2cc, 0x7182148d4066eeb4), + (0x81ac1fe293d599bf, 0xc6f14cd848405530), + (0xa21727db38cb002f, 0xb8ada00e5a506a7c), + (0xca9cf1d206fdc03b, 0xa6d90811f0e4851c), + (0xfd442e4688bd304a, 0x908f4a166d1da663), + (0x9e4a9cec15763e2e, 0x9a598e4e043287fe), + (0xc5dd44271ad3cdba, 0x40eff1e1853f29fd), + (0xf7549530e188c128, 0xd12bee59e68ef47c), + (0x9a94dd3e8cf578b9, 0x82bb74f8301958ce), + (0xc13a148e3032d6e7, 0xe36a52363c1faf01), + (0xf18899b1bc3f8ca1, 0xdc44e6c3cb279ac1), + (0x96f5600f15a7b7e5, 0x29ab103a5ef8c0b9), + (0xbcb2b812db11a5de, 0x7415d448f6b6f0e7), + (0xebdf661791d60f56, 0x111b495b3464ad21), + (0x936b9fcebb25c995, 0xcab10dd900beec34), + (0xb84687c269ef3bfb, 0x3d5d514f40eea742), + (0xe65829b3046b0afa, 0xcb4a5a3112a5112), + (0x8ff71a0fe2c2e6dc, 0x47f0e785eaba72ab), + (0xb3f4e093db73a093, 0x59ed216765690f56), + (0xe0f218b8d25088b8, 0x306869c13ec3532c), + (0x8c974f7383725573, 0x1e414218c73a13fb), + (0xafbd2350644eeacf, 0xe5d1929ef90898fa), + (0xdbac6c247d62a583, 0xdf45f746b74abf39), + (0x894bc396ce5da772, 0x6b8bba8c328eb783), + (0xab9eb47c81f5114f, 0x66ea92f3f326564), + (0xd686619ba27255a2, 0xc80a537b0efefebd), + (0x8613fd0145877585, 0xbd06742ce95f5f36), + (0xa798fc4196e952e7, 0x2c48113823b73704), + (0xd17f3b51fca3a7a0, 0xf75a15862ca504c5), + (0x82ef85133de648c4, 0x9a984d73dbe722fb), + (0xa3ab66580d5fdaf5, 0xc13e60d0d2e0ebba), + (0xcc963fee10b7d1b3, 0x318df905079926a8), + (0xffbbcfe994e5c61f, 0xfdf17746497f7052), + (0x9fd561f1fd0f9bd3, 0xfeb6ea8bedefa633), + (0xc7caba6e7c5382c8, 0xfe64a52ee96b8fc0), + (0xf9bd690a1b68637b, 0x3dfdce7aa3c673b0), + (0x9c1661a651213e2d, 0x6bea10ca65c084e), + (0xc31bfa0fe5698db8, 0x486e494fcff30a62), + (0xf3e2f893dec3f126, 0x5a89dba3c3efccfa), + (0x986ddb5c6b3a76b7, 0xf89629465a75e01c), + (0xbe89523386091465, 0xf6bbb397f1135823), + (0xee2ba6c0678b597f, 0x746aa07ded582e2c), + (0x94db483840b717ef, 0xa8c2a44eb4571cdc), + (0xba121a4650e4ddeb, 0x92f34d62616ce413), + (0xe896a0d7e51e1566, 0x77b020baf9c81d17), + (0x915e2486ef32cd60, 0xace1474dc1d122e), + (0xb5b5ada8aaff80b8, 0xd819992132456ba), + (0xe3231912d5bf60e6, 0x10e1fff697ed6c69), + (0x8df5efabc5979c8f, 0xca8d3ffa1ef463c1), + (0xb1736b96b6fd83b3, 0xbd308ff8a6b17cb2), + (0xddd0467c64bce4a0, 0xac7cb3f6d05ddbde), + (0x8aa22c0dbef60ee4, 0x6bcdf07a423aa96b), + (0xad4ab7112eb3929d, 0x86c16c98d2c953c6), + (0xd89d64d57a607744, 0xe871c7bf077ba8b7), + (0x87625f056c7c4a8b, 0x11471cd764ad4972), + (0xa93af6c6c79b5d2d, 0xd598e40d3dd89bcf), + (0xd389b47879823479, 0x4aff1d108d4ec2c3), + (0x843610cb4bf160cb, 0xcedf722a585139ba), + (0xa54394fe1eedb8fe, 0xc2974eb4ee658828), + (0xce947a3da6a9273e, 0x733d226229feea32), + (0x811ccc668829b887, 0x806357d5a3f525f), + (0xa163ff802a3426a8, 0xca07c2dcb0cf26f7), + (0xc9bcff6034c13052, 0xfc89b393dd02f0b5), + (0xfc2c3f3841f17c67, 0xbbac2078d443ace2), + (0x9d9ba7832936edc0, 0xd54b944b84aa4c0d), + (0xc5029163f384a931, 0xa9e795e65d4df11), + (0xf64335bcf065d37d, 0x4d4617b5ff4a16d5), + (0x99ea0196163fa42e, 0x504bced1bf8e4e45), + (0xc06481fb9bcf8d39, 0xe45ec2862f71e1d6), + (0xf07da27a82c37088, 0x5d767327bb4e5a4c), + (0x964e858c91ba2655, 0x3a6a07f8d510f86f), + (0xbbe226efb628afea, 0x890489f70a55368b), + (0xeadab0aba3b2dbe5, 0x2b45ac74ccea842e), + (0x92c8ae6b464fc96f, 0x3b0b8bc90012929d), + (0xb77ada0617e3bbcb, 0x9ce6ebb40173744), + (0xe55990879ddcaabd, 0xcc420a6a101d0515), + (0x8f57fa54c2a9eab6, 0x9fa946824a12232d), + (0xb32df8e9f3546564, 0x47939822dc96abf9), + (0xdff9772470297ebd, 0x59787e2b93bc56f7), + (0x8bfbea76c619ef36, 0x57eb4edb3c55b65a), + (0xaefae51477a06b03, 0xede622920b6b23f1), + (0xdab99e59958885c4, 0xe95fab368e45eced), + (0x88b402f7fd75539b, 0x11dbcb0218ebb414), + (0xaae103b5fcd2a881, 0xd652bdc29f26a119), + (0xd59944a37c0752a2, 0x4be76d3346f0495f), + (0x857fcae62d8493a5, 0x6f70a4400c562ddb), + (0xa6dfbd9fb8e5b88e, 0xcb4ccd500f6bb952), + (0xd097ad07a71f26b2, 0x7e2000a41346a7a7), + (0x825ecc24c873782f, 0x8ed400668c0c28c8), + (0xa2f67f2dfa90563b, 0x728900802f0f32fa), + (0xcbb41ef979346bca, 0x4f2b40a03ad2ffb9), + (0xfea126b7d78186bc, 0xe2f610c84987bfa8), + (0x9f24b832e6b0f436, 0xdd9ca7d2df4d7c9), + (0xc6ede63fa05d3143, 0x91503d1c79720dbb), + (0xf8a95fcf88747d94, 0x75a44c6397ce912a), + (0x9b69dbe1b548ce7c, 0xc986afbe3ee11aba), + (0xc24452da229b021b, 0xfbe85badce996168), + (0xf2d56790ab41c2a2, 0xfae27299423fb9c3), + (0x97c560ba6b0919a5, 0xdccd879fc967d41a), + (0xbdb6b8e905cb600f, 0x5400e987bbc1c920), + (0xed246723473e3813, 0x290123e9aab23b68), + (0x9436c0760c86e30b, 0xf9a0b6720aaf6521), + (0xb94470938fa89bce, 0xf808e40e8d5b3e69), + (0xe7958cb87392c2c2, 0xb60b1d1230b20e04), + (0x90bd77f3483bb9b9, 0xb1c6f22b5e6f48c2), + (0xb4ecd5f01a4aa828, 0x1e38aeb6360b1af3), + (0xe2280b6c20dd5232, 0x25c6da63c38de1b0), + (0x8d590723948a535f, 0x579c487e5a38ad0e), + (0xb0af48ec79ace837, 0x2d835a9df0c6d851), + (0xdcdb1b2798182244, 0xf8e431456cf88e65), + (0x8a08f0f8bf0f156b, 0x1b8e9ecb641b58ff), + (0xac8b2d36eed2dac5, 0xe272467e3d222f3f), + (0xd7adf884aa879177, 0x5b0ed81dcc6abb0f), + (0x86ccbb52ea94baea, 0x98e947129fc2b4e9), + (0xa87fea27a539e9a5, 0x3f2398d747b36224), + (0xd29fe4b18e88640e, 0x8eec7f0d19a03aad), + (0x83a3eeeef9153e89, 0x1953cf68300424ac), + (0xa48ceaaab75a8e2b, 0x5fa8c3423c052dd7), + (0xcdb02555653131b6, 0x3792f412cb06794d), + (0x808e17555f3ebf11, 0xe2bbd88bbee40bd0), + (0xa0b19d2ab70e6ed6, 0x5b6aceaeae9d0ec4), + (0xc8de047564d20a8b, 0xf245825a5a445275), + (0xfb158592be068d2e, 0xeed6e2f0f0d56712), + (0x9ced737bb6c4183d, 0x55464dd69685606b), + (0xc428d05aa4751e4c, 0xaa97e14c3c26b886), + (0xf53304714d9265df, 0xd53dd99f4b3066a8), + (0x993fe2c6d07b7fab, 0xe546a8038efe4029), + (0xbf8fdb78849a5f96, 0xde98520472bdd033), + (0xef73d256a5c0f77c, 0x963e66858f6d4440), + (0x95a8637627989aad, 0xdde7001379a44aa8), + (0xbb127c53b17ec159, 0x5560c018580d5d52), + (0xe9d71b689dde71af, 0xaab8f01e6e10b4a6), + (0x9226712162ab070d, 0xcab3961304ca70e8), + (0xb6b00d69bb55c8d1, 0x3d607b97c5fd0d22), + (0xe45c10c42a2b3b05, 0x8cb89a7db77c506a), + (0x8eb98a7a9a5b04e3, 0x77f3608e92adb242), + (0xb267ed1940f1c61c, 0x55f038b237591ed3), + (0xdf01e85f912e37a3, 0x6b6c46dec52f6688), + (0x8b61313bbabce2c6, 0x2323ac4b3b3da015), + (0xae397d8aa96c1b77, 0xabec975e0a0d081a), + (0xd9c7dced53c72255, 0x96e7bd358c904a21), + (0x881cea14545c7575, 0x7e50d64177da2e54), + (0xaa242499697392d2, 0xdde50bd1d5d0b9e9), + (0xd4ad2dbfc3d07787, 0x955e4ec64b44e864), + (0x84ec3c97da624ab4, 0xbd5af13bef0b113e), + (0xa6274bbdd0fadd61, 0xecb1ad8aeacdd58e), + (0xcfb11ead453994ba, 0x67de18eda5814af2), + (0x81ceb32c4b43fcf4, 0x80eacf948770ced7), + (0xa2425ff75e14fc31, 0xa1258379a94d028d), + (0xcad2f7f5359a3b3e, 0x96ee45813a04330), + (0xfd87b5f28300ca0d, 0x8bca9d6e188853fc), + (0x9e74d1b791e07e48, 0x775ea264cf55347e), + (0xc612062576589dda, 0x95364afe032a819e), + (0xf79687aed3eec551, 0x3a83ddbd83f52205), + (0x9abe14cd44753b52, 0xc4926a9672793543), + (0xc16d9a0095928a27, 0x75b7053c0f178294), + (0xf1c90080baf72cb1, 0x5324c68b12dd6339), + (0x971da05074da7bee, 0xd3f6fc16ebca5e04), + (0xbce5086492111aea, 0x88f4bb1ca6bcf585), + (0xec1e4a7db69561a5, 0x2b31e9e3d06c32e6), + (0x9392ee8e921d5d07, 0x3aff322e62439fd0), + (0xb877aa3236a4b449, 0x9befeb9fad487c3), + (0xe69594bec44de15b, 0x4c2ebe687989a9b4), + (0x901d7cf73ab0acd9, 0xf9d37014bf60a11), + (0xb424dc35095cd80f, 0x538484c19ef38c95), + (0xe12e13424bb40e13, 0x2865a5f206b06fba), + (0x8cbccc096f5088cb, 0xf93f87b7442e45d4), + (0xafebff0bcb24aafe, 0xf78f69a51539d749), + (0xdbe6fecebdedd5be, 0xb573440e5a884d1c), + (0x89705f4136b4a597, 0x31680a88f8953031), + (0xabcc77118461cefc, 0xfdc20d2b36ba7c3e), + (0xd6bf94d5e57a42bc, 0x3d32907604691b4d), + (0x8637bd05af6c69b5, 0xa63f9a49c2c1b110), + (0xa7c5ac471b478423, 0xfcf80dc33721d54), + (0xd1b71758e219652b, 0xd3c36113404ea4a9), + (0x83126e978d4fdf3b, 0x645a1cac083126ea), + (0xa3d70a3d70a3d70a, 0x3d70a3d70a3d70a4), + (0xcccccccccccccccc, 0xcccccccccccccccd), + (0x8000000000000000, 0x0), +]; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn gives_up_when_the_product_is_ambiguous() { + // The integration tests check these inputs still convert correctly + // through the text fallback; this checks they reach it. + for (mantissa, scale) in [ + (14_999_775_663_049_351_983, 30), + (16_185_735_156_734_648_777, 35), + (18_309_340_392_174_713_697, 251), + ] { + assert_eq!(decimal_to_f64(false, mantissa, scale), None); + } + assert_eq!(decimal_to_f64(false, 0, 255), Some(0.0)); + assert_eq!(decimal_to_f64(true, 1, 0), Some(-1.0)); + } +} diff --git a/src/error.rs b/src/error.rs index 614756b..dcf35de 100644 --- a/src/error.rs +++ b/src/error.rs @@ -108,12 +108,18 @@ pub enum Error { ObjectDuplicateKey, UnexpectedType, - /// The input nests arrays or objects deeper than 512 levels. + /// The input nests arrays or objects deeper than 512 levels, or a JSONPath + /// expression nests parentheses deeper than 64 levels or has more than 256 + /// `&&` and `||` operators. RecursionLimitExceeded, /// A string, binary value, or nested array/object is 256 MiB or larger, or /// an array/object has 2^29 or more elements, which the JSONB length fields /// cannot represent. ValueTooLarge, + /// Evaluating a JSONPath expression selected or compared more than + /// 1,000,000 items, for example through chained `.**` steps, or copied + /// more than 32 MiB beyond the size of the document. + JsonPathTooComplex, Message(String), Syntax(ParseErrorCode, usize), diff --git a/src/jsonpath/mod.rs b/src/jsonpath/mod.rs index 2a55603..fc44e59 100644 --- a/src/jsonpath/mod.rs +++ b/src/jsonpath/mod.rs @@ -19,5 +19,8 @@ mod selector; pub use parser::parse_json_path; pub(crate) use parser::raw_string; pub(crate) use parser::string; +pub(crate) use path::write_name; +pub(crate) use path::write_quoted_str; +pub(crate) use path::write_raw_name; pub use path::*; pub use selector::Selector; diff --git a/src/jsonpath/parser.rs b/src/jsonpath/parser.rs index 57d9420..15cd81f 100644 --- a/src/jsonpath/parser.rs +++ b/src/jsonpath/parser.rs @@ -13,26 +13,28 @@ // limitations under the License. use std::borrow::Cow; +use std::cell::RefCell; +use std::collections::HashSet; use nom::branch::alt; use nom::bytes::complete::tag; use nom::bytes::complete::tag_no_case; use nom::character::complete::char; +use nom::character::complete::digit1; use nom::character::complete::i32; -use nom::character::complete::i64; use nom::character::complete::multispace0; -use nom::character::complete::u64; +use nom::character::complete::one_of; use nom::character::complete::u8; use nom::combinator::cond; use nom::combinator::map; use nom::combinator::map_res; use nom::combinator::opt; +use nom::combinator::recognize; use nom::combinator::value; use nom::error::Error as NomError; use nom::error::ErrorKind; use nom::multi::many0; use nom::multi::separated_list1; -use nom::number::complete::double; use nom::sequence::delimited; use nom::sequence::pair; use nom::sequence::preceded; @@ -41,6 +43,8 @@ use nom::sequence::terminated; use nom::IResult; use nom::Parser; +use crate::constants::MAX_JSON_PATH_DEPTH; +use crate::constants::MAX_JSON_PATH_LOGICAL_OPERATORS; use crate::constants::UNICODE_LEN; use crate::error::Error; use crate::jsonpath::path::*; @@ -49,7 +53,53 @@ use crate::util::parse_string; /// Parsing the input string to JSON Path. pub fn parse_json_path(input: &[u8]) -> Result, Error> { - match json_path(input) { + // Reject parentheses nested deeper than `MAX_JSON_PATH_DEPTH` before + // parsing. Every recursive rule of the grammar (`(expr)`, `? (expr)`, + // `exists(...)`) consumes a `(`, so this bounds the parser's recursion. + // Characters inside double-quoted string literals are ignored, and a + // backslash escapes the next byte both inside strings and in unquoted + // field names, as in the parser. + // + // `&&` and `||` chains are bounded by `logical_chain` instead: `&` does not + // end an unquoted field name, so only the parser can tell an operator from + // part of a name such as `$.a&&b`. + let mut depth = 0usize; + let mut in_string = false; + let mut escaped = false; + for &c in input { + if escaped { + escaped = false; + continue; + } + match c { + b'\\' => escaped = true, + b'"' => in_string = !in_string, + _ if in_string => {} + b'(' => { + depth += 1; + if depth > MAX_JSON_PATH_DEPTH { + return Err(Error::RecursionLimitExceeded); + } + } + b')' => depth = depth.saturating_sub(1), + _ => {} + } + } + + // `logical_chain` records the operators it folds. Start this call with + // none recorded, and leave none behind for the next call. + LOGICAL_OPERATORS.with(|operators| operators.borrow_mut().clear()); + let result = json_path(input); + let too_many_operators = LOGICAL_OPERATORS.with(|operators| { + let mut operators = operators.borrow_mut(); + let too_many = operators.len() > MAX_JSON_PATH_LOGICAL_OPERATORS; + operators.clear(); + too_many + }); + if too_many_operators { + return Err(Error::RecursionLimitExceeded); + } + match result { Ok((rest, json_path)) => { if !rest.is_empty() { return Err(Error::InvalidJsonPath); @@ -61,6 +111,12 @@ pub fn parse_json_path(input: &[u8]) -> Result, Error> { } } +thread_local! { + /// Positions in the input of the `&&` and `||` operators that the current + /// `parse_json_path` call has folded into expression trees. + static LOGICAL_OPERATORS: RefCell> = RefCell::new(HashSet::new()); +} + fn json_path(input: &[u8]) -> IResult<&[u8], JsonPath<'_>> { map( delimited(multispace0, expr_or_paths, multispace0), @@ -153,7 +209,9 @@ pub(crate) fn string(input: &[u8]) -> IResult<&[u8], Cow<'_, str>> { } } } - if i > 1 && i < input.len() { + // `i == input.len()` means the closing quote is missing. `i == 1` is the + // empty string `""`, a valid key and string literal. + if i < input.len() { if escapes == 0 { if let Ok(s) = std::str::from_utf8(&input[1..i]) { return Ok((&input[i + 1..], Cow::Borrowed(s))); @@ -423,14 +481,76 @@ fn path_value(input: &[u8]) -> IResult<&[u8], PathValue<'_>> { value(PathValue::Null, tag("null")), value(PathValue::Boolean(true), tag("true")), value(PathValue::Boolean(false), tag("false")), - map(u64, |v| PathValue::Number(Number::UInt64(v))), - map(i64, |v| PathValue::Number(Number::Int64(v))), - map(double, |v| PathValue::Number(Number::Float64(v))), + map(number, PathValue::Number), map(string, PathValue::String), )) .parse(input) } +/// Parses a number literal. +/// +/// The whole literal is recognized before its type is chosen. Trying the +/// integer parsers first would take `1` from `1.5` and leave `.5`, which no +/// rule accepts, so literals with a fraction or exponent never parsed. +fn number(input: &[u8]) -> IResult<&[u8], Number> { + alt(( + map_res(number_literal, |literal: &[u8]| { + // `number_literal` only matches ASCII. + let literal = std::str::from_utf8(literal).map_err(|_| ErrorKind::Float)?; + if !literal.contains(['.', 'e', 'E']) { + if let Ok(v) = literal.parse::() { + return Ok(Number::UInt64(v)); + } + if let Ok(v) = literal.parse::() { + return Ok(Number::Int64(v)); + } + } + literal + .parse::() + .map(Number::Float64) + .map_err(|_| ErrorKind::Float) + }), + // `inf`, `infinity` and `nan`, with an optional sign. `Display` + // writes negative infinity as `-inf`, so the sign must be accepted + // for formatted paths to parse again. + map_res( + recognize(( + opt(one_of("+-")), + alt(( + tag_no_case("infinity"), + tag_no_case("inf"), + tag_no_case("nan"), + )), + )), + |literal: &[u8]| { + std::str::from_utf8(literal) + .ok() + .and_then(|literal| literal.parse::().ok()) + .map(Number::Float64) + .ok_or(ErrorKind::Float) + }, + ), + )) + .parse(input) +} + +/// Recognizes `[+-]digits[.digits][e[+-]digits]` or `[+-].digits[e[+-]digits]`. +/// +/// Unlike `nom::number::complete::recognize_float`, an `e` without exponent +/// digits ends the literal instead of failing the whole parse, so `1e` can +/// still be read as a field name. +fn number_literal(input: &[u8]) -> IResult<&[u8], &[u8]> { + recognize(( + opt(one_of("+-")), + alt(( + recognize((digit1, opt((char('.'), opt(digit1))))), + recognize((char('.'), digit1)), + )), + opt((one_of("eE"), opt(one_of("+-")), digit1)), + )) + .parse(input) +} + fn inner_expr(input: &[u8], is_root_expr: bool) -> IResult<&[u8], Expr<'_>> { alt(( map(|i| expr_paths(i, is_root_expr), Expr::Paths), @@ -511,41 +631,59 @@ fn exists_paths(input: &[u8]) -> IResult<&[u8], Vec>> { } fn expr_and(input: &[u8], is_root_expr: bool) -> IResult<&[u8], Expr<'_>> { - map( - separated_list1(delimited(multispace0, tag("&&"), multispace0), |i| { - expr_atom(i, is_root_expr) - }), - |exprs| { - let mut expr = exprs[0].clone(); - for right in exprs.iter().skip(1) { - expr = Expr::BinaryOp { - op: BinaryOperator::And, - left: Box::new(expr), - right: Box::new(right.clone()), - }; - } - expr - }, - ) - .parse(input) + logical_chain(input, "&&", BinaryOperator::And, |i| { + expr_atom(i, is_root_expr) + }) } fn expr_or(input: &[u8], is_root_expr: bool) -> IResult<&[u8], Expr<'_>> { - map( - separated_list1(delimited(multispace0, tag("||"), multispace0), |i| { - expr_and(i, is_root_expr) - }), - |exprs| { - let mut expr = exprs[0].clone(); - for right in exprs.iter().skip(1) { - expr = Expr::BinaryOp { - op: BinaryOperator::Or, - left: Box::new(expr), - right: Box::new(right.clone()), - }; - } - expr - }, - ) - .parse(input) + logical_chain(input, "||", BinaryOperator::Or, |i| { + expr_and(i, is_root_expr) + }) +} + +/// Parses `operand`s separated by `operator` and folds them left into +/// `op` nodes. +/// +/// The fold builds a tree as deep as the chain is long, and formatting, +/// evaluating, cloning, comparing and dropping it recurse once per level. So +/// the chain fails before folding once the whole path has more than +/// [`MAX_JSON_PATH_LOGICAL_OPERATORS`] operators. Operators are recorded by +/// position, so a chain that backtracking parses again is counted once. +fn logical_chain<'a>( + input: &'a [u8], + operator: &'static str, + op: BinaryOperator, + operand: impl FnMut(&'a [u8]) -> IResult<&'a [u8], Expr<'a>>, +) -> IResult<&'a [u8], Expr<'a>> { + let mut positions = Vec::new(); + let separator = |i: &'a [u8]| { + let (i, _) = multispace0(i)?; + let position = i.as_ptr() as usize; + let (i, _) = tag(operator).parse(i)?; + let (i, _) = multispace0(i)?; + positions.push(position); + Ok((i, ())) + }; + let (rest, exprs) = separated_list1(separator, operand).parse(input)?; + // A separator can match before an operand that fails, ending the list. + positions.truncate(exprs.len() - 1); + let too_many = LOGICAL_OPERATORS.with(|operators| { + let mut operators = operators.borrow_mut(); + operators.extend(positions); + operators.len() > MAX_JSON_PATH_LOGICAL_OPERATORS + }); + if too_many { + // `Failure` keeps alternatives from parsing the chain again. + return Err(nom::Err::Failure(NomError::new(input, ErrorKind::TooLarge))); + } + // Fold left, moving the operands instead of cloning their subtrees. + let mut exprs = exprs.into_iter(); + let first = exprs.next().expect("separated_list1 yields an operand"); + let expr = exprs.fold(first, |left, right| Expr::BinaryOp { + op: op.clone(), + left: Box::new(left), + right: Box::new(right), + }); + Ok((rest, expr)) } diff --git a/src/jsonpath/path.rs b/src/jsonpath/path.rs index b40d4c7..17e5b6c 100644 --- a/src/jsonpath/path.rs +++ b/src/jsonpath/path.rs @@ -18,6 +18,7 @@ use std::cmp::Ordering; use std::collections::HashSet; use std::fmt::Display; use std::fmt::Formatter; +use std::fmt::Write; use crate::number::Number; @@ -103,38 +104,33 @@ impl ArrayIndex { /// /// A `HashSet` containing the indices to select. pub fn to_indices(&self, length: usize) -> HashSet { - let length = length as i32; + // `i64` arithmetic avoids overflow for extreme `i32` indices. + let length = length as i64; let mut indices = HashSet::new(); match self { ArrayIndex::Index(idx) => { let idx = Self::convert_index(idx, length); - if idx >= 0 && idx < length { + if (0..length).contains(&idx) { indices.insert(idx as usize); } } ArrayIndex::Slice((start, end)) => { - let start_idx = Self::convert_index(start, length); - let end_idx = Self::convert_index(end, length); - - let start_idx = if start_idx < 0 { 0 } else { start_idx as usize }; - let end_idx = if end_idx >= length { - (length - 1) as usize - } else { - end_idx as usize - }; + // Clamp to the array bounds; an empty range selects nothing. + let start_idx = Self::convert_index(start, length).max(0); + let end_idx = Self::convert_index(end, length).min(length - 1); for idx in start_idx..=end_idx { - indices.insert(idx); + indices.insert(idx as usize); } } } indices } - fn convert_index(index: &Index, length: i32) -> i32 { + fn convert_index(index: &Index, length: i64) -> i64 { match index { - Index::Index(idx) => *idx, - Index::LastIndex(idx) => length + *idx - 1, + Index::Index(idx) => i64::from(*idx), + Index::LastIndex(idx) => length + i64::from(*idx) - 1, } } } @@ -384,13 +380,17 @@ impl Display for Path<'_> { write!(f, "[*]")?; } Path::ColonField(field) => { - write!(f, ":{field}")?; + f.write_char(':')?; + write_name(f, field, false)?; } Path::DotField(field) => { - write!(f, ".{field}")?; + f.write_char('.')?; + write_name(f, field, false)?; } Path::ObjectField(field) => { - write!(f, "[\"{field}\"]")?; + f.write_char('[')?; + write_quoted_str(f, field)?; + f.write_char(']')?; } Path::ArrayIndices(indices) => { write!(f, "[")?; @@ -429,9 +429,7 @@ impl Display for PathValue<'_> { PathValue::Number(v) => { write!(f, "{v}") } - PathValue::String(v) => { - write!(f, "\"{v}\"") - } + PathValue::String(v) => write_quoted_str(f, v), PathValue::Raw(v) => { write!(f, "{}", v.to_string()) } @@ -513,22 +511,14 @@ impl Display for Expr<'_> { write!(f, "{v}")?; } Expr::BinaryOp { op, left, right } => { - if let Expr::BinaryOp { op: left_op, .. } = &**left { - if left_op == &BinaryOperator::And || left_op == &BinaryOperator::Or { - write!(f, "({left})")?; - } else { - write!(f, "{left}")?; - } + if left.needs_parens(op, false) { + write!(f, "({left})")?; } else { write!(f, "{left}")?; } write!(f, " {op} ")?; - if let Expr::BinaryOp { op: right_op, .. } = &**right { - if right_op == &BinaryOperator::And || right_op == &BinaryOperator::Or { - write!(f, "({right})")?; - } else { - write!(f, "{right}")?; - } + if right.needs_parens(op, true) { + write!(f, "({right})")?; } else { write!(f, "{right}")?; } @@ -547,3 +537,98 @@ impl Display for Expr<'_> { Ok(()) } } + +impl Expr<'_> { + /// Returns whether this expression, an operand of `parent`, must be + /// parenthesized for the formatted text to parse back to the same tree. + /// + /// The parser gives `&&` higher precedence than `||` and reads both left to + /// right, so `a || b || c` is `(a || b) || c`. Parenthesizing only where the + /// tree differs from that keeps long chains flat; wrapping every logical + /// operand turned a chain of 100 `||` into 100 nested parentheses, past the + /// parser's nesting limit. + fn needs_parens(&self, parent: &BinaryOperator, is_right: bool) -> bool { + let Expr::BinaryOp { op: child, .. } = self else { + return false; + }; + let is_logical = + |op: &BinaryOperator| matches!(op, BinaryOperator::And | BinaryOperator::Or); + if !is_logical(child) { + return false; + } + match parent { + // `&&` binds tighter, so it never needs parentheses under `||`. + BinaryOperator::Or => child == &BinaryOperator::Or && is_right, + BinaryOperator::And => child == &BinaryOperator::Or || is_right, + // Comparison and arithmetic operands are never logical expressions + // when parsed, so keep the parentheses that mark such a tree. + _ => true, + } + } +} + +/// Characters that end an unquoted name in the JSONPath and key-path grammars. +const RAW_NAME_DELIMITERS: &str = " ,.:{}[]()?@$|<>!=+-*/%\"'"; + +/// Writes `s` as a double-quoted string literal that the path parsers read back +/// as the same string. +pub(crate) fn write_quoted_str(f: &mut impl Write, s: &str) -> std::fmt::Result { + f.write_char('"')?; + for c in s.chars() { + match c { + '"' => f.write_str("\\\"")?, + '\\' => f.write_str("\\\\")?, + c if c.is_control() => write!(f, "\\u{:04X}", u32::from(c))?, + c => f.write_char(c)?, + } + } + f.write_char('"') +} + +/// Returns whether `name` cannot be written as an unquoted name as is. +/// +/// Such characters would end the name, be skipped as whitespace, or start a +/// different token. With `leading_digit_is_index`, a leading digit also counts, +/// so that a key-path name is not read back as an array index. +fn needs_escaping(c: char, first: bool, leading_digit_is_index: bool) -> bool { + c == '\\' + || c.is_whitespace() + || c.is_control() + || RAW_NAME_DELIMITERS.contains(c) + || (leading_digit_is_index && first && c.is_ascii_digit()) +} + +/// Writes `name` unquoted, escaping as `\uXXXX` any character that would not +/// read back unchanged. +pub(crate) fn write_raw_name( + f: &mut impl Write, + name: &str, + leading_digit_is_index: bool, +) -> std::fmt::Result { + for (i, c) in name.chars().enumerate() { + if needs_escaping(c, i == 0, leading_digit_is_index) { + write!(f, "\\u{:04X}", u32::from(c))?; + } else { + f.write_char(c)?; + } + } + Ok(()) +} + +/// Writes `name` unquoted when it reads back unchanged, and quoted otherwise. +pub(crate) fn write_name( + f: &mut impl Write, + name: &str, + leading_digit_is_index: bool, +) -> std::fmt::Result { + let plain = !name.is_empty() + && !name + .chars() + .enumerate() + .any(|(i, c)| needs_escaping(c, i == 0, leading_digit_is_index)); + if plain { + f.write_str(name) + } else { + write_quoted_str(f, name) + } +} diff --git a/src/jsonpath/selector.rs b/src/jsonpath/selector.rs index 6d87d41..0230ba4 100644 --- a/src/jsonpath/selector.rs +++ b/src/jsonpath/selector.rs @@ -16,10 +16,15 @@ use std::borrow::Cow; use std::cmp::Ordering; use std::collections::VecDeque; +use crate::constants::enter_container; +use crate::constants::MAX_JSON_PATH_EXTRA_COPY_BYTES; +use crate::constants::MAX_JSON_PATH_ITEMS; use crate::core::ArrayBuilder; use crate::core::ArrayIterator; +use crate::core::ExtensionItem; use crate::core::JsonbItem; use crate::core::JsonbItemType; +use crate::core::NumberItem; use crate::core::ObjectValueIterator; use crate::error::Result; use crate::jsonpath::ArrayIndex; @@ -29,6 +34,7 @@ use crate::jsonpath::JsonPath; use crate::jsonpath::Path; use crate::jsonpath::PathValue; use crate::jsonpath::RecursiveLevel; +use crate::jsonpath::RecursiveLevelEnd; use crate::jsonpath::UnaryOperator; use crate::number::Number; use crate::to_owned_jsonb; @@ -39,7 +45,19 @@ use crate::RawJsonb; #[derive(Debug)] enum ExprValue<'a> { Values(Vec>), - Value(Box>), + /// A literal, borrowed from the path so that a large string literal is not + /// copied once per evaluated item. + Value(&'a PathValue<'a>), +} + +impl<'a> ExprValue<'a> { + /// Borrows the values to compare. + fn as_slice(&self) -> &[PathValue<'a>] { + match self { + ExprValue::Values(vals) => vals, + ExprValue::Value(val) => std::slice::from_ref(*val), + } + } } impl ExprValue<'_> { @@ -55,8 +73,8 @@ impl ExprValue<'_> { _ => Err(Error::InvalidJsonPath), } } - ExprValue::Value(val) => match *val { - PathValue::Number(num) => Ok(num), + ExprValue::Value(val) => match val { + PathValue::Number(num) => Ok(num.clone()), _ => Err(Error::InvalidJsonPath), }, } @@ -75,8 +93,8 @@ impl ExprValue<'_> { } Ok(nums) } - ExprValue::Value(val) => match *val { - PathValue::Number(num) => Ok(vec![num]), + ExprValue::Value(val) => match val { + PathValue::Number(num) => Ok(vec![num.clone()]), _ => Err(Error::InvalidJsonPath), }, } @@ -92,6 +110,11 @@ pub struct Selector<'a> { root_jsonb: RawJsonb<'a>, /// A queue holding the JSONB items that match the path criteria during execution. items: VecDeque>, + /// Work units left for the current evaluation; see [`MAX_JSON_PATH_ITEMS`]. + budget: usize, + /// Bytes the current evaluation may still copy; see + /// [`MAX_JSON_PATH_EXTRA_COPY_BYTES`]. + byte_budget: usize, } impl<'a> Selector<'a> { @@ -101,10 +124,72 @@ impl<'a> Selector<'a> { /// /// * `root_jsonb` - The `RawJsonb` data to select from. pub fn new(root_jsonb: RawJsonb<'a>) -> Selector<'a> { - Self { + let mut selector = Self { root_jsonb, items: VecDeque::new(), - } + budget: 0, + byte_budget: 0, + }; + selector.reset_budgets(); + selector + } + + /// Restores the full budgets for a new evaluation. The byte budget adds + /// the document size, so any single value can be copied at least once. + fn reset_budgets(&mut self) { + self.budget = MAX_JSON_PATH_ITEMS; + self.byte_budget = MAX_JSON_PATH_EXTRA_COPY_BYTES.saturating_add(self.root_jsonb.len()); + } + + /// Spends `units` of the evaluation budget. + /// + /// # Errors + /// + /// Returns `Error::JsonPathTooComplex` once the budget is used up. + fn charge(&mut self, units: usize) -> Result<()> { + self.budget = self + .budget + .checked_sub(units) + .ok_or(Error::JsonPathTooComplex)?; + Ok(()) + } + + /// Spends the byte budget on copying `item`. + /// + /// # Errors + /// + /// Returns `Error::JsonPathTooComplex` once the byte budget is used up. + fn charge_copy(&mut self, item: &JsonbItem<'_>) -> Result<()> { + let len = match item { + JsonbItem::Null | JsonbItem::Boolean(_) => 1, + JsonbItem::Number(NumberItem::Raw(data)) + | JsonbItem::Extension(ExtensionItem::Raw(data)) => data.len(), + // Decoded values are at most a few dozen bytes once encoded. + JsonbItem::Number(NumberItem::Number(_)) + | JsonbItem::Extension(ExtensionItem::Extension(_)) => 64, + JsonbItem::String(s) => s.len(), + JsonbItem::Raw(raw) => raw.len(), + JsonbItem::Owned(owned) => owned.len(), + }; + // Each copy also carries a 4-byte header or JEntry. + self.byte_budget = self + .byte_budget + .checked_sub(len.saturating_add(4)) + .ok_or(Error::JsonPathTooComplex)?; + Ok(()) + } + + /// Copies `item` into an `OwnedJsonb`, charging the byte budget. + fn copy_item(&mut self, item: JsonbItem<'_>) -> Result { + self.charge_copy(&item)?; + OwnedJsonb::from_item(item) + } + + /// Appends a selected item, charging it to the budget. + fn push_item(&mut self, item: JsonbItem<'a>) -> Result<()> { + self.charge(1)?; + self.items.push_back(item); + Ok(()) } /// Executes the `JsonPath` and collects all matching items into a `Vec`. @@ -147,7 +232,7 @@ impl<'a> Selector<'a> { self.execute(json_path)?; let mut values = Vec::with_capacity(self.items.len()); while let Some(item) = self.items.pop_front() { - let value = OwnedJsonb::from_item(item)?; + let value = self.copy_item(item)?; values.push(value); } Ok(values) @@ -190,6 +275,7 @@ impl<'a> Selector<'a> { self.execute(json_path)?; let mut builder = ArrayBuilder::with_capacity(self.items.len()); while let Some(item) = self.items.pop_front() { + self.charge_copy(&item)?; builder.push_jsonb_item(item); } builder.build() @@ -236,7 +322,7 @@ impl<'a> Selector<'a> { pub fn select_first(&mut self, json_path: &'a JsonPath<'a>) -> Result> { self.execute(json_path)?; if let Some(item) = self.items.pop_front() { - let value = OwnedJsonb::from_item(item)?; + let value = self.copy_item(item)?; Ok(Some(value)) } else { Ok(None) @@ -292,12 +378,13 @@ impl<'a> Selector<'a> { if self.items.len() > 1 { let mut builder = ArrayBuilder::with_capacity(self.items.len()); while let Some(item) = self.items.pop_front() { + self.charge_copy(&item)?; builder.push_jsonb_item(item); } let array = builder.build()?; Ok(Some(array)) } else if let Some(item) = self.items.pop_front() { - let value = OwnedJsonb::from_item(item)?; + let value = self.copy_item(item)?; Ok(Some(value)) } else { Ok(None) @@ -415,6 +502,7 @@ impl<'a> Selector<'a> { fn execute(&mut self, json_path: &'a JsonPath<'a>) -> Result<()> { // add root jsonb let root_item = JsonbItem::Raw(self.root_jsonb); + self.reset_budgets(); self.items.clear(); self.items.push_front(root_item); @@ -446,7 +534,7 @@ impl<'a> Selector<'a> { let item = self.items.pop_front().unwrap(); let res = self.eval_filter_expr(item.clone(), expr)?.unwrap_or(false); if res { - self.items.push_back(item); + self.push_item(item)?; } } } @@ -501,7 +589,7 @@ impl<'a> Selector<'a> { if let Some(mut object_val_iter) = object_val_iter_opt { for result in &mut object_val_iter { let val_item = result?; - self.items.push_back(val_item); + self.push_item(val_item)?; } } } @@ -510,8 +598,8 @@ impl<'a> Selector<'a> { if let Some(mut object_val_iter) = object_val_iter_opt { for result in &mut object_val_iter { let val_item = result?; - let owned_item = OwnedJsonb::from_item(val_item)?; - self.items.push_back(JsonbItem::Owned(owned_item)); + let owned_item = self.copy_item(val_item)?; + self.push_item(JsonbItem::Owned(owned_item))?; } } } @@ -521,17 +609,27 @@ impl<'a> Selector<'a> { Ok(()) } + /// Selects `parent_item` and its descendants for `.**`. + /// + /// `curr_level` is the number of containers enclosing `parent_item`. fn recursive_select_values( &mut self, parent_item: JsonbItem<'a>, - curr_level: u8, + curr_level: usize, recursive_level_opt: &Option, ) -> Result<()> { - let (is_match, should_continue) = if let Some(recursive_level) = recursive_level_opt { - recursive_level.check_recursive_level(curr_level) - } else { - (true, true) + let (is_match, should_continue) = match (recursive_level_opt, u8::try_from(curr_level)) { + (None, _) => (true, true), + (Some(recursive_level), Ok(level)) => recursive_level.check_recursive_level(level), + // Level bounds are `u8`, so deeper levels only match an open `to last` range. + (Some(recursive_level), Err(_)) => { + let open_ended = matches!(recursive_level.end, Some(RecursiveLevelEnd::Last)); + (open_ended, open_ended) + } }; + // Charge every visited node, so levels that are walked but not + // selected (`.**{100}`) are bounded too. + self.charge(1)?; if is_match { self.items.push_back(parent_item.clone()); } @@ -543,44 +641,44 @@ impl<'a> Selector<'a> { JsonbItem::Raw(raw) => { let object_val_iter_opt = ObjectValueIterator::new(raw)?; if let Some(mut object_val_iter) = object_val_iter_opt { + let next_level = enter_container(curr_level)?; for result in &mut object_val_iter { let val_item = result?; - self.recursive_select_values( - val_item, - curr_level + 1, - recursive_level_opt, - )?; + self.recursive_select_values(val_item, next_level, recursive_level_opt)?; } } let array_iter_opt = ArrayIterator::new(raw)?; if let Some(mut array_iter) = array_iter_opt { + let next_level = enter_container(curr_level)?; for item_result in &mut array_iter { let item = item_result?; - self.recursive_select_values(item, curr_level + 1, recursive_level_opt)?; + self.recursive_select_values(item, next_level, recursive_level_opt)?; } } } JsonbItem::Owned(ref owned) => { let object_val_iter_opt = ObjectValueIterator::new(owned.as_raw())?; if let Some(mut object_val_iter) = object_val_iter_opt { + let next_level = enter_container(curr_level)?; for result in &mut object_val_iter { let val_item = result?; - let owned_item = OwnedJsonb::from_item(val_item)?; + let owned_item = self.copy_item(val_item)?; self.recursive_select_values( JsonbItem::Owned(owned_item), - curr_level + 1, + next_level, recursive_level_opt, )?; } } let array_iter_opt = ArrayIterator::new(owned.as_raw())?; if let Some(mut array_iter) = array_iter_opt { + let next_level = enter_container(curr_level)?; for item_result in &mut array_iter { let item = item_result?; - let owned_item = OwnedJsonb::from_item(item)?; + let owned_item = self.copy_item(item)?; self.recursive_select_values( JsonbItem::Owned(owned_item), - curr_level + 1, + next_level, recursive_level_opt, )?; } @@ -607,7 +705,7 @@ impl<'a> Selector<'a> { if let Some(val_item) = raw.get_object_value_by_key_name(&key_name, |name, key| key.eq(name))? { - self.items.push_back(val_item); + self.push_item(val_item)?; } } JsonbItem::Owned(ref owned) => { @@ -615,8 +713,8 @@ impl<'a> Selector<'a> { if let Some(val_item) = raw.get_object_value_by_key_name(&key_name, |name, key| key.eq(name))? { - let owned_item = OwnedJsonb::from_item(val_item)?; - self.items.push_back(JsonbItem::Owned(owned_item)); + let owned_item = self.copy_item(val_item)?; + self.push_item(JsonbItem::Owned(owned_item))?; } } _ => {} @@ -628,7 +726,7 @@ impl<'a> Selector<'a> { let jsonb_item_type = parent_item.jsonb_item_type()?; if !matches!(jsonb_item_type, JsonbItemType::Array(_)) { // In lax mode, bracket wildcard allow Scalar and Object value. - self.items.push_back(parent_item); + self.push_item(parent_item)?; return Ok(()); }; @@ -638,7 +736,7 @@ impl<'a> Selector<'a> { if let Some(mut array_iter) = array_iter_opt { for item_result in &mut array_iter { let item = item_result?; - self.items.push_back(item); + self.push_item(item)?; } } } @@ -647,8 +745,8 @@ impl<'a> Selector<'a> { if let Some(mut array_iter) = array_iter_opt { for item_result in &mut array_iter { let item = item_result?; - let owned_item = OwnedJsonb::from_item(item)?; - self.items.push_back(JsonbItem::Owned(owned_item)); + let owned_item = self.copy_item(item)?; + self.push_item(JsonbItem::Owned(owned_item))?; } } } @@ -678,7 +776,7 @@ impl<'a> Selector<'a> { for (i, item_result) in &mut array_iter.enumerate() { let item = item_result?; if indices.contains(&i) { - self.items.push_back(item); + self.push_item(item)?; } } } @@ -689,8 +787,8 @@ impl<'a> Selector<'a> { for (i, item_result) in &mut array_iter.enumerate() { let item = item_result?; if indices.contains(&i) { - let owned_item = OwnedJsonb::from_item(item)?; - self.items.push_back(JsonbItem::Owned(owned_item)); + let owned_item = self.copy_item(item)?; + self.push_item(JsonbItem::Owned(owned_item))?; } } } @@ -706,7 +804,7 @@ impl<'a> Selector<'a> { Expr::UnaryOp { op, operand } => { let res_items = self.eval_unary_arithmetic_func(item.clone(), op, operand)?; for res_item in res_items { - self.items.push_back(res_item); + self.push_item(res_item)?; } } Expr::BinaryOp { op, left, right } => match op { @@ -718,7 +816,7 @@ impl<'a> Selector<'a> { let res_items = self.eval_binary_arithmetic_func(item.clone(), op, left, right)?; for res_item in res_items { - self.items.push_back(res_item); + self.push_item(res_item)?; } } _ => { @@ -728,7 +826,7 @@ impl<'a> Selector<'a> { } else { JsonbItem::Null }; - self.items.push_back(res_item); + self.push_item(res_item)?; } }, Expr::ExistsFunc(_) => { @@ -738,11 +836,11 @@ impl<'a> Selector<'a> { } else { JsonbItem::Null }; - self.items.push_back(res_item); + self.push_item(res_item)?; } Expr::Value(val) => { let res_item = self.eval_value(val)?; - self.items.push_back(res_item); + self.push_item(res_item)?; } Expr::Paths(_) => { return Err(Error::InvalidJsonPath); @@ -853,8 +951,7 @@ impl<'a> Selector<'a> { | BinaryOperator::StartsWith => { let lhs = self.convert_expr_val(item.clone(), left)?; let rhs = self.convert_expr_val(item.clone(), right)?; - let res = self.eval_compare(op, &lhs, &rhs); - Ok(res) + self.eval_compare(op, &lhs, &rhs) } _ => Ok(None), }, @@ -897,7 +994,7 @@ impl<'a> Selector<'a> { let item = self.items.pop_front().unwrap(); let res = self.eval_filter_expr(item.clone(), expr)?.unwrap_or(false); if res { - self.items.push_back(item); + self.push_item(item)?; } } } @@ -916,7 +1013,7 @@ impl<'a> Selector<'a> { expr: &'a Expr<'a>, ) -> Result> { match expr { - Expr::Value(value) => Ok(ExprValue::Value(value.clone())), + Expr::Value(value) => Ok(ExprValue::Value(value)), Expr::Paths(paths) => { let mut filter_items = self.select_by_filter_paths(item, paths)?; @@ -949,6 +1046,7 @@ impl<'a> Selector<'a> { continue; } }; + self.charge(1)?; values.push(value); } } else { @@ -966,6 +1064,7 @@ impl<'a> Selector<'a> { continue; } }; + self.charge(1)?; values.push(value); } continue; @@ -974,6 +1073,7 @@ impl<'a> Selector<'a> { continue; } }; + self.charge(1)?; values.push(value); } Ok(ExprValue::Values(values)) @@ -987,36 +1087,27 @@ impl<'a> Selector<'a> { op: &BinaryOperator, lhs: &ExprValue<'a>, rhs: &ExprValue<'a>, - ) -> Option { - let (lvals, rvals) = match (lhs, rhs) { - (ExprValue::Value(lhs), ExprValue::Value(rhs)) => { - (vec![*lhs.clone()], vec![*rhs.clone()]) - } - (ExprValue::Values(lhses), ExprValue::Value(rhs)) => { - (lhses.clone(), vec![*rhs.clone()]) - } - (ExprValue::Value(lhs), ExprValue::Values(rhses)) => { - (vec![*lhs.clone()], rhses.clone()) - } - (ExprValue::Values(lhses), ExprValue::Values(rhses)) => (lhses.clone(), rhses.clone()), - }; - - for lval in lvals.iter() { - for rval in rvals.iter() { - let res = self.compare_value(op, lval.clone(), rval.clone())?; - if res { - return Some(true); + ) -> Result> { + // Compare borrowed values: cloning a string operand would copy it + // once per pair, outside the byte budget. + for lval in lhs.as_slice() { + for rval in rhs.as_slice() { + self.charge(1)?; + match self.compare_value(op, lval, rval) { + Some(true) => return Ok(Some(true)), + Some(false) => {} + None => return Ok(None), } } } - Some(false) + Ok(Some(false)) } fn compare_value( &mut self, op: &BinaryOperator, - lhs: PathValue<'a>, - rhs: PathValue<'a>, + lhs: &PathValue<'a>, + rhs: &PathValue<'a>, ) -> Option { // container value can't compare values. if matches!(lhs, PathValue::Raw(_)) || matches!(rhs, PathValue::Raw(_)) { @@ -1024,12 +1115,12 @@ impl<'a> Selector<'a> { } if op == &BinaryOperator::StartsWith { let res = match (lhs, rhs) { - (PathValue::String(lhs), PathValue::String(rhs)) => Some(lhs.starts_with(&*rhs)), + (PathValue::String(lhs), PathValue::String(rhs)) => Some(lhs.starts_with(&**rhs)), (_, _) => None, }; return res; } - let order = lhs.partial_cmp(&rhs); + let order = lhs.partial_cmp(rhs); if let Some(order) = order { let res = match op { BinaryOperator::Eq => order == Ordering::Equal, diff --git a/src/keypath.rs b/src/keypath.rs index fd6ff7f..0bfa6b1 100644 --- a/src/keypath.rs +++ b/src/keypath.rs @@ -31,6 +31,9 @@ use nom::Parser; use crate::jsonpath::raw_string; use crate::jsonpath::string; +use crate::jsonpath::write_name; +use crate::jsonpath::write_quoted_str; +use crate::jsonpath::write_raw_name; use crate::Error; /// Represents a set of key path chains. @@ -92,10 +95,14 @@ impl Display for KeyPath<'_> { write!(f, "{idx}")?; } KeyPath::QuotedName(name) => { - write!(f, "\"{name}\"")?; + write_quoted_str(f, name)?; } KeyPath::Name(name) => { - write!(f, "{name}")?; + if name.is_empty() { + write_quoted_str(f, name)?; + } else { + write_raw_name(f, name, true)?; + } } } Ok(()) @@ -123,7 +130,7 @@ impl Display for OwnedKeyPath { write!(f, "{idx}")?; } OwnedKeyPath::Name(name) => { - write!(f, "{name}")?; + write_name(f, name, true)?; } } Ok(()) diff --git a/src/lib.rs b/src/lib.rs index e5224c1..1fa4198 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -65,6 +65,7 @@ mod constants; pub mod core; +mod eisel_lemire; mod error; mod extension; mod from; diff --git a/src/number.rs b/src/number.rs index 27ff7a3..c93189b 100644 --- a/src/number.rs +++ b/src/number.rs @@ -18,6 +18,7 @@ use std::fmt::Debug; use std::fmt::Display; use std::fmt::Formatter; +use crate::eisel_lemire; use crate::error::Result; use crate::Error; @@ -146,8 +147,7 @@ impl Decimal64 { /// This is useful when you need to perform floating-point operations, /// but note that it may introduce precision loss inherent to floating-point arithmetic. pub fn to_float64(&self) -> f64 { - let div = 10_f64.powi(self.scale as i32); - self.value as f64 / div + decimal_to_f64(self.value.into(), self.scale) } } @@ -170,8 +170,7 @@ impl Decimal128 { /// Note that for very large values, this conversion may lose precision /// as f64 has limited precision compared to i128. pub fn to_float64(&self) -> f64 { - let div = 10_f64.powi(self.scale as i32); - self.value as f64 / div + decimal_to_f64(self.value, self.scale) } } @@ -195,9 +194,63 @@ impl Decimal256 { /// For extremely large values, significant precision loss may occur /// as f64 has much lower precision than i256. pub fn to_float64(&self) -> f64 { - let div = 10_f64.powi(self.scale as i32); - self.value.as_f64() / div + if let Ok(value) = i128::try_from(self.value) { + return decimal_to_f64(value, self.scale); + } + // `itoa` stops at `i128`, so wider values parse their decimal text. + // The buffer fits the longest `i256` decimal, so formatting succeeds. + let mut buf = StackBuf::::new(); + match format_decimal_i256(&mut buf, self.value, self.scale.into()) { + // Decimal formatting only writes an optional `-`, digits and one `.`. + Ok(()) => fast_float2::parse(buf.as_bytes()).unwrap_or(f64::NAN), + Err(_) => f64::NAN, + } + } +} + +/// Largest power of ten that `f64` represents exactly. +const MAX_EXACT_F64_POWER_OF_10: u8 = 22; + +/// Longest `e-` text: a sign, the 39 digits of an `i128`, +/// `e-`, and a three-digit `u8` scale. +const SCIENTIFIC_TEXT_CAPACITY: usize = 1 + 39 + 2 + 3; + +/// Returns the `f64` nearest to `value / 10^scale`. +/// +/// When `|value| <= 2^53` and `10^scale` are both exact in `f64`, one IEEE +/// division rounds correctly. Converting a larger value first and dividing +/// after rounds twice, which can land on the neighboring `f64`: +/// 12345678901239047 / 100 gave 123456789012390.48 instead of +/// 123456789012390.47, so equal decimal and float values compared unequal. +/// Other values go through Eisel-Lemire, or failing that parse +/// `e-`; both round once. +fn decimal_to_f64(value: i128, scale: u8) -> f64 { + if value.unsigned_abs() <= 1 << f64::MANTISSA_DIGITS && scale <= MAX_EXACT_F64_POWER_OF_10 { + // Converting through `i64` avoids the slower `i128` to `f64` routine; + // the value is exact in both. + return value as i64 as f64 / 10_f64.powi(i32::from(scale)); + } + // Eisel-Lemire computes the rounding from the integers directly; it gives + // up on rare near-halfway products, and does not take mantissas above + // `u64`, which then parse their text. + if let Ok(mantissa) = u64::try_from(value.unsigned_abs()) { + if let Some(float) = eisel_lemire::decimal_to_f64(value < 0, mantissa, scale) { + return float; + } } + // Fallback: parse `e-`. Exponent notation needs no decimal + // point or leading zeros, so only two integers are written. + let mut digits = itoa::Buffer::new(); + let digits = digits.format(value).as_bytes(); + let mut exponent = itoa::Buffer::new(); + let exponent = exponent.format(scale).as_bytes(); + let mut text = [0; SCIENTIFIC_TEXT_CAPACITY]; + let (mantissa, rest) = text.split_at_mut(digits.len()); + mantissa.copy_from_slice(digits); + rest[..2].copy_from_slice(b"e-"); + rest[2..2 + exponent.len()].copy_from_slice(exponent); + let len = digits.len() + 2 + exponent.len(); + fast_float2::parse(&text[..len]).unwrap_or(f64::NAN) } impl Number { @@ -264,15 +317,17 @@ fn write_json_f64(v: f64, buf: &mut Vec) { } } +/// Longest formatted decimal: a sign, 77 digits of an `i256`, `0.`, and up to +/// 255 leading fractional zeros from the scale. +const DECIMAL_TEXT_CAPACITY: usize = 1 + 77 + 2 + 255; + /// A fixed-capacity buffer that formatting writes into without allocating. /// Writing more than `N` bytes fails. -#[cfg(feature = "arbitrary_precision")] struct StackBuf { buf: [u8; N], len: usize, } -#[cfg(feature = "arbitrary_precision")] impl StackBuf { fn new() -> Self { Self { @@ -286,7 +341,6 @@ impl StackBuf { } } -#[cfg(feature = "arbitrary_precision")] impl std::fmt::Write for StackBuf { fn write_str(&mut self, s: &str) -> std::fmt::Result { let end = self.len + s.len(); @@ -438,8 +492,10 @@ impl Serialize for Number { serialize_struct.end() } } - // Without arbitrary_precision, decimals serialize as `as_f64`, - // the same value `Number::write_json` writes. + // Without arbitrary_precision, decimals serialize as the nearest + // `f64`. Converting the unscaled value first and dividing after + // rounded twice, so a decimal could serialize as a neighboring + // float of the one `as_f64` returns. #[cfg(not(feature = "arbitrary_precision"))] Number::Decimal64(_) | Number::Decimal128(_) | Number::Decimal256(_) => { serializer.serialize_f64(self.as_f64()) @@ -583,27 +639,26 @@ impl Number { .checked_neg() .map(Number::Int64) .ok_or(Error::Message("Int64 overflow".to_string())), - Number::UInt64(v) => { - if let Ok(v) = i64::try_from(*v) { - v.checked_neg() - .map(Number::Int64) - .ok_or(Error::Message("Int64 overflow".to_string())) - } else { - Err(Error::Message("Int64 overflow".to_string())) - } - } + // `-2^63` is the one negated `u64` above `i64::MAX` that fits. + Number::UInt64(v) => Self::from_integer_result(Some(-i128::from(*v)), false), Number::Float64(v) => Ok(Number::Float64(*v * -1.0)), Number::Decimal64(v) => { + let Some(neg_value) = v.value.checked_neg() else { + return Err(Error::Message("Decimal64 overflow".to_string())); + }; let neg_dec = Decimal64 { scale: v.scale, - value: -v.value, + value: neg_value, }; Ok(Number::Decimal64(neg_dec)) } Number::Decimal128(v) => { + let Some(neg_value) = v.value.checked_neg() else { + return Err(Error::Message("Decimal128 overflow".to_string())); + }; let neg_dec = Decimal128 { scale: v.scale, - value: -v.value, + value: neg_value, }; Ok(Number::Decimal128(neg_dec)) } @@ -624,39 +679,10 @@ impl Number { /// /// This method returns an error if the sum would overflow. pub fn add(&self, other: Number) -> Result { + if let Some((a, b, both_signed)) = self.integer_operands(&other) { + return Self::from_integer_result(a.checked_add(b), both_signed); + } match (self, other) { - (Number::Int64(a), Number::Int64(b)) => a - .checked_add(b) - .map(Number::Int64) - .ok_or(Error::Message("Int64 overflow".to_string())), - (Number::UInt64(a), Number::UInt64(b)) => a - .checked_add(b) - .map(Number::UInt64) - .ok_or(Error::Message("UInt64 overflow".to_string())), - (Number::Int64(a), Number::UInt64(b)) => { - if *a < 0 { - a.checked_add(b as i64) - .map(Number::Int64) - .ok_or(Error::Message("Int64 overflow".to_string())) - } else { - (*a as u64) - .checked_add(b) - .map(Number::UInt64) - .ok_or(Error::Message("UInt64 overflow".to_string())) - } - } - (Number::UInt64(a), Number::Int64(b)) => { - if b < 0 { - (*a as i64) - .checked_add(b) - .map(Number::Int64) - .ok_or(Error::Message("Int64 overflow".to_string())) - } else { - a.checked_add(b as u64) - .map(Number::UInt64) - .ok_or(Error::Message("UInt64 overflow".to_string())) - } - } (Number::Float64(a), Number::Float64(b)) => Ok(Number::Float64(a + b)), (a, b) => { let a_float = a.as_f64(); @@ -670,23 +696,10 @@ impl Number { /// /// This method returns an error if the difference would overflow. pub fn sub(&self, other: Number) -> Result { + if let Some((a, b, both_signed)) = self.integer_operands(&other) { + return Self::from_integer_result(a.checked_sub(b), both_signed); + } match (self, other) { - (Number::Int64(a), Number::Int64(b)) => a - .checked_sub(b) - .map(Number::Int64) - .ok_or(Error::Message("Int64 overflow".to_string())), - (Number::UInt64(a), Number::UInt64(b)) => (*a as i64) - .checked_sub(b as i64) - .map(Number::Int64) - .ok_or(Error::Message("UInt64 overflow".to_string())), - (Number::Int64(a), Number::UInt64(b)) => a - .checked_sub(b as i64) - .map(Number::Int64) - .ok_or(Error::Message("Int64 overflow".to_string())), - (Number::UInt64(a), Number::Int64(b)) => (*a as i64) - .checked_sub(b) - .map(Number::Int64) - .ok_or(Error::Message("Int64 overflow".to_string())), (Number::Float64(a), Number::Float64(b)) => Ok(Number::Float64(a - b)), (a, b) => { let a_float = a.as_f64(); @@ -700,39 +713,10 @@ impl Number { /// /// This method returns an error if the product would overflow. pub fn mul(&self, other: Number) -> Result { + if let Some((a, b, both_signed)) = self.integer_operands(&other) { + return Self::from_integer_result(a.checked_mul(b), both_signed); + } match (self, other) { - (Number::Int64(a), Number::Int64(b)) => a - .checked_mul(b) - .map(Number::Int64) - .ok_or(Error::Message("Int64 overflow".to_string())), - (Number::UInt64(a), Number::UInt64(b)) => a - .checked_mul(b) - .map(Number::UInt64) - .ok_or(Error::Message("UInt64 overflow".to_string())), - (Number::Int64(a), Number::UInt64(b)) => { - if *a < 0 { - a.checked_mul(b as i64) - .map(Number::Int64) - .ok_or(Error::Message("Int64 overflow".to_string())) - } else { - (*a as u64) - .checked_mul(b) - .map(Number::UInt64) - .ok_or(Error::Message("UInt64 overflow".to_string())) - } - } - (Number::UInt64(a), Number::Int64(b)) => { - if b < 0 { - (*a as i64) - .checked_mul(b) - .map(Number::Int64) - .ok_or(Error::Message("Int64 overflow".to_string())) - } else { - a.checked_mul(b as u64) - .map(Number::UInt64) - .ok_or(Error::Message("UInt64 overflow".to_string())) - } - } (Number::Float64(a), Number::Float64(b)) => Ok(Number::Float64(a * b)), (a, b) => { let a_float = a.as_f64(); @@ -758,53 +742,13 @@ impl Number { /// /// This method returns an error if the divisor is zero. pub fn rem(&self, other: Number) -> Result { - match (self, other) { - (Number::Int64(a), Number::Int64(b)) => { - if b == 0 { - return Err(Error::Message("Division by zero".to_string())); - } - a.checked_rem(b) - .map(Number::Int64) - .ok_or(Error::Message("Int64 overflow".to_string())) - } - (Number::UInt64(a), Number::UInt64(b)) => { - if b == 0 { - return Err(Error::Message("Division by zero".to_string())); - } - a.checked_rem(b) - .map(Number::UInt64) - .ok_or(Error::Message("UInt64 overflow".to_string())) - } - (Number::Int64(a), Number::UInt64(b)) => { - if b == 0 { - return Err(Error::Message("Division by zero".to_string())); - } - if *a < 0 { - a.checked_rem(b as i64) - .map(Number::Int64) - .ok_or(Error::Message("Int64 overflow".to_string())) - } else { - (*a as u64) - .checked_rem(b) - .map(Number::UInt64) - .ok_or(Error::Message("UInt64 overflow".to_string())) - } - } - (Number::UInt64(a), Number::Int64(b)) => { - if b == 0 { - return Err(Error::Message("Division by zero".to_string())); - } - if b < 0 { - (*a as i64) - .checked_rem(b) - .map(Number::Int64) - .ok_or(Error::Message("Int64 overflow".to_string())) - } else { - a.checked_rem(b as u64) - .map(Number::UInt64) - .ok_or(Error::Message("UInt64 overflow".to_string())) - } + if let Some((a, b, both_signed)) = self.integer_operands(&other) { + if b == 0 { + return Err(Error::Message("Division by zero".to_string())); } + return Self::from_integer_result(a.checked_rem(b), both_signed); + } + match (self, other) { (Number::Float64(a), Number::Float64(b)) => { if b == 0.0 { return Err(Error::Message("Division by zero".to_string())); @@ -823,6 +767,46 @@ impl Number { } } +impl Number { + /// Returns the exact values of this number and `other` if both are + /// integers, and whether both are `Int64`. + fn integer_operands(&self, other: &Number) -> Option<(i128, i128, bool)> { + let value = |n: &Number| match n { + Number::Int64(v) => Some(i128::from(*v)), + Number::UInt64(v) => Some(i128::from(*v)), + _ => None, + }; + let both_signed = matches!((self, other), (Number::Int64(_), Number::Int64(_))); + Some((value(self)?, value(other)?, both_signed)) + } + + /// Builds the result of integer arithmetic from its exact value. + /// + /// Two `Int64` operands give an `Int64` result. Otherwise a non-negative + /// result is a `UInt64` and a negative one an `Int64`. A value outside + /// those ranges, or no value at all, is an overflow. Computing in `i128` + /// first keeps mixed `Int64`/`UInt64` operands from being cast to the + /// wrong type, which made `u64::MAX + (-1)` return `-2`. + fn from_integer_result(value: Option, both_signed: bool) -> Result { + let overflow = || { + let ty = if both_signed { "Int64" } else { "UInt64" }; + Error::Message(format!("{ty} overflow")) + }; + let value = value.ok_or_else(overflow)?; + if both_signed { + return i64::try_from(value) + .map(Number::Int64) + .map_err(|_| overflow()); + } + match u64::try_from(value) { + Ok(v) => Ok(Number::UInt64(v)), + Err(_) => i64::try_from(value) + .map(Number::Int64) + .map_err(|_| overflow()), + } + } +} + fn exact_float_to_int(value: f64) -> Option where T: Bounded + AsPrimitive, @@ -1047,12 +1031,15 @@ impl PartialOrd for &Number { impl Ord for Number { /// Returns the ordering of the number compared to another number. /// - /// This method implements precise comparison between different number types: - /// - When comparing decimal types with other types, it converts the non-decimal type - /// to a decimal representation to preserve precision - /// - When comparing between decimal types of different precision, it upgrades the lower - /// precision decimal to match the higher precision one - /// - Only falls back to floating-point comparison as a last resort + /// Integers and decimals compare by their exact values. A float compares + /// as the shortest decimal that parses back to it, so the float `0.1` + /// equals the decimal `0.1`. `NaN` is greater than every other number and + /// equal to itself, as in `OrderedFloat`. + /// + /// Each number thus orders by one exact value, which keeps the order + /// transitive. Converting integers and decimals to `f64` to compare them + /// with floats did not: `2^53 + 1` equaled the float `2^53`, which equals + /// `2^53`, though `2^53 + 1 > 2^53`. #[inline] fn cmp(&self, other: &Self) -> Ordering { match (self, other) { @@ -1071,9 +1058,7 @@ impl Ord for Number { { l_val.cmp(&r_val) } else { - let l = OrderedFloat(self.as_f64()); - let r = OrderedFloat(other.as_f64()); - l.cmp(&r) + self.cmp_exact(other) } } } @@ -1088,9 +1073,7 @@ impl Ord for Number { { l_val.cmp(&r_val) } else { - let l = OrderedFloat(self.as_f64()); - let r = OrderedFloat(other.as_f64()); - l.cmp(&r) + self.cmp_exact(other) } } } @@ -1114,10 +1097,8 @@ impl Ord for Number { return l_val.cmp(&r.value); } } - // multiply overflow, fallback to used float compare - let l = OrderedFloat(self.as_f64()); - let r = OrderedFloat(other.as_f64()); - l.cmp(&r) + // The scaled value overflows `i256`. + self.cmp_exact(other) } } @@ -1309,13 +1290,222 @@ impl Ord for Number { self.cmp(&Number::Decimal256(r_decimal)) } - // Fall back to float comparison for any other combinations - (_, _) => { - let l = OrderedFloat(self.as_f64()); - let r = OrderedFloat(other.as_f64()); - l.cmp(&r) + (Number::Float64(l), _) => other.cmp_with_float(*l).reverse(), + (_, Number::Float64(r)) => self.cmp_with_float(*r), + } + } +} + +impl Number { + /// Compares this integer or decimal with `float`. + /// + /// `as_f64` rounds to the nearest float. If that is not `float`, the exact + /// value and `float`'s shortest decimal round to different floats, so they + /// order like those floats. Only numbers that round to `float` itself + /// need the exact comparison. + fn cmp_with_float(&self, float: f64) -> Ordering { + if float.is_nan() { + return Ordering::Less; + } + match self.as_f64().partial_cmp(&float) { + Some(Ordering::Equal) | None => self.cmp_exact(&Number::Float64(float)), + Some(order) => order, + } + } + + /// Compares two numbers by their exact values, taking a float as its + /// shortest decimal. + /// + /// This is slower than the comparisons in `cmp`, which use it only where + /// they cannot decide: decimals whose rescaled values overflow, and + /// numbers that round to the float they are compared with. Two floats + /// compare as floats, which orders them the same way. + fn cmp_exact(&self, other: &Number) -> Ordering { + if let (Number::Float64(l), Number::Float64(r)) = (self, other) { + return OrderedFloat(*l).cmp(&OrderedFloat(*r)); + } + match (ExactDecimal::new(self), ExactDecimal::new(other)) { + (Some(l), Some(r)) => l.cmp(&r), + // Only non-finite floats have no decimal, and `cmp_with_float` + // orders those without coming here. + _ => OrderedFloat(self.as_f64()).cmp(&OrderedFloat(other.as_f64())), + } + } +} + +/// A finite number as its exact value `(-1)^negative * magnitude / 10^scale`. +struct ExactDecimal { + negative: bool, + magnitude: U256, + /// 0 to 255 for decimals. Floats range from -292, for + /// `1.7976931348623157e308`, to under 340, for the smallest floats. + scale: i32, +} + +impl ExactDecimal { + /// Returns the exact value of `number`, or `None` for `NaN` and the + /// infinities. + fn new(number: &Number) -> Option { + let (negative, magnitude, scale) = match number { + Number::Int64(v) => (*v < 0, U256::from(v.unsigned_abs()), 0), + Number::UInt64(v) => (false, U256::from(*v), 0), + Number::Decimal64(d) => (d.value < 0, U256::from(d.value.unsigned_abs()), d.scale), + Number::Decimal128(d) => (d.value < 0, U256::from(d.value.unsigned_abs()), d.scale), + Number::Decimal256(d) => (d.value < i256::ZERO, d.value.unsigned_abs(), d.scale), + Number::Float64(v) => return Self::from_f64(*v), + }; + Some(Self { + negative, + magnitude, + scale: i32::from(scale), + }) + } + + /// Returns the shortest decimal that parses back to `value`, which is + /// the text `zmij` writes, or `None` for `NaN` and the infinities. + fn from_f64(value: f64) -> Option { + if !value.is_finite() { + return None; + } + let mut buf = zmij::Buffer::new(); + // An optional `-`, digits with an optional `.` fraction, and an + // optional exponent: `0.1`, `123.0`, `1e+300`, `-5e-324`. There are + // at most 17 significant digits, so the mantissa fits a `u128`. + let text = buf.format_finite(value).as_bytes(); + let (negative, text) = match text.split_first() { + Some((b'-', rest)) => (true, rest), + _ => (false, text), + }; + let (digits, exponent) = match text.iter().position(|&b| b == b'e') { + Some(e) => (&text[..e], &text[e + 1..]), + None => (text, &[][..]), + }; + let mut mantissa = 0u128; + let mut scale = 0i32; + let mut in_fraction = false; + for &b in digits { + if b == b'.' { + in_fraction = true; + } else { + mantissa = mantissa * 10 + u128::from(b - b'0'); + scale += i32::from(in_fraction); } } + let (exponent_negative, exponent_digits) = match exponent.split_first() { + Some((b'-', rest)) => (true, rest), + Some((b'+', rest)) => (false, rest), + _ => (false, exponent), + }; + let exponent = exponent_digits + .iter() + .fold(0i32, |e, &b| e * 10 + i32::from(b - b'0')); + if exponent_negative { + scale += exponent; + } else { + scale -= exponent; + } + Some(Self { + negative, + magnitude: U256::from(mantissa), + scale, + }) + } +} + +impl Ord for ExactDecimal { + fn cmp(&self, other: &Self) -> Ordering { + // -1, 0 or 1; zero has either sign, as for `-0.0`. + let sign = |d: &Self| match (d.magnitude == U256::ZERO, d.negative) { + (true, _) => 0, + (false, true) => -1, + (false, false) => 1, + }; + let (l, r) = (sign(self), sign(other)); + if l != r || l == 0 { + return l.cmp(&r); + } + let order = self.cmp_magnitude(other); + if l < 0 { + order.reverse() + } else { + order + } + } +} + +impl PartialOrd for ExactDecimal { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl PartialEq for ExactDecimal { + fn eq(&self, other: &Self) -> bool { + self.cmp(other) == Ordering::Equal + } +} + +impl Eq for ExactDecimal {} + +impl ExactDecimal { + /// Compares the magnitudes by bringing both to the larger scale. + fn cmp_magnitude(&self, other: &Self) -> Ordering { + if self.scale == other.scale { + return self.magnitude.cmp(&other.magnitude); + } + let scale = self.scale.max(other.scale); + let rescale = |d: &Self| { + let mut wide = WideUint::new(d.magnitude); + wide.mul_pow10(scale.abs_diff(d.scale)); + wide + }; + rescale(self).cmp(&rescale(other)) + } +} + +/// A 2048-bit unsigned integer, in little-endian 64-bit limbs. +/// +/// That fits every rescaled magnitude in `ExactDecimal::cmp_magnitude`, which +/// compares a float with an integer or decimal, or two decimals, but never two +/// floats. The largest is a float mantissa below 2^57 times 10^547, between +/// scales -292 and 255, which is below 2^1875. Decimal magnitudes are below +/// 2^256 and need at most 10^340, which gives below 2^1386. +#[derive(PartialEq, Eq, PartialOrd, Ord)] +struct WideUint([u64; 32]); + +impl WideUint { + fn new(value: U256) -> Self { + let (high, low) = value.into_words(); + let mut limbs = [0; 32]; + limbs[0] = low as u64; + limbs[1] = (low >> 64) as u64; + limbs[2] = high as u64; + limbs[3] = (high >> 64) as u64; + // The derived `Ord` compares limbs from the first, so store them + // most significant first. + limbs.reverse(); + Self(limbs) + } + + fn mul_small(&mut self, factor: u64) { + let mut carry = 0u128; + for limb in self.0.iter_mut().rev() { + let product = u128::from(*limb) * u128::from(factor) + carry; + *limb = product as u64; + carry = product >> 64; + } + debug_assert_eq!(carry, 0, "WideUint overflow"); + } + + fn mul_pow10(&mut self, mut exponent: u32) { + const POW10_19: u64 = 10_000_000_000_000_000_000; + while exponent >= 19 { + self.mul_small(POW10_19); + exponent -= 19; + } + if exponent > 0 { + self.mul_small(10_u64.pow(exponent)); + } } } diff --git a/tests/allocation.rs b/tests/allocation.rs index 1097e76..c3d9837 100644 --- a/tests/allocation.rs +++ b/tests/allocation.rs @@ -13,7 +13,7 @@ // limitations under the License. //! Checks that work on untrusted input allocates in proportion to the input, -//! not to the lengths it claims. This is a +//! not to the lengths it claims or to how often a value is reused. This is a //! separate test binary because it installs a global allocator that records //! allocation sizes. @@ -22,6 +22,7 @@ use std::alloc::Layout; use std::alloc::System; use std::cell::Cell; +use jsonb::jsonpath::parse_json_path; use jsonb::*; struct RecordSizes; @@ -30,6 +31,7 @@ thread_local! { // Per thread, so tests running in parallel do not see each other's // allocations. Constant-initialized cells never allocate themselves. static LARGEST: Cell = const { Cell::new(0) }; + static TOTAL: Cell = const { Cell::new(0) }; } // SAFETY: forwards every call to `System` unchanged; it only records sizes. @@ -37,6 +39,7 @@ unsafe impl GlobalAlloc for RecordSizes { unsafe fn alloc(&self, layout: Layout) -> *mut u8 { let size = layout.size(); LARGEST.with(|largest| largest.set(largest.get().max(size))); + TOTAL.with(|total| total.set(total.get() + size)); // SAFETY: the caller upholds `GlobalAlloc::alloc`'s contract. unsafe { System.alloc(layout) } } @@ -57,6 +60,13 @@ fn largest_allocation(f: impl FnOnce()) -> usize { LARGEST.with(Cell::get) } +/// Runs `f` and returns the total size of the allocations it made. +fn total_allocation(f: impl FnOnce()) -> usize { + TOTAL.with(|total| total.set(0)); + f(); + TOTAL.with(Cell::get) +} + #[test] fn claimed_lengths_do_not_size_allocations() { // JSON text read as JSONB: `[` is an object header claiming about 457 @@ -90,3 +100,27 @@ fn claimed_lengths_do_not_size_allocations() { // Text that is not valid JSONB still parses as JSON text. assert_eq!(from_slice(b"[1,2,3]").unwrap().to_string(), "[1,2,3]"); } + +#[test] +fn json_path_comparisons_do_not_copy_literals() { + // A 100 KiB literal with an escape is stored as an owned string. Every + // comparison used to clone it, so comparing 2000 strings against it + // allocated about 200 to 600 MiB, outside the copy budget (found in review). + let doc: OwnedJsonb = format!("[{}]", vec![r#""x""#; 2000].join(",")) + .parse() + .unwrap(); + let literal = format!(r"\n{}", "a".repeat(100 * 1024)); + for path in [ + format!(r#"$[*] ? (@ == "{literal}")"#), + format!(r#"$[*] ? (@ starts with "{literal}")"#), + format!(r#"$ ? ($[*] == "{literal}")"#), + ] { + let path = parse_json_path(path.as_bytes()).unwrap(); + let mut selected = None; + let total = total_allocation(|| { + selected = Some(doc.as_raw().select_by_path(&path).map(|v| v.len())); + }); + assert_eq!(selected, Some(Ok(0))); + assert!(total < 1024 * 1024, "allocated {total} bytes"); + } +} diff --git a/tests/it/functions.rs b/tests/it/functions.rs index 663a3ab..bcd0f90 100644 --- a/tests/it/functions.rs +++ b/tests/it/functions.rs @@ -143,6 +143,10 @@ fn test_path_exists() { (r#"{"a":1,"b":2}"#, r#"$.c"#, false), (r#"{"a":1,"b":2}"#, r#"$.a ? (@ == 1)"#, true), (r#"{"a":1,"b":2}"#, r#"$.a ? (@ > 1)"#, false), + // Literals with a fraction or exponent used to fail to parse. + (r#"{"a":1.5,"b":2}"#, r#"$.a ? (@ > 1.25)"#, true), + (r#"{"a":1.5,"b":2}"#, r#"$.a ? (@ > 1.75)"#, false), + (r#"{"a":1500,"b":2}"#, r#"$.a ? (@ == 1.5e3)"#, true), (r#"{"a":1,"b":[1,2,3]}"#, r#"$.b[0]"#, true), (r#"{"a":1,"b":[1,2,3]}"#, r#"$.b[3]"#, false), ( diff --git a/tests/it/jsonpath_parser.rs b/tests/it/jsonpath_parser.rs index 3368bfe..aa90826 100644 --- a/tests/it/jsonpath_parser.rs +++ b/tests/it/jsonpath_parser.rs @@ -117,3 +117,127 @@ fn test_json_path_error() { assert!(res.is_err()); } } + +#[test] +fn test_json_path_number_literals() { + // Each case is (input, formatted). Literals with a fraction or exponent + // used to fail to parse: the integer parser took `1` from `1.5` and left + // `.5`, which no rule accepts (found by fuzzing). + let cases = [ + ("$ ? (@ > 1.5)", "$?(@ > 1.5)"), + ("$ ? (@ == -1.5e3)", "$?(@ == -1500.0)"), + ("$ ? (@ == 1E+3)", "$?(@ == 1000.0)"), + ("$ ? (@ == 1.)", "$?(@ == 1.0)"), + ("$ ? (@ == .5)", "$?(@ == 0.5)"), + ("$ ? (@ == -0.0)", "$?(@ == -0.0)"), + ( + "$ ? (@ == 18446744073709551615)", + "$?(@ == 18446744073709551615)", + ), + ( + "$ ? (@ == -9223372036854775808)", + "$?(@ == -9223372036854775808)", + ), + ( + "$ ? (@ == 18446744073709551616)", + "$?(@ == 1.8446744073709552e+19)", + ), + (".0/5", "0.0 / 5"), + ("1e16 / 5", "1e+16 / 5"), + ("inf / 5", "inf / 5"), + // Overflowing literals become infinities, which `Display` writes with + // a sign, so signed `inf`, `infinity` and `nan` must parse too. + ("0 > -1e400", "0 > -inf"), + ("0 > -inf", "0 > -inf"), + ("$ ? (@ == -Infinity)", "$?(@ == -inf)"), + ("0 > +nan", "0 > NaN"), + // Not number literals: these stay field names. + ("0.0", ".0.0"), + ("1e.a", ".1e.a"), + ]; + for (input, formatted) in cases { + let path = parse_json_path(input.as_bytes()).unwrap(); + let text = path.to_string(); + assert_eq!(text, formatted, "{input}"); + assert_eq!(parse_json_path(text.as_bytes()).unwrap(), path, "{input}"); + } +} + +#[test] +fn test_json_path_logical_parentheses() { + // Each case is (input, formatted). Parentheses are kept only where the + // tree differs from the parser's precedence (`&&` over `||`) and + // left-to-right grouping. + let cases = [ + ( + "$ ? (@.a == 1 || @.b == 2 || @.c == 3)", + "$?(@.a == 1 || @.b == 2 || @.c == 3)", + ), + ( + "$ ? ((@.a == 1 && @.b == 2) && @.c == 3)", + "$?(@.a == 1 && @.b == 2 && @.c == 3)", + ), + ( + "$ ? (@.a == 1 || @.b == 2 && @.c == 3)", + "$?(@.a == 1 || @.b == 2 && @.c == 3)", + ), + ( + "$ ? ((@.a == 1 || @.b == 2) && @.c == 3)", + "$?((@.a == 1 || @.b == 2) && @.c == 3)", + ), + ( + "$ ? (@.a == 1 && (@.b == 2 || @.c == 3))", + "$?(@.a == 1 && (@.b == 2 || @.c == 3))", + ), + ( + "$ ? (@.a == 1 || (@.b == 2 || @.c == 3))", + "$?(@.a == 1 || (@.b == 2 || @.c == 3))", + ), + ( + "$ ? (@.a == 1 && (@.b == 2 && @.c == 3))", + "$?(@.a == 1 && (@.b == 2 && @.c == 3))", + ), + ]; + for (input, formatted) in cases { + let path = parse_json_path(input.as_bytes()).unwrap(); + let text = path.to_string(); + assert_eq!(text, formatted, "{input}"); + assert_eq!(parse_json_path(text.as_bytes()).unwrap(), path, "{input}"); + } + + // A chain of 120 `||` used to format with 119 nested parentheses, past + // the parser's limit of 64 (found by fuzzing). + let chain = vec!["$.a - 5"; 120].join(" || "); + let path = parse_json_path(chain.as_bytes()).unwrap(); + let text = path.to_string(); + assert_eq!(text, chain); + assert_eq!(parse_json_path(text.as_bytes()).unwrap(), path); +} + +#[test] +fn test_json_path_empty_string() { + use jsonb::OwnedJsonb; + + // `""` is a valid string literal and a valid quoted name. The quoted-string + // parser used to reject it, so comparing with an empty string failed. + let doc: OwnedJsonb = r#"{"":{"x":1},"a":"","b":"c"}"#.parse().unwrap(); + let cases = [ + (r#"$ ? (@.a == "")"#, r#"$?(@.a == "")"#, true), + (r#"$ ? (@.b == "")"#, r#"$?(@.b == "")"#, false), + (r#"$."".x"#, r#"$."".x"#, true), + (r#"$[""]"#, r#"$[""]"#, true), + (r#"$.b starts with """#, r#"$.b starts with """#, true), + ]; + for (input, formatted, exists) in cases { + let path = parse_json_path(input.as_bytes()).unwrap(); + let text = path.to_string(); + assert_eq!(text, formatted, "{input}"); + assert_eq!(parse_json_path(text.as_bytes()).unwrap(), path, "{input}"); + let result = if path.is_predicate() { + doc.as_raw().path_match(&path).unwrap() == Some(true) + } else { + doc.as_raw().path_exists(&path).unwrap() + }; + assert_eq!(result, exists, "{input}"); + } +} diff --git a/tests/it/keypath_parser.rs b/tests/it/keypath_parser.rs index 85bb6e4..43cadae 100644 --- a/tests/it/keypath_parser.rs +++ b/tests/it/keypath_parser.rs @@ -45,3 +45,37 @@ fn test_json_path_error() { assert!(res.is_err()); } } + +#[test] +fn test_key_path_empty_name() { + use jsonb::keypath::OwnedKeyPaths; + use jsonb::OwnedJsonb; + + // An empty quoted name is a valid key, as in PostgreSQL's '{"",0}'. The + // quoted-string parser used to require at least one character. + let doc: OwnedJsonb = r#"{"":[{"x":1}],"a":{"":2}}"#.parse().unwrap(); + let cases = [ + (r#"{""}"#, Some(r#"[{"x":1}]"#), "''"), + (r#"{"",0,x}"#, Some("1"), "''[0].x"), + (r#"{a,""}"#, Some("2"), "a.''"), + (r#"{a,"",b}"#, None, "a.''.b"), + ]; + for (text, expect, canonical) in cases { + let key_paths = parse_key_paths(text.as_bytes()).unwrap(); + let got = doc + .as_raw() + .get_by_keypath(key_paths.paths.iter()) + .unwrap() + .map(|v| v.to_string()); + assert_eq!(got.as_deref(), expect, "{text}"); + + let formatted = key_paths.to_string(); + assert_eq!(formatted, text); + assert_eq!(parse_key_paths(formatted.as_bytes()).unwrap(), key_paths); + + let owned = key_paths.to_owned(); + assert_eq!(owned.to_canonical_path(), canonical, "{text}"); + let decoded = OwnedKeyPaths::from_canonical_path(canonical).unwrap(); + assert_eq!(decoded.as_key_paths(), owned.as_key_paths(), "{text}"); + } +} diff --git a/tests/it/malformed.rs b/tests/it/malformed.rs index dc318de..b8db47e 100644 --- a/tests/it/malformed.rs +++ b/tests/it/malformed.rs @@ -29,6 +29,10 @@ use rand::{Rng, SeedableRng}; /// Nesting limit for arrays and objects (`MAX_NESTING_DEPTH` in the crate). const MAX_DEPTH: usize = 512; +/// Parenthesis nesting limit for JSONPath (`MAX_JSON_PATH_DEPTH` in the crate). +const MAX_PATH_DEPTH: usize = 64; +/// `&&` and `||` limit for JSONPath (`MAX_JSON_PATH_LOGICAL_OPERATORS` in the crate). +const MAX_PATH_LOGICAL_OPERATORS: usize = 256; /// Runs `f` on a thread with a large stack. /// @@ -407,6 +411,14 @@ fn decimal_comparison_with_huge_scale_difference_does_not_panic() { assert_eq!(tiny.partial_cmp(&small), Some(std::cmp::Ordering::Less)); } +#[test] +fn json_path_rejects_unterminated_string() { + for path in [r#"""#, r#""a"#, r#"$."a"#, r#"$["a"#] { + assert!(parse_json_path(path.as_bytes()).is_err(), "{path}"); + assert!(parse_key_paths(path.as_bytes()).is_err(), "{path}"); + } +} + /// Encodes `depth` nested arrays directly, in linear time, for very deep inputs. fn deep_arrays_bytes(depth: usize) -> Vec { let mut out = Vec::with_capacity(depth * 8); @@ -451,6 +463,7 @@ fn text_parsers_enforce_nesting_limit() { /// Runs every recursive read API on `raw` and reports whether each succeeded. fn recursive_api_results(raw: RawJsonb<'_>) -> Vec<(&'static str, bool)> { + let all = parse_json_path(b"$.**").unwrap(); vec![ ("to_value", raw.to_value().is_ok()), ("to_pretty_string", raw.to_pretty_string() != "null"), @@ -467,6 +480,7 @@ fn recursive_api_results(raw: RawJsonb<'_>) -> Vec<(&'static str, bool)> { "convert_to_comparable", !raw.convert_to_comparable().is_empty(), ), + ("select_by_path", raw.select_by_path(&all).is_ok()), ("jaq", jaq_json::Val::try_from(raw).is_ok()), ( "visit_scalar_key_paths", @@ -521,6 +535,159 @@ fn binary_apis_reject_very_deep_input() { }); } +#[test] +fn json_path_enforces_paren_limit() { + let path = |depth: usize| { + let inner = depth - 1; + format!("$ ? ({}@ == 1{})", "(".repeat(inner), ")".repeat(inner)) + }; + with_big_stack(move || { + assert!(parse_json_path(path(MAX_PATH_DEPTH).as_bytes()).is_ok()); + }); + for depth in [MAX_PATH_DEPTH + 1, 100_000] { + assert_eq!( + parse_json_path(path(depth).as_bytes()).err(), + Some(Error::RecursionLimitExceeded), + "depth {depth}" + ); + } + // Parentheses inside string literals do not count. + let quoted = format!(r#"$ ? (@ == "{}")"#, "(".repeat(1000)); + assert!(parse_json_path(quoted.as_bytes()).is_ok()); +} + +#[test] +fn json_path_enforces_logical_operator_limit() { + // `&&` and `||` chains parse without recursion but build a left-nested + // tree as deep as the chain, which formatting, evaluation, cloning, + // comparison and dropping walk recursively. 100000 operators used to + // overflow the stack in all of them (found in review). + let filter = |operators: usize, op: &str| { + let operands = vec!["@ == 0"; operators + 1]; + format!("$ ? ({})", operands.join(&format!(" {op} "))) + }; + let doc: OwnedJsonb = "[0,1]".parse().unwrap(); + // The limit runs on the test thread's default stack, without `with_big_stack`. + for op in ["||", "&&"] { + let text = filter(MAX_PATH_LOGICAL_OPERATORS, op); + let path = parse_json_path(text.as_bytes()).unwrap(); + let formatted = path.to_string(); + assert_eq!(parse_json_path(formatted.as_bytes()).unwrap(), path.clone()); + assert_eq!(doc.as_raw().select_by_path(&path).unwrap().len(), 1); + for operators in [MAX_PATH_LOGICAL_OPERATORS + 1, 100_000] { + assert_eq!( + parse_json_path(filter(operators, op).as_bytes()).err(), + Some(Error::RecursionLimitExceeded), + "{operators} {op}" + ); + } + } + // The limit counts operators across the whole path, including root + // expressions and nested filters. + let root = vec!["$[0] == 0"; MAX_PATH_LOGICAL_OPERATORS + 2].join(" || "); + assert!(parse_json_path(root.as_bytes()).is_err()); + let half = vec!["@ == 0"; MAX_PATH_LOGICAL_OPERATORS / 2 + 1].join(" || "); + let nested = format!("$ ? (exists(@ ? ({half})) || {half})"); + assert!(parse_json_path(nested.as_bytes()).is_err()); + // Operators inside string literals do not count. + let quoted = format!(r#"$ ? (@ == "{}")"#, "||&&".repeat(1000)); + assert!(parse_json_path(quoted.as_bytes()).is_ok()); +} + +#[test] +fn json_path_logical_operator_limit_counts_only_operators() { + // `&` does not end an unquoted field name, so `a&&a` is one name. The + // limit used to be counted by scanning the text, which rejected names + // with more than 256 `&&`, and broke `Display` round-trips of quoted ones + // because `Display` writes them unquoted (found in review). + let doc: OwnedJsonb = r#"{"a&&a": 1}"#.parse().unwrap(); + let path = parse_json_path(b"$.a&&a").unwrap(); + assert_eq!(doc.as_raw().select_by_path(&path).unwrap().len(), 1); + let name = format!("{}a", "a&&".repeat(1000)); + for text in [format!("$.{name}"), format!(r#"$."{name}""#)] { + let path = parse_json_path(text.as_bytes()).unwrap(); + let formatted = path.to_string(); + assert_eq!(parse_json_path(formatted.as_bytes()).unwrap(), path); + } + + // Each form has exactly `k` operators. The parser backtracks over some of + // them, for example trying a root expression before a plain path, so a + // chain can be parsed more than once; each operator must count once. + fn chain(k: usize, op: &str) -> String { + vec!["@ == 0"; k + 1].join(&format!(" {op} ")) + } + /// A name and a function that writes the form with `k` operators. + type Form = (&'static str, fn(usize) -> String); + let forms: [Form; 8] = [ + ("filter with ||", |k| format!("$ ? ({})", chain(k, "||"))), + ("filter with &&", |k| format!("$ ? ({})", chain(k, "&&"))), + ("filter after a step", |k| { + format!("$.a ? ({})", chain(k, "||")) + }), + ("filter on items", |k| { + format!("$.a[*] ? ({})", chain(k, "&&")) + }), + ("root expression", |k| { + vec!["$.a[0] == 0"; k + 1].join(" || ") + }), + ("mixed && and ||", |k| { + let mut text = String::from("$ ? (@ == 0"); + for i in 0..k { + text.push_str(if i % 2 == 0 { + " && @ == 0" + } else { + " || @ == 0" + }); + } + text + ")" + }), + ("split by parentheses", |k| { + let left = k / 2; + let right = k - left - 1; + format!("$ ? (({}) && ({}))", chain(left, "||"), chain(right, "||")) + }), + ("split by a nested filter", |k| { + let inner = k / 2; + let outer = k - inner - 1; + format!( + "$ ? (exists(@ ? ({})) || {})", + chain(inner, "||"), + chain(outer, "||") + ) + }), + ]; + let doc: OwnedJsonb = r#"{"a": [0, 1]}"#.parse().unwrap(); + for (form, path) in forms { + let text = path(MAX_PATH_LOGICAL_OPERATORS); + let parsed = parse_json_path(text.as_bytes()).unwrap_or_else(|e| panic!("{form}: {e:?}")); + let formatted = parsed.to_string(); + assert_eq!( + parse_json_path(formatted.as_bytes()).as_ref(), + Ok(&parsed), + "{form}: round-trip" + ); + assert!(doc.as_raw().select_by_path(&parsed).is_ok(), "{form}"); + assert_eq!( + parse_json_path(path(MAX_PATH_LOGICAL_OPERATORS + 1).as_bytes()).err(), + Some(Error::RecursionLimitExceeded), + "{form}" + ); + } +} + +#[test] +fn recursive_descent_reaches_levels_beyond_255() { + with_big_stack(|| { + // Levels are `u8`, so an open `to last` range is the only way to match + // past level 255; the innermost `[]` of 300 nested arrays is level 299. + let deep = nested_arrays(300); + let path = parse_json_path(b"$.**{255 to last}").unwrap(); + let selected = deep.as_raw().select_by_path(&path).unwrap(); + assert_eq!(selected.len(), 300 - 255); + assert_eq!(selected.last().unwrap().to_string(), "[]"); + }); +} + #[test] fn oversized_container_headers_are_rejected_early() { // Each header declares 2^29 - 1 entries but no JEntries follow. @@ -804,6 +971,64 @@ fn signed_indexes_do_not_overflow() { .to_string(), "[1,2,3,1]" ); + for path in [ + "$[last + 2147483647]", + "$[last - 2147483647]", + "$[-2147483648]", + ] { + let path = parse_json_path(path.as_bytes()).unwrap(); + assert!(raw.select_by_path(&path).unwrap().is_empty()); + } +} + +#[test] +fn json_path_slices_outside_the_array_select_nothing() { + // These used to loop over `0..=usize::MAX` after an end index underflowed. + let array = "[1,2]".parse::().unwrap(); + let empty = "[]".parse::().unwrap(); + for (doc, path) in [ + (&array, "$[0 to last - 5]"), + (&empty, "$[0 to 3]"), + (&array, "$[5 to 9]"), + ] { + let path = parse_json_path(path.as_bytes()).unwrap(); + assert!(doc.as_raw().select_by_path(&path).unwrap().is_empty()); + } + let path = parse_json_path(b"$[1 to 9]").unwrap(); + let selected = array.as_raw().select_by_path(&path).unwrap(); + assert_eq!(selected.len(), 1); + assert_eq!(selected[0].to_string(), "2"); +} + +#[test] +fn path_display_round_trips_special_names() { + // Names that need quoting or escaping when shown. The parsers reject empty + // quoted names, so `""` is not included. + for name in [r#"st"ore"#, "a b", "a.b", "a\\b", "$x", "1st", "tab\there"] { + let quoted = serde_json::to_string(name).unwrap(); + for text in [ + format!("$.{quoted}"), + format!("$[{quoted}]"), + format!("$:{quoted}"), + format!("a:{quoted}"), + ] { + let path = parse_json_path(text.as_bytes()).unwrap(); + let shown = path.to_string(); + assert_eq!( + parse_json_path(shown.as_bytes()).unwrap(), + path, + "{text} shown as {shown}" + ); + } + let text = format!("{{{quoted},0}}"); + let key_paths = parse_key_paths(text.as_bytes()).unwrap(); + let shown = key_paths.to_string(); + assert_eq!( + parse_key_paths(shown.as_bytes()).unwrap(), + key_paths, + "{text} shown as {shown}" + ); + } } /// Runs `f` on its own thread and fails if it does not finish within 10 seconds. @@ -885,3 +1110,126 @@ fn indexing_reports_errors_in_skipped_elements() { let path = parse_key_paths(b"{2}").unwrap(); assert_eq!(raw.get_by_keypath(path.paths.iter()), Ok(None)); } + +#[test] +fn json_path_evaluation_is_bounded() { + // Chained `.**` steps produce about C(depth + k - 1, k) items: 4 steps on + // 100 nested arrays select 4.4 million, the fuzzer's 90 steps far more. + let nested: OwnedJsonb = format!("{}{}", "[".repeat(100), "]".repeat(100)) + .parse() + .unwrap(); + let fuzzed = "$.**.**.**.**.**.**.**.**.**.**.**.**.**.**.**.**.**.**.**.**.**.**.**.**.**.**.**.**.**.**.**.**.**.**.**.**.**.**.**.**.**.**.**.**.**.**.**.**.**.**.**.**.**.**.**.**.**.**.* .**.**.**.**.**.**.**.**.**.**.**.* .**.**.**.**.**.**.**.**.**.**.*[1].**[1]"; + for (doc, path) in [ + (nested.clone(), "$.**.**.**.**".to_string()), + (nested.clone(), fuzzed.to_string()), + (r#"{"a":[[[1]]]}"#.parse().unwrap(), fuzzed.to_string()), + ] { + let result = within_timeout(&path.clone(), move || { + let path = parse_json_path(path.as_bytes()).unwrap(); + let raw = doc.as_raw(); + [ + raw.select_by_path(&path).map(|_| ()), + raw.select_array_by_path(&path).map(|_| ()), + raw.select_first_by_path(&path).map(|_| ()), + raw.select_value_by_path(&path).map(|_| ()), + raw.path_exists(&path).map(|_| ()), + ] + }); + for r in result { + assert_eq!(r, Err(Error::JsonPathTooComplex)); + } + } + + // Comparisons check every pair until one matches. No pair matches here, + // so 2000 x 2000 values is 4 million comparisons. + let list = + |range: std::ops::Range| range.map(|i| i.to_string()).collect::>().join(","); + let values: OwnedJsonb = format!(r#"{{"a":[{}],"b":[{}]}}"#, list(0..2000), list(-2000..0)) + .parse() + .unwrap(); + let path = parse_json_path(b"$ ? (@.a[*] < @.b[0])").unwrap(); + assert_eq!(values.as_raw().path_exists(&path), Ok(false)); + let path = parse_json_path(b"$ ? (@.a[*] < @.b[*])").unwrap(); + assert_eq!( + values.as_raw().path_exists(&path), + Err(Error::JsonPathTooComplex) + ); + + // Ordinary paths on large documents stay well within the budget, and + // the budget is reset for every evaluation. + let large: OwnedJsonb = format!( + "[{}]", + (0..300_000) + .map(|i| format!("{{\"a\":{i}}}")) + .collect::>() + .join(",") + ) + .parse() + .unwrap(); + let path = parse_json_path(b"$[*].a").unwrap(); + for _ in 0..3 { + assert_eq!(large.as_raw().select_by_path(&path).unwrap().len(), 300_000); + } + let path = parse_json_path(b"$.**").unwrap(); + assert_eq!(large.as_raw().select_by_path(&path).unwrap().len(), 600_001); +} + +#[test] +fn json_path_copies_are_bounded() { + // Debug builds need more than the default stack to walk 500 levels. + with_big_stack(|| { + // Every ancestor of the string is selected and copied with it: 125,751 + // items, within the item budget, but about 300 MB of copies. + let doc: OwnedJsonb = format!( + "{}\"{}\"{}", + "[".repeat(500), + "x".repeat(1024), + "]".repeat(500) + ) + .parse() + .unwrap(); + let path = parse_json_path(b"$.**.**").unwrap(); + let raw = doc.as_raw(); + assert_eq!( + raw.select_by_path(&path).err(), + Some(Error::JsonPathTooComplex) + ); + assert_eq!( + raw.select_array_by_path(&path).err(), + Some(Error::JsonPathTooComplex) + ); + assert_eq!( + raw.select_value_by_path(&path).err(), + Some(Error::JsonPathTooComplex) + ); + // Checks that copy nothing are not limited by the byte budget. + assert_eq!(raw.path_exists(&path), Ok(true)); + // A single `.**` copies 500 KiB, which is allowed. + let path = parse_json_path(b"$.**").unwrap(); + assert_eq!(raw.select_by_path(&path).unwrap().len(), 501); + }); + + // Any single value can be copied, even one larger than 32 MiB, because + // the budget includes the document size. + let big: OwnedJsonb = format!(r#"{{"a":"{}","b":1}}"#, "x".repeat(33 << 20)) + .parse() + .unwrap(); + let raw = big.as_raw(); + let path = parse_json_path(b"$.a").unwrap(); + assert_eq!( + raw.select_first_by_path(&path).unwrap().unwrap().len(), + // Scalar header and JEntry, then the string. + 8 + (33 << 20) + ); + let path = parse_json_path(b"$").unwrap(); + assert_eq!( + raw.select_first_by_path(&path).unwrap().unwrap().len(), + raw.len() + ); + // Copying it twice goes 33 MiB past the document size. + let path = parse_json_path(b"$.**").unwrap(); + assert_eq!( + raw.select_by_path(&path).err(), + Some(Error::JsonPathTooComplex) + ); +} diff --git a/tests/it/number.rs b/tests/it/number.rs index 60d311b..86410f8 100644 --- a/tests/it/number.rs +++ b/tests/it/number.rs @@ -34,11 +34,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -#[cfg(feature = "arbitrary_precision")] -use jsonb::Decimal64; +use std::cmp::Ordering; + +use ethnum::i256; use jsonb::RawJsonb; use jsonb::{parse_value, parse_value_standard_mode}; -use jsonb::{Number, Value}; +use jsonb::{Decimal128, Decimal256, Decimal64, Number, Value}; use ordered_float::OrderedFloat; #[test] @@ -2333,6 +2334,426 @@ fn decimal_text_is_limited_to_128_bytes() { } } +/// Without `arbitrary_precision`, decimals serialize as `f64`, which must be +/// the same `f64` as `as_f64`. The serializer used to convert the unscaled +/// value to `f64` before dividing, rounding twice, so `123456789012390.47` +/// serialized as `123456789012390.48`. +#[cfg(not(feature = "arbitrary_precision"))] +#[test] +fn decimal_serializes_as_correctly_rounded_f64() { + let cases: [(i64, u8, &str); 3] = [ + (12_345_678_901_239_047, 2, "123456789012390.47"), + (-12_345_678_901_239_047, 3, "-12345678901239.047"), + (1, 23, "1e-23"), + ]; + for (value, scale, expect) in cases { + let numbers = [ + Number::Decimal64(Decimal64 { scale, value }), + Number::Decimal128(Decimal128 { + scale, + value: value.into(), + }), + Number::Decimal256(Decimal256 { + scale, + value: value.into(), + }), + ]; + for number in numbers { + assert_eq!( + serde_json::to_string(&number).unwrap(), + expect, + "{number:?}" + ); + assert_eq!( + serde_json::to_value(&number).unwrap().as_f64(), + Some(number.as_f64()) + ); + } + } +} + +#[test] +fn decimal_to_f64_rounds_once() { + // Each case is (decimal value, scale, f64 parsed from the same digits). + // Converting the unscaled value to f64 before dividing rounded twice for + // values above 2^53 and scales above 22 (found by fuzzing). + let cases: [(i64, u8, f64); 4] = [ + (12_345_678_901_239_047, 2, 123456789012390.47), + (-12_345_678_901_239_047, 3, -12345678901239.047), + (1, 23, 1e-23), + (i64::MAX, 0, 9223372036854775807.0), + ]; + for (value, scale, expect) in cases { + let decimal = Decimal64 { scale, value }; + assert_eq!(decimal.to_float64(), expect, "{value}e-{scale}"); + let decimal = Decimal128 { + scale, + value: value.into(), + }; + assert_eq!(decimal.to_float64(), expect, "{value}e-{scale}"); + let decimal = Decimal256 { + scale, + value: value.into(), + }; + assert_eq!(decimal.to_float64(), expect, "{value}e-{scale}"); + } + let decimal = Decimal256 { + scale: 76, + value: i256::MAX, + }; + assert_eq!(decimal.to_float64(), 5.789604461865809); + + // So a float and the decimal its text parses back to compare equal. + for text in ["1.2345678901239047e14", "12345678901239047e-2", "1e-23"] { + let float = parse_value(text.as_bytes()).unwrap(); + assert!(matches!(float, Value::Number(Number::Float64(_))), "{text}"); + let float = float.to_vec().unwrap(); + let reparsed = parse_value(RawJsonb::new(&float).to_string().as_bytes()) + .unwrap() + .to_vec() + .unwrap(); + assert_eq!( + RawJsonb::new(&reparsed).partial_cmp(&RawJsonb::new(&float)), + Some(Ordering::Equal), + "{text}" + ); + } +} + +#[test] +fn decimal_to_f64_matches_std_parse() { + // `to_float64` must return the correctly rounded `value / 10^scale`, which + // the standard library's parser computes independently from the same + // digits. Covers both conversion paths (exact division and parsing) and + // the boundaries between them: 2^53, scale 22, `i64`/`i128` limits. + let mut values: Vec = vec![0, i128::MAX, i128::MIN, i128::MIN + 1]; + for bound in [1_i128 << 53, i64::MAX.into(), i64::MIN.into()] { + values.extend([bound - 1, bound, bound + 1]); + } + for power in 0..39 { + let ten = 10_i128.pow(power); + values.extend([ten - 1, ten, ten + 1]); + } + // xorshift with random shifts, so every magnitude shows up deterministically. + let mut state: u64 = 0x2545_f491_4f6c_dd1d; + let mut next = || { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state + }; + for _ in 0..300 { + let bits = (u128::from(next()) << 64 | u128::from(next())) as i128; + values.push(bits >> (next() % 128)); + } + let negated: Vec = values.iter().filter_map(|v| v.checked_neg()).collect(); + values.extend(negated); + + for scale in 0..=u8::MAX { + for &value in &values { + let expect: f64 = format!("{value}e-{scale}").parse().unwrap(); + if let Ok(value) = i64::try_from(value) { + let got = Decimal64 { scale, value }.to_float64(); + assert_eq!(got.to_bits(), expect.to_bits(), "{value}e-{scale}"); + } + let got = Decimal128 { scale, value }.to_float64(); + assert_eq!(got.to_bits(), expect.to_bits(), "{value}e-{scale}"); + let got = Decimal256 { + scale, + value: value.into(), + } + .to_float64(); + assert_eq!(got.to_bits(), expect.to_bits(), "{value}e-{scale}"); + } + // `i256` values beyond `i128` take the decimal text path. + for value in [i256::MAX, i256::MIN + 1, i256::from(i128::MAX) * 10 + 7] { + for value in [value, -value] { + let expect: f64 = format!("{value}e-{scale}").parse().unwrap(); + let got = Decimal256 { scale, value }.to_float64(); + assert_eq!(got.to_bits(), expect.to_bits(), "{value}e-{scale}"); + } + } + } +} + +#[test] +fn decimal_to_f64_eisel_lemire_edge_cases() { + // Each value is checked against the standard library's parse of the same + // digits, as `e-`. + fn check(value: i128, scale: u8) { + let expect: f64 = format!("{value}e-{scale}").parse().unwrap(); + let got = Decimal128 { scale, value }.to_float64(); + assert_eq!(got.to_bits(), expect.to_bits(), "{value}e-{scale}"); + if let Ok(value) = i64::try_from(value) { + let got = Decimal64 { scale, value }.to_float64(); + assert_eq!(got.to_bits(), expect.to_bits(), "{value}e-{scale}"); + } + } + // Exact ties between two `f64` values round to the even mantissa: + // 2^53 + 1 lies halfway between 2^53 and 2^53 + 2. Scaling by 10^k keeps + // the tie and exercises the round-to-even exponents. + for base in [ + (1_i128 << 53) + 1, + (1 << 53) + 3, + (1 << 54) + 2, + (1 << 54) + 6, + ] { + for k in 0..4 { + let value = base * 10_i128.pow(k); + check(value, k as u8); + check(-value, k as u8); + } + } + // Products whose low 64 bits are all ones for a scale above 27: here + // Eisel-Lemire cannot decide the rounding and the text parse decides. + // Each value has its top bit set and `value * t ≡ -1 (mod 2^64)`, where + // `t` is the first word of the 128-bit power-of-five entry for the scale. + for (value, scale) in [ + (14_999_775_663_049_351_983_i128, 30), + (16_185_735_156_734_648_777, 35), + (11_378_377_293_076_434_211, 37), + (18_309_340_392_174_713_697, 251), + ] { + check(value, scale); + check(-value, scale); + } + // `u64` limits: the largest mantissa Eisel-Lemire takes, and the first + // that falls back because it does not fit. + for value in [u64::MAX.into(), i128::from(u64::MAX) + 1] { + for scale in [0, 1, 19, 20, 22, 23, 38, 255] { + check(value, scale); + check(-value, scale); + } + } +} + +#[test] +fn integer_arithmetic_is_exact() { + use Number::{Int64, UInt64}; + // Mixed `Int64`/`UInt64` operands used to be cast to one type first, so + // results were silently wrong (found by fuzzing). + let cases: [(&str, Number, Number, Option); 14] = [ + ( + "add", + UInt64(u64::MAX), + Int64(-1), + Some(UInt64(u64::MAX - 1)), + ), + ( + "add", + Int64(-1), + UInt64(u64::MAX), + Some(UInt64(u64::MAX - 1)), + ), + ( + "add", + Int64(i64::MIN), + UInt64(u64::MAX), + Some(UInt64(i64::MAX as u64)), + ), + ("sub", UInt64(u64::MAX), UInt64(0), Some(UInt64(u64::MAX))), + ("sub", UInt64(5), UInt64(10), Some(Int64(-5))), + ( + "sub", + UInt64(u64::MAX - 1), + Int64(-1), + Some(UInt64(u64::MAX)), + ), + ("sub", Int64(0), UInt64(u64::MAX), None), + ("mul", Int64(-1), UInt64(u64::MAX), None), + ("mul", Int64(-1), UInt64(1 << 63), Some(Int64(i64::MIN))), + ("rem", Int64(i64::MIN), Int64(-1), Some(Int64(0))), + ("rem", Int64(-7), UInt64(u64::MAX), Some(Int64(-7))), + ("rem", UInt64(u64::MAX), Int64(-2), Some(UInt64(1))), + // Two `Int64` operands keep an `Int64` result. + ("mul", Int64(i64::MIN), Int64(-1), None), + ("add", Int64(i64::MAX), Int64(1), None), + ]; + for (op, a, b, expect) in cases { + let got = match op { + "add" => a.add(b.clone()), + "sub" => a.sub(b.clone()), + "mul" => a.mul(b.clone()), + _ => a.rem(b.clone()), + }; + match (got, expect) { + (Ok(got), Some(expect)) => assert_eq!( + format!("{got:?}"), + format!("{expect:?}"), + "{a:?} {op} {b:?}" + ), + (Err(_), None) => {} + (got, expect) => panic!("{a:?} {op} {b:?}: got {got:?}, expected {expect:?}"), + } + } + assert!(Int64(1).rem(UInt64(0)).is_err()); +} + +#[test] +fn neg_reports_overflow() { + // Negating the minimum decimal used to overflow (a panic with overflow + // checks); `-(2^63)` as `UInt64` used to be rejected although it fits. + let decimal = Number::Decimal64(Decimal64 { + scale: 2, + value: i64::MIN, + }); + assert!(decimal.neg().is_err()); + let decimal = Number::Decimal128(Decimal128 { + scale: 2, + value: i128::MIN, + }); + assert!(decimal.neg().is_err()); + let negated = Number::UInt64(1 << 63).neg().unwrap(); + assert_eq!( + format!("{negated:?}"), + format!("{:?}", Number::Int64(i64::MIN)) + ); + assert!(Number::UInt64((1 << 63) + 1).neg().is_err()); + assert!(Number::Int64(i64::MIN).neg().is_err()); +} + +/// Integers and decimals compare by their exact values, and a float as its +/// shortest decimal. Converting integers and decimals to `f64` to compare them +/// with floats made the order intransitive (found by fuzzing): `2^53 + 1` +/// equaled the float `2^53`, which equals `2^53`, though `2^53 + 1 > 2^53`. +#[test] +fn number_order_is_exact_and_transitive() { + use Ordering::*; + let two_53 = 1u64 << 53; + let dec64 = |value: i64, scale: u8| Number::Decimal64(Decimal64 { scale, value }); + let dec128 = |value: i128, scale: u8| Number::Decimal128(Decimal128 { scale, value }); + let dec256 = |value: i256, scale: u8| Number::Decimal256(Decimal256 { scale, value }); + let i256_max_div_1e16 = i256::MAX / i256::from(10_000_000_000_000_000_i64); + let float = Number::Float64; + let cases = [ + // Integers that round to the float they are compared with. + (Number::UInt64(two_53 + 1), float(two_53 as f64), Greater), + (Number::UInt64(two_53), float(two_53 as f64), Equal), + ( + Number::Int64(-(two_53 as i64) - 1), + float(-(two_53 as f64)), + Less, + ), + // `u64::MAX as f64` is 2^64, whose shortest decimal is + // 18446744073709552000. + (Number::UInt64(u64::MAX), float(u64::MAX as f64), Less), + ( + dec128(18446744073709551617, 0), + float(u64::MAX as f64), + Less, + ), + ( + dec128(18446744073709552000, 0), + float(u64::MAX as f64), + Equal, + ), + // The float 0.1 is the decimal 0.1, not its binary value + // 0.1000000000000000055511151231257827... + (dec64(1, 1), float(0.1), Equal), + ( + dec128(1_000_000_000_000_000_055_511_151_231_257_827, 34), + float(0.1), + Greater, + ), + (dec64(1, 255), float(1e-255), Equal), + (dec64(1, 255), float(5e-324), Greater), + (dec256(i256::MAX, 0), float(f64::MAX), Less), + // The fuzzed input: 12345678901234567891.0, 1.2345678901234567e19 + // and 12345678901234567890. + ( + dec128(123456789012345678910, 1), + float(1.2345678901234567e19), + Greater, + ), + ( + Number::UInt64(12345678901234567890), + float(1.2345678901234567e19), + Greater, + ), + ( + dec128(123456789012345678910, 1), + Number::UInt64(12345678901234567890), + Greater, + ), + // Decimals whose rescaled values overflow `i256`, which compared as + // `f64` and so as equal. + ( + dec256(i256_max_div_1e16 + 1, 0), + dec256(i256::MAX, 16), + Greater, + ), + (dec256(i256_max_div_1e16, 0), dec256(i256::MAX, 16), Less), + (dec256(i256::ONE, 0), dec256(i256::ONE, 200), Greater), + // `NaN` is above every number, as in `OrderedFloat`, and zero has + // either sign. + (float(f64::NAN), float(f64::INFINITY), Greater), + (float(f64::NAN), dec256(i256::MAX, 0), Greater), + (float(f64::NAN), float(f64::NAN), Equal), + (float(f64::INFINITY), dec256(i256::MAX, 0), Greater), + (float(f64::NEG_INFINITY), dec256(i256::MIN, 0), Less), + (float(-0.0), Number::Int64(0), Equal), + (float(-0.0), dec64(0, 3), Equal), + (float(-1e-300), dec64(0, 3), Less), + ]; + for (l, r, expect) in &cases { + assert_eq!(l.cmp(r), *expect, "{l:?} vs {r:?}"); + assert_eq!(r.cmp(l), expect.reverse(), "{r:?} vs {l:?}"); + } + + // Every triple of these numbers is ordered transitively, and the + // comparable encoding, which converts numbers to `f64`, never reverses + // their order. + let mut numbers: Vec = cases + .iter() + .flat_map(|(l, r, _)| [l.clone(), r.clone()]) + .collect(); + for v in [two_53 - 1, two_53 + 2, u64::MAX - 1] { + numbers.extend([Number::UInt64(v), float(v as f64)]); + } + numbers.extend([ + float(f64::from_bits((two_53 as f64).to_bits() + 1)), + dec64(i64::MIN, 0), + ]); + let comparable = |n: &Number| { + let encoded = Value::Number(n.clone()).to_vec().unwrap(); + RawJsonb::new(&encoded).convert_to_comparable() + }; + for x in &numbers { + for y in &numbers { + let order = x.cmp(y); + assert_eq!(order, y.cmp(x).reverse(), "{x:?} vs {y:?}"); + let encoded = comparable(x).cmp(&comparable(y)); + assert!( + encoded == order || encoded == Equal, + "comparable encoding of {x:?} vs {y:?} is {encoded:?}, but they compare {order:?}" + ); + for z in &numbers { + if order != Greater && y.cmp(z) != Greater { + assert_ne!(x.cmp(z), Greater, "{x:?} <= {y:?} <= {z:?}"); + } + } + } + } + + // The same three values as JSONB, from JSON text. Without + // `arbitrary_precision`, the first parses as the same float as the second. + let raw = |text: &str| text.parse::().unwrap(); + #[cfg(feature = "arbitrary_precision")] + { + let a = raw("12345678901234567891.0"); + let b = raw("1.2345678901234567e19"); + let c = raw("12345678901234567890"); + assert_eq!(a.as_raw().partial_cmp(&b.as_raw()), Some(Greater)); + assert_eq!(c.as_raw().partial_cmp(&b.as_raw()), Some(Greater)); + assert_eq!(a.as_raw().partial_cmp(&c.as_raw()), Some(Greater)); + } + + // A decimal in a document still equals the float literal of a JSONPath. + let doc = raw(r#"{"a": 1.1}"#); + let path = jsonb::jsonpath::parse_json_path(b"$.a ? (@ == 1.1)").unwrap(); + assert_eq!(doc.as_raw().select_by_path(&path).unwrap().len(), 1); +} + #[test] fn comparable_encoding_treats_negative_zero_as_zero() { // `-0.0` compares equal to `0` but used to encode below it. diff --git a/tests/it/parser.rs b/tests/it/parser.rs index 52c8c2a..1aad7bf 100644 --- a/tests/it/parser.rs +++ b/tests/it/parser.rs @@ -213,10 +213,6 @@ fn test_parse_f64() { ("0.00e+00", Value::Number(Number::Float64(0.0))), ("0.00e-00", Value::Number(Number::Float64(0.0))), ("3.5E-2147483647", Value::Number(Number::Float64(0.0))), - ( - "0.0100000000000000000001", - Value::Number(Number::Float64(0.01)), - ), ( &i64_min_minus_one, Value::Number(Number::Float64((i64::MIN as f64) - 1.0)), @@ -234,14 +230,6 @@ fn test_parse_f64() { "100e-777777777777777777777777777", Value::Number(Number::Float64(0.0)), ), - ( - "1010101010101010101010101010101010101010", - Value::Number(Number::Float64(1.010_101_010_101_01e39)), - ), - ( - "0.1010101010101010101010101010101010101010", - Value::Number(Number::Float64(0.101_010_101_010_101_01)), - ), ( "0e1000000000000000000000000000000000000000000000", Value::Number(Number::Float64(0.0)), @@ -286,6 +274,47 @@ fn test_parse_f64() { test_parse_ok(tests.clone()); test_parse_standard_ok(tests); + // More significant digits than `f64` holds. The standard mode parses them + // as the nearest `f64`, and so does the extended mode without + // `arbitrary_precision`; with it, they parse as exact decimals. These used + // to expect floats from both modes, and passed only because decimals + // compared with floats as `f64`. + let long = [ + ("0.0100000000000000000001", 0.01), + ( + "1010101010101010101010101010101010101010", + 1.010_101_010_101_01e39, + ), + ( + "0.1010101010101010101010101010101010101010", + 0.101_010_101_010_101_01, + ), + ]; + let floats = long.map(|(s, f)| (s, Value::Number(Number::Float64(f)))); + test_parse_standard_ok(floats.to_vec()); + #[cfg(not(feature = "arbitrary_precision"))] + test_parse_ok(floats.to_vec()); + #[cfg(feature = "arbitrary_precision")] + { + let digits: ethnum::i256 = "1010101010101010101010101010101010101010".parse().unwrap(); + let decimals = [ + Number::Decimal128(jsonb::Decimal128 { + scale: 22, + value: 100000000000000000001, + }), + Number::Decimal256(jsonb::Decimal256 { + scale: 0, + value: digits, + }), + Number::Decimal256(jsonb::Decimal256 { + scale: 40, + value: digits, + }), + ]; + let tests = long.iter().zip(decimals); + test_parse_ok(tests.map(|((s, _), n)| (*s, Value::Number(n))).collect()); + } + // Extended JSON number syntax let extended_tests = vec![ ("+1", Value::Number(Number::Int64(1))), @@ -568,6 +597,9 @@ fn test_value_display_non_finite_floats_as_null() { assert_eq!(formatted, raw); assert_eq!(Number::Float64(f64::NAN).to_string(), "NaN"); assert_eq!(Number::Float64(f64::NEG_INFINITY).to_string(), "-inf"); + + let path = jsonb::jsonpath::parse_json_path(b"$ ? (@ > -inf)").unwrap(); + assert_eq!(path.to_string(), "$?(@ > -inf)"); } #[test] diff --git a/tests/it/testdata/json_path.txt b/tests/it/testdata/json_path.txt index 4cecbfc..0bc2633 100644 --- a/tests/it/testdata/json_path.txt +++ b/tests/it/testdata/json_path.txt @@ -555,7 +555,7 @@ JsonPath { ---------- Input ---------- $."st\"ore"."book\uD83D\uDC8E" ---------- Output --------- -$.st"ore.book💎 +$."st\"ore".book💎 ---------- AST ------------ JsonPath { paths: [ From 6a344904334b591fa001209be653a3c10513103f Mon Sep 17 00:00:00 2001 From: baishen Date: Tue, 17 Nov 2026 10:37:05 +0800 Subject: [PATCH 2/3] fix --- src/constants.rs | 6 +++--- src/error.rs | 6 +++--- src/jsonpath/path.rs | 36 +++++++++++++++++++++++++++++++++++ src/jsonpath/selector.rs | 24 +++++++++++++---------- tests/it/malformed.rs | 41 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 97 insertions(+), 16 deletions(-) diff --git a/src/constants.rs b/src/constants.rs index 3241502..0347554 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -31,9 +31,9 @@ pub(crate) const MAX_JSON_PATH_DEPTH: usize = 64; /// The parser counts them because unquoted field names may contain `&&`. pub(crate) const MAX_JSON_PATH_LOGICAL_OPERATORS: usize = 256; -/// Maximum work units for one JSONPath evaluation. Each selected item, filter -/// value, node visited by `.**` and compared pair costs one unit; without a -/// limit, chained `.**` steps grow combinatorially with the nesting depth. +/// Base work units for one JSONPath evaluation; see `Selector::reset_budgets`. +/// Each selected item, filter value, node visited by `.**` and compared pair +/// costs one unit. pub(crate) const MAX_JSON_PATH_ITEMS: usize = 1_000_000; /// Bytes one JSONPath evaluation may copy beyond the document size. A value diff --git a/src/error.rs b/src/error.rs index dcf35de..58b01c8 100644 --- a/src/error.rs +++ b/src/error.rs @@ -116,9 +116,9 @@ pub enum Error { /// an array/object has 2^29 or more elements, which the JSONB length fields /// cannot represent. ValueTooLarge, - /// Evaluating a JSONPath expression selected or compared more than - /// 1,000,000 items, for example through chained `.**` steps, or copied - /// more than 32 MiB beyond the size of the document. + /// Evaluating a JSONPath expression did more work than 1,000,000 units + /// plus the document size times the path size, for example through + /// chained `.**` steps, or copied more than 32 MiB beyond the document. JsonPathTooComplex, Message(String), diff --git a/src/jsonpath/path.rs b/src/jsonpath/path.rs index 17e5b6c..1b8cde0 100644 --- a/src/jsonpath/path.rs +++ b/src/jsonpath/path.rs @@ -32,6 +32,28 @@ impl JsonPath<'_> { pub fn is_predicate(&self) -> bool { self.paths.len() == 1 && matches!(self.paths[0], Path::Expr(_)) } + + /// Returns the number of steps and expression nodes in the path, at least 1. + pub(crate) fn node_count(&self) -> usize { + Path::node_count_of(&self.paths).max(1) + } +} + +impl Path<'_> { + /// Returns the number of steps and expression nodes in `paths`. + fn node_count_of(paths: &[Path<'_>]) -> usize { + paths + .iter() + .map(Path::node_count) + .fold(0, usize::saturating_add) + } + + fn node_count(&self) -> usize { + match self { + Path::FilterExpr(expr) | Path::Expr(expr) => expr.node_count().saturating_add(1), + _ => 1, + } + } } /// Represents a valid JSON Path. @@ -539,6 +561,20 @@ impl Display for Expr<'_> { } impl Expr<'_> { + /// Returns the number of nodes in this expression, including path steps. + fn node_count(&self) -> usize { + match self { + Expr::Paths(paths) => Path::node_count_of(paths), + Expr::Value(_) => 1, + Expr::BinaryOp { left, right, .. } => left + .node_count() + .saturating_add(right.node_count()) + .saturating_add(1), + Expr::UnaryOp { operand, .. } => operand.node_count().saturating_add(1), + Expr::ExistsFunc(paths) => Path::node_count_of(paths).saturating_add(1), + } + } + /// Returns whether this expression, an operand of `parent`, must be /// parenthesized for the formatted text to parse back to the same tree. /// diff --git a/src/jsonpath/selector.rs b/src/jsonpath/selector.rs index 0230ba4..a3f15e4 100644 --- a/src/jsonpath/selector.rs +++ b/src/jsonpath/selector.rs @@ -110,7 +110,7 @@ pub struct Selector<'a> { root_jsonb: RawJsonb<'a>, /// A queue holding the JSONB items that match the path criteria during execution. items: VecDeque>, - /// Work units left for the current evaluation; see [`MAX_JSON_PATH_ITEMS`]. + /// Work units left for the current evaluation; see `reset_budgets`. budget: usize, /// Bytes the current evaluation may still copy; see /// [`MAX_JSON_PATH_EXTRA_COPY_BYTES`]. @@ -124,20 +124,24 @@ impl<'a> Selector<'a> { /// /// * `root_jsonb` - The `RawJsonb` data to select from. pub fn new(root_jsonb: RawJsonb<'a>) -> Selector<'a> { - let mut selector = Self { + // `execute` sets the budgets for each evaluation. + Self { root_jsonb, items: VecDeque::new(), budget: 0, byte_budget: 0, - }; - selector.reset_budgets(); - selector + } } - /// Restores the full budgets for a new evaluation. The byte budget adds - /// the document size, so any single value can be copied at least once. - fn reset_budgets(&mut self) { - self.budget = MAX_JSON_PATH_ITEMS; + /// Restores the budgets for evaluating `json_path`. + /// + /// The work budget grows with the document and the path, so linear + /// queries on large documents fit, while chained `.**` steps and pairwise + /// comparisons still run out. The byte budget adds the document size, so + /// any single value can be copied once. + fn reset_budgets(&mut self, json_path: &JsonPath<'_>) { + let scaled = self.root_jsonb.len().saturating_mul(json_path.node_count()); + self.budget = MAX_JSON_PATH_ITEMS.saturating_add(scaled); self.byte_budget = MAX_JSON_PATH_EXTRA_COPY_BYTES.saturating_add(self.root_jsonb.len()); } @@ -502,7 +506,7 @@ impl<'a> Selector<'a> { fn execute(&mut self, json_path: &'a JsonPath<'a>) -> Result<()> { // add root jsonb let root_item = JsonbItem::Raw(self.root_jsonb); - self.reset_budgets(); + self.reset_budgets(json_path); self.items.clear(); self.items.push_front(root_item); diff --git a/tests/it/malformed.rs b/tests/it/malformed.rs index b8db47e..4f53b1e 100644 --- a/tests/it/malformed.rs +++ b/tests/it/malformed.rs @@ -1174,6 +1174,47 @@ fn json_path_evaluation_is_bounded() { assert_eq!(large.as_raw().select_by_path(&path).unwrap().len(), 600_001); } +#[test] +fn json_path_budget_scales_with_the_input() { + // A fixed budget of 1,000,000 units rejected `$[*] ? (@ == 2)` above about + // 333,000 elements (found in review). + let count = |doc: &OwnedJsonb, path: &str| { + let path = parse_json_path(path.as_bytes()).unwrap(); + doc.as_raw() + .select_by_path(&path) + .unwrap_or_else(|e| panic!("{path}: {e:?}")) + .len() + }; + let array = |n: usize, item: &dyn Fn(usize) -> String| -> OwnedJsonb { + format!("[{}]", (0..n).map(item).collect::>().join(",")) + .parse() + .unwrap() + }; + + let n = 1_000_000; + let ints = array(n, &|i| (i % 1000).to_string()); + assert_eq!(count(&ints, "$[*]"), n); + assert_eq!(count(&ints, "$[*] ? (@ == 2)"), n / 1000); + assert_eq!(count(&ints, "$[*] ? (@ > 0 && @ < 10)"), 9 * n / 1000); + assert_eq!( + count(&ints, "$[*] ? (@ == 2 || @ == 3 || @ == 4)"), + 3 * n / 1000 + ); + assert_eq!(count(&ints, "$.**"), n + 1); + + // The smallest values, 4 bytes each; every comparison is still charged. + let nulls = array(n, &|_| "null".to_string()); + assert_eq!(count(&nulls, "$[*] ? (@ == 1 || @ == 2 || @ == 3)"), 0); + + let m = 400_000; + let objects = array(m, &|i| format!(r#"{{"a":{},"b":"x"}}"#, i % 1000)); + assert_eq!( + count(&objects, r#"$[*] ? (@.a == 1 && @.b == "x")"#), + m / 1000 + ); + assert_eq!(count(&objects, "$.**"), 3 * m + 1); +} + #[test] fn json_path_copies_are_bounded() { // Debug builds need more than the default stack to walk 500 levels. From da755d6b9f27fb68e16bb57e918e49cada7f81c1 Mon Sep 17 00:00:00 2001 From: baishen Date: Tue, 17 Nov 2026 12:49:43 +0800 Subject: [PATCH 3/3] check decimal --- src/constants.rs | 6 +++ src/core/databend/util.rs | 15 +++---- src/number.rs | 61 +++++++++++++++++++++++++ src/parser.rs | 4 +- tests/it/decode.rs | 95 +++++++++++++++++++++++++++++++++++++++ tests/it/json_text.rs | 19 +++++--- tests/it/number.rs | 33 +++++++------- tests/it/parser.rs | 22 +++++++++ 8 files changed, 224 insertions(+), 31 deletions(-) diff --git a/src/constants.rs b/src/constants.rs index 0347554..57ec462 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use ethnum::i256; + // JSON text constants pub(crate) const UNICODE_LEN: usize = 4; @@ -103,6 +105,10 @@ pub(crate) const DECIMAL64_MIN: i128 = -999_999_999_999_999_999i128; pub(crate) const DECIMAL64_MAX: i128 = 999_999_999_999_999_999i128; pub(crate) const DECIMAL128_MIN: i128 = -99_999_999_999_999_999_999_999_999_999_999_999_999i128; pub(crate) const DECIMAL128_MAX: i128 = 99_999_999_999_999_999_999_999_999_999_999_999_999i128; +pub(crate) const DECIMAL256_MIN: i256 = + ethnum::int!("-9999999999999999999999999999999999999999999999999999999999999999999999999999"); +pub(crate) const DECIMAL256_MAX: i256 = + ethnum::int!("9999999999999999999999999999999999999999999999999999999999999999999999999999"); pub(crate) const NUMBER_STRUCT_TOKEN: &str = "$jsonb::private::Number"; pub(crate) const NUMBER_STRUCT_FIELD_SCALE: &str = "$jsonb::private::Number::Scale"; diff --git a/src/core/databend/util.rs b/src/core/databend/util.rs index b40240d..a6d4c18 100644 --- a/src/core/databend/util.rs +++ b/src/core/databend/util.rs @@ -503,34 +503,29 @@ impl Number { 9 => { let value = i64::from_be_bytes(bytes[1..9].try_into().unwrap()); let scale = u8::from_be_bytes(bytes[9..10].try_into().unwrap()); - let dec = Decimal64 { scale, value }; - Number::Decimal64(dec) + Number::Decimal64(Decimal64::new(scale, value)?) } 17 => { let value = i128::from_be_bytes(bytes[1..17].try_into().unwrap()); let scale = u8::from_be_bytes(bytes[17..18].try_into().unwrap()); - let dec = Decimal128 { scale, value }; - Number::Decimal128(dec) + Number::Decimal128(Decimal128::new(scale, value)?) } 18 => { // Compatible with deprecated Decimal128 formats, including precision let value = i128::from_be_bytes(bytes[1..17].try_into().unwrap()); let scale = u8::from_be_bytes(bytes[18..19].try_into().unwrap()); - let dec = Decimal128 { scale, value }; - Number::Decimal128(dec) + Number::Decimal128(Decimal128::new(scale, value)?) } 33 => { let value = i256::from_be_bytes(bytes[1..33].try_into().unwrap()); let scale = u8::from_be_bytes(bytes[33..34].try_into().unwrap()); - let dec = Decimal256 { scale, value }; - Number::Decimal256(dec) + Number::Decimal256(Decimal256::new(scale, value)?) } 34 => { // Compatible with deprecated Decimal256 formats, including precision let value = i256::from_be_bytes(bytes[1..33].try_into().unwrap()); let scale = u8::from_be_bytes(bytes[34..35].try_into().unwrap()); - let dec = Decimal256 { scale, value }; - Number::Decimal256(dec) + Number::Decimal256(Decimal256::new(scale, value)?) } _ => { return Err(Error::InvalidJsonbNumber); diff --git a/src/number.rs b/src/number.rs index c93189b..66a9adc 100644 --- a/src/number.rs +++ b/src/number.rs @@ -18,6 +18,13 @@ use std::fmt::Debug; use std::fmt::Display; use std::fmt::Formatter; +use crate::constants::DECIMAL128_MAX; +use crate::constants::DECIMAL256_MAX; +use crate::constants::DECIMAL256_MIN; +use crate::constants::DECIMAL64_MAX; +use crate::constants::MAX_DECIMAL128_PRECISION; +use crate::constants::MAX_DECIMAL256_PRECISION; +use crate::constants::MAX_DECIMAL64_PRECISION; use crate::eisel_lemire; use crate::error::Result; use crate::Error; @@ -142,6 +149,22 @@ pub struct Decimal64 { } impl Decimal64 { + /// Creates a decimal of `value / 10^scale`. + /// + /// # Errors + /// + /// Returns `Error::InvalidJsonbNumber` if `scale` or the digits of `value` + /// exceed precision 18. + pub fn new(scale: u8, value: i64) -> Result { + if usize::from(scale) <= MAX_DECIMAL64_PRECISION + && value.unsigned_abs() <= DECIMAL64_MAX as u64 + { + Ok(Self { scale, value }) + } else { + Err(Error::InvalidJsonbNumber) + } + } + /// Converts the decimal value to a floating-point representation. /// /// This is useful when you need to perform floating-point operations, @@ -165,6 +188,22 @@ pub struct Decimal128 { } impl Decimal128 { + /// Creates a decimal of `value / 10^scale`. + /// + /// # Errors + /// + /// Returns `Error::InvalidJsonbNumber` if `scale` or the digits of `value` + /// exceed precision 38. + pub fn new(scale: u8, value: i128) -> Result { + if usize::from(scale) <= MAX_DECIMAL128_PRECISION + && value.unsigned_abs() <= DECIMAL128_MAX as u128 + { + Ok(Self { scale, value }) + } else { + Err(Error::InvalidJsonbNumber) + } + } + /// Converts the decimal value to a floating-point representation. /// /// Note that for very large values, this conversion may lose precision @@ -189,6 +228,22 @@ pub struct Decimal256 { } impl Decimal256 { + /// Creates a decimal of `value / 10^scale`. + /// + /// # Errors + /// + /// Returns `Error::InvalidJsonbNumber` if `scale` or the digits of `value` + /// exceed precision 76. + pub fn new(scale: u8, value: i256) -> Result { + if usize::from(scale) <= MAX_DECIMAL256_PRECISION + && (DECIMAL256_MIN..=DECIMAL256_MAX).contains(&value) + { + Ok(Self { scale, value }) + } else { + Err(Error::InvalidJsonbNumber) + } + } + /// Converts the decimal value to a floating-point representation. /// /// For extremely large values, significant precision loss may occur @@ -1798,6 +1853,12 @@ mod tests { use super::*; use std::cmp::Ordering; + #[test] + fn decimal256_bounds_have_76_digits() { + assert_eq!(DECIMAL256_MAX, i256::from(10).pow(76) - 1); + assert_eq!(DECIMAL256_MIN, -DECIMAL256_MAX); + } + #[test] fn test_number_comparison() { // Test same type comparisons diff --git a/src/parser.rs b/src/parser.rs index 35e0c7c..5aa134d 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -925,7 +925,9 @@ impl<'a> Parser<'a> { } #[cfg(feature = "arbitrary_precision")] { - if value <= (DECIMAL128_MAX as u128 + 1) { + // Unlike `i64`, a `Decimal128` holds 38 digits, so + // `-10^38` needs a `Decimal256` as `10^38` does. + if value <= DECIMAL128_MAX as u128 { return Ok(JsonAst::Number(Number::Decimal128(Decimal128 { scale: 0, value: -(value as i128), diff --git a/tests/it/decode.rs b/tests/it/decode.rs index 8835dcf..724e307 100644 --- a/tests/it/decode.rs +++ b/tests/it/decode.rs @@ -191,6 +191,101 @@ fn test_decode_decimal() { } } +/// Every encoder keeps a decimal's scale and digits within the precision of +/// its width, 18, 38 or 76. Decoding used to accept any `u8` scale and any +/// value, which made callers overflow `10^scale`. +#[test] +fn decimals_beyond_their_precision_are_invalid() { + let e18 = 10_i64.pow(18); + let e38 = 10_i128.pow(38); + let e76 = I256::from(10).pow(76); + let valid = [ + Number::Decimal64(Decimal64 { + scale: 18, + value: e18 - 1, + }), + Number::Decimal64(Decimal64 { + scale: 0, + value: 1 - e18, + }), + Number::Decimal128(Decimal128 { + scale: 38, + value: e38 - 1, + }), + Number::Decimal128(Decimal128 { + scale: 0, + value: 1 - e38, + }), + Number::Decimal256(Decimal256 { + scale: 76, + value: e76 - 1, + }), + Number::Decimal256(Decimal256 { + scale: 0, + value: I256::ONE - e76, + }), + ]; + for number in valid { + let data = Value::Number(number.clone()).to_vec().unwrap(); + assert_eq!(from_slice(&data).unwrap(), Value::Number(number)); + } + let invalid = [ + Number::Decimal64(Decimal64 { + scale: 19, + value: 1, + }), + Number::Decimal64(Decimal64 { + scale: 255, + value: 1, + }), + Number::Decimal64(Decimal64 { + scale: 0, + value: e18, + }), + Number::Decimal64(Decimal64 { + scale: 2, + value: i64::MIN, + }), + Number::Decimal128(Decimal128 { + scale: 39, + value: 1, + }), + Number::Decimal128(Decimal128 { + scale: 0, + value: e38, + }), + Number::Decimal128(Decimal128 { + scale: 0, + value: -e38, + }), + Number::Decimal256(Decimal256 { + scale: 77, + value: I256::ONE, + }), + Number::Decimal256(Decimal256 { + scale: 0, + value: e76, + }), + Number::Decimal256(Decimal256 { + scale: 0, + value: I256::MIN, + }), + ]; + for number in invalid { + let data = Value::Number(number.clone()).to_vec().unwrap(); + assert!(from_slice(&data).is_err(), "{number:?}"); + } + // The deprecated widths, which also store a precision byte before the + // scale, with scales 39 and 77. + let deprecated = [ + b"\x20\0\0\0\x20\0\0\x13\x70\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x04\xD2\x26\x27".to_vec(), + b"\x20\0\0\0\x20\0\0\x23\x70\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x04\xD2\x4C\x4D".to_vec(), + ]; + for data in deprecated { + assert!(from_slice(&data).is_err(), "{data:02x?}"); + } +} + #[test] fn test_decode_array() { let tests = vec![( diff --git a/tests/it/json_text.rs b/tests/it/json_text.rs index 203f714..d286888 100644 --- a/tests/it/json_text.rs +++ b/tests/it/json_text.rs @@ -72,21 +72,30 @@ fn text() -> impl Strategy { "[a-z \"\\\\/\u{0}-\u{1f}\u{7f}é中😀]{0,6}" } +/// The high word of `10^76 - 1`. +const DECIMAL256_HIGH: i128 = 29_387_358_770_557_187_699_218_413_430_556_141_945; + fn number() -> impl Strategy { prop_oneof![ any::().prop_map(Number::Int64), any::().prop_map(Number::UInt64), // Includes NaN and infinities, which serde_json writes as `null`. any::().prop_map(Number::Float64), - (any::(), any::()) + // Decimals within their precision; decoding rejects the others, which + // the byte mutations cover. + ( + 0..=18u8, + -999_999_999_999_999_999..=999_999_999_999_999_999i64 + ) .prop_map(|(scale, value)| Number::Decimal64(Decimal64 { scale, value })), - (any::(), any::()) + (0..=38u8, -(10i128.pow(38) - 1)..=10i128.pow(38) - 1) .prop_map(|(scale, value)| Number::Decimal128(Decimal128 { scale, value })), - // Large scales overflow the decimal formatting buffer, which is an error. - (any::(), any::(), any::()).prop_map(|(scale, hi, lo)| { + // A high word below `DECIMAL256_HIGH` in magnitude keeps the value + // below `DECIMAL256_HIGH * 2^128 <= 10^76 - 1` for any low word. + (0..=76u8, any::(), any::()).prop_map(|(scale, hi, lo)| { Number::Decimal256(Decimal256 { scale, - value: ethnum::I256::from_words(hi, lo), + value: ethnum::I256::from_words(hi % DECIMAL256_HIGH, lo), }) }), ] diff --git a/tests/it/number.rs b/tests/it/number.rs index 86410f8..d3f92cc 100644 --- a/tests/it/number.rs +++ b/tests/it/number.rs @@ -2310,9 +2310,9 @@ fn test_parse_float() { } /// With `arbitrary_precision`, decimals serialize as their exact text, which -/// may be at most 128 bytes long. Longer texts need a scale above 125, which -/// the text parser never produces; serializing them fails, and `to_string` -/// falls back to the raw bytes. +/// may be at most 128 bytes long. Longer texts need a scale above 125, so +/// serializing them fails. No valid JSONB decimal has such a scale: decoding +/// rejects it, and `to_string` falls back to the raw bytes. #[cfg(feature = "arbitrary_precision")] #[test] fn decimal_text_is_limited_to_128_bytes() { @@ -2321,16 +2321,15 @@ fn decimal_text_is_limited_to_128_bytes() { let number = Number::Decimal64(Decimal64 { scale, value: 1 }); let expect = format!("0.{}1", "0".repeat(scale as usize - 1)); assert_eq!(expect.len() <= 128, fits); - let raw = Value::Number(number.clone()).to_vec().unwrap(); - let raw = RawJsonb::new(&raw); if fits { assert_eq!(serde_json::to_string(&number).unwrap(), expect); - assert_eq!(raw.to_string(), expect); } else { assert!(serde_json::to_string(&number).is_err(), "scale {scale}"); - assert_ne!(raw.to_string(), expect); - assert_eq!(raw.to_pretty_string(), "null"); } + let raw = Value::Number(number).to_vec().unwrap(); + let raw = RawJsonb::new(&raw); + assert_ne!(raw.to_string(), expect); + assert_eq!(raw.to_pretty_string(), "null"); } } @@ -2702,7 +2701,8 @@ fn number_order_is_exact_and_transitive() { // Every triple of these numbers is ordered transitively, and the // comparable encoding, which converts numbers to `f64`, never reverses - // their order. + // their order. Decimals beyond their precision, such as `i64::MIN`, are + // not valid JSONB and have no comparable encoding. let mut numbers: Vec = cases .iter() .flat_map(|(l, r, _)| [l.clone(), r.clone()]) @@ -2716,17 +2716,20 @@ fn number_order_is_exact_and_transitive() { ]); let comparable = |n: &Number| { let encoded = Value::Number(n.clone()).to_vec().unwrap(); - RawJsonb::new(&encoded).convert_to_comparable() + jsonb::from_slice(&encoded).ok()?; + Some(RawJsonb::new(&encoded).convert_to_comparable()) }; for x in &numbers { for y in &numbers { let order = x.cmp(y); assert_eq!(order, y.cmp(x).reverse(), "{x:?} vs {y:?}"); - let encoded = comparable(x).cmp(&comparable(y)); - assert!( - encoded == order || encoded == Equal, - "comparable encoding of {x:?} vs {y:?} is {encoded:?}, but they compare {order:?}" - ); + if let (Some(cx), Some(cy)) = (comparable(x), comparable(y)) { + let encoded = cx.cmp(&cy); + assert!( + encoded == order || encoded == Equal, + "comparable encoding of {x:?} vs {y:?} is {encoded:?}, but they compare {order:?}" + ); + } for z in &numbers { if order != Greater && y.cmp(z) != Greater { assert_ne!(x.cmp(z), Greater, "{x:?} <= {y:?} <= {z:?}"); diff --git a/tests/it/parser.rs b/tests/it/parser.rs index 1aad7bf..f2a90d8 100644 --- a/tests/it/parser.rs +++ b/tests/it/parser.rs @@ -618,3 +618,25 @@ fn test_parse_negative_hex_at_i64_min() { "-9223372036854775809" ); } + +/// `-0x4b3b4ca85a86c47a098a224000000000` is `-10^38`, one digit beyond a +/// `Decimal128`; it used to become one, which decoding rejects. +#[cfg(feature = "arbitrary_precision")] +#[test] +fn test_parse_negative_hex_beyond_decimal128() { + let value = parse_value(b"-0x4b3b4ca85a86c47a098a223fffffffff").unwrap(); + assert!( + matches!(value, Value::Number(Number::Decimal128(ref d)) if d.value == 1 - 10_i128.pow(38)), + "{value:?}" + ); + let value = parse_value(b"-0x4b3b4ca85a86c47a098a224000000000").unwrap(); + assert!( + matches!(value, Value::Number(Number::Decimal256(_))), + "{value:?}" + ); + let data = value.to_vec().unwrap(); + assert_eq!( + jsonb::from_slice(&data).unwrap().to_string(), + format!("-1{}", "0".repeat(38)) + ); +}