From e4fd3eb8510ec11c6a0d1f0256679164f5b33871 Mon Sep 17 00:00:00 2001 From: Zachary Dremann Date: Mon, 5 Oct 2026 13:07:37 +0000 Subject: [PATCH 1/4] CI: key the build cache on the native target features CI builds with `-C target-cpu=native`, and without `--target`, Cargo applies `RUSTFLAGS` to proc macros too. The cached `fearless_simd_macros` proc macro was built on a runner with more CPU features than later runners which restored it from the cache, so rustc crashed with SIGILL when it loaded the macro. Cached dependencies could fail the same way at test time. Add a hash of the target features `-C target-cpu=native` enables (from `rustc --print cfg`) to the cache key, so a cache is only restored on runners with the same target features. Unlike hashing all of /proc/cpuinfo's flags, runners with different CPU models but the same target features still share a cache. --- .github/workflows/test.yml | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d6a0b520..885209a2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,8 +35,17 @@ jobs: toolchain: ${{ matrix.rust }} components: rustfmt, clippy + # Builds use `-C target-cpu=native`, so cached artifacts (including proc macros, which run in + # the compiler) only work on runners with the same target features + - name: Identify native target features + id: native + run: | + echo "features=$(rustc --print cfg -C target-cpu=native | grep '^target_feature=' | sha256sum | cut -c1-12)" >> "$GITHUB_OUTPUT" + - name: Caching uses: Swatinem/rust-cache@v2 + with: + key: native-${{ steps.native.outputs.features }} - name: Check # clippy will also do a build check @@ -85,8 +94,17 @@ jobs: repository: "RoaringBitmap/real-roaring-datasets" path: "benchmarks/real-roaring-datasets" + # Builds use `-C target-cpu=native`, so cached artifacts (including proc macros, which run in + # the compiler) only work on runners with the same target features + - name: Identify native target features + id: native + run: | + echo "features=$(rustc --print cfg -C target-cpu=native | grep '^target_feature=' | sha256sum | cut -c1-12)" >> "$GITHUB_OUTPUT" + - name: Caching uses: Swatinem/rust-cache@v2 + with: + key: native-${{ steps.native.outputs.features }} - name: Initialize rust toolchain uses: dtolnay/rust-toolchain@master @@ -126,8 +144,17 @@ jobs: toolchain: nightly components: miri + # Builds use `-C target-cpu=native`, so cached artifacts (including proc macros, which run in + # the compiler) only work on runners with the same target features + - name: Identify native target features + id: native + run: | + echo "features=$(rustc --print cfg -C target-cpu=native | grep '^target_feature=' | sha256sum | cut -c1-12)" >> "$GITHUB_OUTPUT" + - name: Caching uses: Swatinem/rust-cache@v2 + with: + key: native-${{ steps.native.outputs.features }} - name: Setup miri run: cargo miri setup @@ -150,8 +177,17 @@ jobs: with: toolchain: nightly + # Builds use `-C target-cpu=native`, so cached artifacts (including proc macros, which run in + # the compiler) only work on runners with the same target features + - name: Identify native target features + id: native + run: | + echo "features=$(rustc --print cfg -C target-cpu=native | grep '^target_feature=' | sha256sum | cut -c1-12)" >> "$GITHUB_OUTPUT" + - name: Caching uses: Swatinem/rust-cache@v2 + with: + key: native-${{ steps.native.outputs.features }} - name: Install cargo fuzz run: cargo install cargo-fuzz From aed44c225256c1c911cac4f726a93a61be5bf043 Mon Sep 17 00:00:00 2001 From: Zachary Dremann Date: Sat, 3 Oct 2026 05:35:01 +0000 Subject: [PATCH 2/4] Port the simd feature from nightly core::simd to fearless_simd The `simd` feature now builds on stable Rust. The vectorized array store operations (or, and, xor, sub) are ported to fearless_simd with the same algorithms: fearless_simd needs either `std` or `libm`, so `libm` is enabled for no_std builds; with `std` enabled, fearless_simd uses std instead. Adds tests comparing each vectorized op against its scalar version at the baseline and detected SIMD levels. CI now checks and tests `simd` on stable, including no_std. --- .github/workflows/test.yml | 13 +- Cargo.toml | 2 + README.md | 7 +- fuzz/Cargo.lock | 47 +- roaring/Cargo.toml | 7 +- .../src/bitmap/store/array_store/vector.rs | 445 ++++++++++-------- .../src/bitmap/store/array_store/visitor.rs | 13 +- roaring/src/lib.rs | 1 - 8 files changed, 315 insertions(+), 220 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 885209a2..7cb9e6fa 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -58,8 +58,9 @@ jobs: cargo clippy -p roaring --all-targets --features serde -- -Dclippy::todo -D warnings - name: Check SIMD - if: matrix.rust == 'nightly' - run: cargo clippy -p roaring --all-targets --all-features -- -D warnings + run: | + cargo clippy -p roaring --all-targets --all-features -- -D warnings + cargo clippy -p roaring --all-targets --no-default-features --features simd -- -D warnings - name: Check formatting run: cargo fmt --all -- --check @@ -78,7 +79,7 @@ jobs: - default - no-std include: - - rust: nightly + - rust: stable features: simd env: RUSTFLAGS: "-C target-cpu=native -C opt-level=3" @@ -124,8 +125,10 @@ jobs: run: cargo test -p roaring --no-default-features - name: SIMD test - if: matrix.rust == 'nightly' && matrix.features == 'simd' - run: cargo +nightly test -p roaring --features simd + if: matrix.features == 'simd' + run: | + cargo test -p roaring --features simd + cargo test -p roaring --no-default-features --features simd miri: runs-on: ubuntu-latest diff --git a/Cargo.toml b/Cargo.toml index ff3f9007..8ff74a02 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,8 @@ roaring = { path = "roaring" } bytemuck = "1.25.2" byteorder = "1.5.0" criterion = "0.5" +fearless_simd = { version = "1.0.0", default-features = false } +fearless_simd_macros = "0.1.0" git2 = { version = "0.21", default-features = false } indicatif = "0.17" itertools = "0.15" diff --git a/README.md b/README.md index 132733ac..29794096 100644 --- a/README.md +++ b/README.md @@ -70,5 +70,8 @@ https://github.com/RoaringBitmap/roaring-rs/actions/workflows/test.yml/badge.svg ## Experimental features -The `simd` feature is in active development. It has not been tested. If you would like to build with `simd` note that -`std::simd` is only available in Rust nightly. +The `simd` feature is in active development. It is implemented with the [fearless_simd] crate and works on stable +Rust. When the `std` feature is enabled, the best available SIMD instruction set is detected at runtime on x86 and +x86-64; otherwise, it is determined by the target features enabled at compile time (e.g. with `-C target-cpu=native`). + +[fearless_simd]: https://docs.rs/fearless_simd diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index 8545b3f4..c0aef096 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -19,9 +19,9 @@ checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" [[package]] name = "bytemuck" -version = "1.23.0" +version = "1.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9134a6ef01ce4b366b50689c94f82c14bc72bc5d0386829828a2e2752ef7958c" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" [[package]] name = "byteorder" @@ -72,7 +72,27 @@ checksum = "30542c1ad912e0e3d22a1935c290e12e8a29d704a420177a31faad4a601a0800" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.101", +] + +[[package]] +name = "fearless_simd" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3772b63c40606beea8fe1f7b06a60de27ceb9bfc2d9dad3427368ca41dc7e62" +dependencies = [ + "libm", +] + +[[package]] +name = "fearless_simd_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e91d94c6885390b81782d7ee60fd6583b33907333a8b340de25ccf468353315" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", ] [[package]] @@ -113,6 +133,12 @@ dependencies = [ "cc", ] +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + [[package]] name = "proc-macro2" version = "1.0.95" @@ -139,10 +165,12 @@ checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" [[package]] name = "roaring" -version = "0.11.4" +version = "0.11.5" dependencies = [ "bytemuck", "byteorder", + "fearless_simd", + "fearless_simd_macros", ] [[package]] @@ -171,6 +199,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "unicode-ident" version = "1.0.18" diff --git a/roaring/Cargo.toml b/roaring/Cargo.toml index b3c50c01..826194ef 100644 --- a/roaring/Cargo.toml +++ b/roaring/Cargo.toml @@ -20,12 +20,15 @@ license = "MIT OR Apache-2.0" bytemuck = { workspace = true, optional = true } byteorder = { workspace = true, optional = true } serde = { workspace = true, optional = true } +# `libm` is needed for no_std builds; with `std` enabled, fearless_simd uses std instead +fearless_simd = { workspace = true, optional = true, features = ["libm"] } +fearless_simd_macros = { workspace = true, optional = true } [features] default = ["std"] serde = ["dep:serde", "std"] -simd = [] -std = ["dep:bytemuck", "dep:byteorder"] +simd = ["dep:fearless_simd", "dep:fearless_simd_macros"] +std = ["dep:bytemuck", "dep:byteorder", "fearless_simd?/std"] [dev-dependencies] proptest = { workspace = true } diff --git a/roaring/src/bitmap/store/array_store/vector.rs b/roaring/src/bitmap/store/array_store/vector.rs index 6fc14a9a..5710034b 100644 --- a/roaring/src/bitmap/store/array_store/vector.rs +++ b/roaring/src/bitmap/store/array_store/vector.rs @@ -4,18 +4,47 @@ //! Prior work: Schlegel et al., Fast Sorted-Set Intersection using SIMD Instructions //! //! Rust port notes: -//! The x86 PCMPESTRM instruction has been replaced with a simple vector or-shift -//! While several more instructions, this is what is available through LLVM intrinsics -//! and is portable. +//! The x86 PCMPESTRM instruction has been replaced with a portable all-pairs comparison +//! (see `matrix_cmp_u16`), which takes more instructions but is available on every SIMD level. +//! +//! The public functions select a SIMD level at runtime with `fearless_simd::dispatch!`, and call +//! implementations generic over the SIMD level. Those are annotated with `#[simd]`, so that they +//! are compiled with the matching target features enabled. +//! +//! The small helpers they call are `#[inline(always)]` rather than `#[simd]`: they get the target +//! features by being inlined, and `#[simd]` does not force inlining, which can leave them out of +//! line in the hot loops. #![cfg(feature = "simd")] use super::scalar; -use core::simd::cmp::{SimdPartialEq, SimdPartialOrd}; -use core::simd::{mask16x8, u16x8, u8x16, Mask, Select as _, Simd, SimdElement, ToBytes}; +use crate::bitmap::store::array_store::visitor::BinaryOperationVisitor; +use fearless_simd::prelude::*; +use fearless_simd::{dispatch, mask16x8, u16x8, Level}; +use fearless_simd_macros::simd; + +/// The number of lanes in a `u16x8` +const LANES: usize = 8; + +/// The SIMD level to use. +/// +/// With `std`, this is detected at runtime on x86/x86-64 (and cached by `fearless_simd`). +/// Otherwise, it is determined by the target features enabled at compile time. +#[inline] +fn level() -> Level { + #[cfg(feature = "std")] + return Level::new(); + #[cfg(not(feature = "std"))] + return Level::baseline(); +} -// a one-pass SSE union algorithm +// a one-pass union algorithm pub fn or(lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) { + dispatch!(level(), simd => or_impl(simd, lhs, rhs, visitor)) +} + +#[simd] +fn or_impl(simd: S, lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) { // De-duplicates `slice` in place // Returns the end index of the deduplicated slice. // elements after the return value are not guaranteed to be unique or in order @@ -31,11 +60,14 @@ pub fn or(lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) { pos } - #[inline] - fn handle_vector(old: u16x8, new: u16x8, f: impl FnOnce(u16x8, u8)) { - let tmp: u16x8 = Shr1::concat_swizzle(new, old); + // returns `new`, with a mask of its values which were not already written, + // assuming that the previously written vector was `old` + #[inline(always)] + fn handle_vector(old: u16x8, new: u16x8) -> (u16x8, u8) { + // `[old[7], new[0], ..., new[6]]` + let tmp: u16x8 = old.slide::<7>(new); let mask = 255 - tmp.simd_eq(new).to_bitmask() as u8; - f(new, mask); + (new, mask) } if (lhs.len() < 8) || (rhs.len() < 8) { @@ -46,21 +78,22 @@ pub fn or(lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) { let len1: usize = lhs.len() / 8; let len2: usize = rhs.len() / 8; - let v_a: u16x8 = load(lhs); - let v_b: u16x8 = load(rhs); + let v_a: u16x8 = load(simd, lhs); + let v_b: u16x8 = load(simd, rhs); let [mut v_min, mut v_max] = simd_merge_u16(v_a, v_b); let mut i = 1; let mut j = 1; - handle_vector(Simd::splat(u16::MAX), v_min, |v, m| visitor.visit_vector(v, m)); - let mut v_prev: u16x8 = v_min; + let (v, m) = handle_vector(u16x8::splat(simd, u16::MAX), v_min); + visitor.visit_vector(v, m); + let mut v_prev: u16x8 = v_min; if (i < len1) && (j < len2) { - let mut v: u16x8; + let mut v: u16x8; let mut cur_a: u16 = lhs[8 * i]; let mut cur_b: u16 = rhs[8 * j]; loop { if cur_a <= cur_b { - v = load(&lhs[8 * i..]); + v = load(simd, &lhs[8 * i..]); i += 1; if i < len1 { cur_a = lhs[8 * i]; @@ -68,7 +101,7 @@ pub fn or(lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) { break; } } else { - v = load(&rhs[8 * j..]); + v = load(simd, &rhs[8 * j..]); j += 1; if j < len2 { cur_b = rhs[8 * j]; @@ -77,11 +110,13 @@ pub fn or(lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) { } } [v_min, v_max] = simd_merge_u16(v, v_max); - handle_vector(v_prev, v_min, |v, m| visitor.visit_vector(v, m)); + let (v, m) = handle_vector(v_prev, v_min); + visitor.visit_vector(v, m); v_prev = v_min; } [v_min, v_max] = simd_merge_u16(v, v_max); - handle_vector(v_prev, v_min, |v, m| visitor.visit_vector(v, m)); + let (v, m) = handle_vector(v_prev, v_min); + visitor.visit_vector(v, m); v_prev = v_min; } @@ -92,11 +127,9 @@ pub fn or(lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) { // // copy the small end on a tmp buffer let mut buffer: [u16; 16] = [0; 16]; - let mut rem = 0; - handle_vector(v_prev, v_max, |v, m| { - store(swizzle_to_front(v, m), buffer.as_mut_slice()); - rem = m.count_ones() as usize; - }); + let (v, m) = handle_vector(v_prev, v_max); + store(swizzle_to_front(v, m), buffer.as_mut_slice()); + let mut rem = m.count_ones() as usize; let (tail_a, tail_b, tail_len) = if i == len1 { (&lhs[8 * i..], &rhs[8 * j..], lhs.len() - 8 * len1) @@ -117,33 +150,38 @@ pub fn or(lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) { } pub fn and(lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) { - let st_a = (lhs.len() / u16x8::LEN) * u16x8::LEN; - let st_b = (rhs.len() / u16x8::LEN) * u16x8::LEN; + dispatch!(level(), simd => and_impl(simd, lhs, rhs, visitor)) +} + +#[simd] +fn and_impl(simd: S, lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) { + let st_a = (lhs.len() / LANES) * LANES; + let st_b = (rhs.len() / LANES) * LANES; let mut i: usize = 0; let mut j: usize = 0; if (i < st_a) && (j < st_b) { - let mut v_a: u16x8 = load(&lhs[i..]); - let mut v_b: u16x8 = load(&rhs[j..]); + let mut v_a: u16x8 = load(simd, &lhs[i..]); + let mut v_b: u16x8 = load(simd, &rhs[j..]); loop { let mask = matrix_cmp_u16(v_a, v_b).to_bitmask() as u8; visitor.visit_vector(v_a, mask); - let a_max: u16 = lhs[i + u16x8::LEN - 1]; - let b_max: u16 = rhs[j + u16x8::LEN - 1]; + let a_max: u16 = lhs[i + LANES - 1]; + let b_max: u16 = rhs[j + LANES - 1]; if a_max <= b_max { - i += u16x8::LEN; + i += LANES; if i == st_a { break; } - v_a = load(&lhs[i..]); + v_a = load(simd, &lhs[i..]); } if b_max <= a_max { - j += u16x8::LEN; + j += LANES; if j == st_b { break; } - v_b = load(&rhs[j..]); + v_b = load(simd, &rhs[j..]); } } } @@ -152,8 +190,13 @@ pub fn and(lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) scalar::and(&lhs[i..], &rhs[j..], visitor); } -// a one-pass SSE xor algorithm +// a one-pass xor algorithm pub fn xor(lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) { + dispatch!(level(), simd => xor_impl(simd, lhs, rhs, visitor)) +} + +#[simd] +fn xor_impl(simd: S, lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) { /// De-duplicates `slice` in place, removing _both_ duplicates /// Returns the end index of the xor-ed slice. /// elements after the return value are not guaranteed to be unique or in order @@ -171,17 +214,19 @@ pub fn xor(lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) pos } - // write vector new, while omitting repeated values assuming that previously - // written vector was "old" - #[inline] - fn handle_vector(old: u16x8, new: u16x8, f: impl FnOnce(u16x8, u8)) { - let tmp1: u16x8 = Shr2::concat_swizzle(new, old); - let tmp2: u16x8 = Shr1::concat_swizzle(new, old); - let eq_l: mask16x8 = tmp2.simd_eq(tmp1); - let eq_r: mask16x8 = tmp2.simd_eq(new); - let eq_l_or_r: mask16x8 = eq_l | eq_r; + // returns the vector to write, and a mask of the values to write from it, + // omitting repeated values assuming that previously written vector was "old" + #[inline(always)] + fn handle_vector(old: u16x8, new: u16x8) -> (u16x8, u8) { + // `[old[6], old[7], new[0], ..., new[5]]` + let tmp1: u16x8 = old.slide::<6>(new); + // `[old[7], new[0], ..., new[6]]` + let tmp2: u16x8 = old.slide::<7>(new); + let eq_l: mask16x8 = tmp2.simd_eq(tmp1); + let eq_r: mask16x8 = tmp2.simd_eq(new); + let eq_l_or_r: mask16x8 = eq_l | eq_r; let mask: u8 = eq_l_or_r.to_bitmask() as u8; - f(tmp2, 255 - mask); + (tmp2, 255 - mask) } if (lhs.len() < 8) || (rhs.len() < 8) { @@ -192,21 +237,22 @@ pub fn xor(lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) let len1: usize = lhs.len() / 8; let len2: usize = rhs.len() / 8; - let v_a: u16x8 = load(lhs); - let v_b: u16x8 = load(rhs); + let v_a: u16x8 = load(simd, lhs); + let v_b: u16x8 = load(simd, rhs); let [mut v_min, mut v_max] = simd_merge_u16(v_a, v_b); let mut i = 1; let mut j = 1; - handle_vector(Simd::splat(u16::MAX), v_min, |v, m| visitor.visit_vector(v, m)); - let mut v_prev: u16x8 = v_min; + let (v, m) = handle_vector(u16x8::splat(simd, u16::MAX), v_min); + visitor.visit_vector(v, m); + let mut v_prev: u16x8 = v_min; if (i < len1) && (j < len2) { - let mut v: u16x8; + let mut v: u16x8; let mut cur_a: u16 = lhs[8 * i]; let mut cur_b: u16 = rhs[8 * j]; loop { if cur_a <= cur_b { - v = load(&lhs[8 * i..]); + v = load(simd, &lhs[8 * i..]); i += 1; if i < len1 { cur_a = lhs[8 * i]; @@ -214,7 +260,7 @@ pub fn xor(lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) break; } } else { - v = load(&rhs[8 * j..]); + v = load(simd, &rhs[8 * j..]); j += 1; if j < len2 { cur_b = rhs[8 * j]; @@ -223,11 +269,13 @@ pub fn xor(lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) } } [v_min, v_max] = simd_merge_u16(v, v_max); - handle_vector(v_prev, v_min, |v, m| visitor.visit_vector(v, m)); + let (v, m) = handle_vector(v_prev, v_min); + visitor.visit_vector(v, m); v_prev = v_min; } [v_min, v_max] = simd_merge_u16(v, v_max); - handle_vector(v_prev, v_min, |v, m| visitor.visit_vector(v, m)); + let (v, m) = handle_vector(v_prev, v_min); + visitor.visit_vector(v, m); v_prev = v_min; } @@ -239,13 +287,14 @@ pub fn xor(lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) // last value of vecMax, let mut buffer: [u16; 17] = [0; 17]; // remaining size - let mut rem = 0; - handle_vector(v_prev, v_max, |v, m| { - store(swizzle_to_front(v, m), buffer.as_mut_slice()); - rem = m.count_ones() as usize; - }); - - let arr_max = v_max.as_array(); + let (v, m) = handle_vector(v_prev, v_max); + store(swizzle_to_front(v, m), buffer.as_mut_slice()); + let mut rem = m.count_ones() as usize; + + // Store `v_max` rather than reading its lanes with indexing, which makes LLVM split + // `v_max` into pieces in the merge loop above + let mut arr_max = [0u16; LANES]; + v_max.store_array(&mut arr_max); let vec7 = arr_max[7]; let vec6 = arr_max[6]; if vec6 != vec7 { @@ -272,6 +321,11 @@ pub fn xor(lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) } pub fn sub(lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) { + dispatch!(level(), simd => sub_impl(simd, lhs, rhs, visitor)) +} + +#[simd] +fn sub_impl(simd: S, lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) { // we handle the degenerate cases if lhs.is_empty() { return; @@ -280,14 +334,14 @@ pub fn sub(lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) return; } - let st_a = (lhs.len() / u16x8::LEN) * u16x8::LEN; - let st_b = (rhs.len() / u16x8::LEN) * u16x8::LEN; + let st_a = (lhs.len() / LANES) * LANES; + let st_b = (rhs.len() / LANES) * LANES; let mut i = 0; let mut j = 0; if (i < st_a) && (j < st_b) { - let mut v_a: u16x8 = load(&lhs[i..]); - let mut v_b: u16x8 = load(&rhs[j..]); + let mut v_a: u16x8 = load(simd, &lhs[i..]); + let mut v_b: u16x8 = load(simd, &rhs[j..]); // we have a running mask which indicates which values from a have been // spotted in b, these don't get written out. let mut runningmask_a_found_in_b: u8 = 0; @@ -297,28 +351,28 @@ pub fn sub(lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) let a_found_in_b: u8 = matrix_cmp_u16(v_a, v_b).to_bitmask() as u8; runningmask_a_found_in_b |= a_found_in_b; // we always compare the last values of A and B - let a_max: u16 = lhs[i + u16x8::LEN - 1]; - let b_max: u16 = rhs[j + u16x8::LEN - 1]; + let a_max: u16 = lhs[i + LANES - 1]; + let b_max: u16 = rhs[j + LANES - 1]; if a_max <= b_max { // Ok. In this code path, we are ready to write our v_a // because there is no need to read more from B, they will // all be large values. let bitmask_belongs_to_difference = runningmask_a_found_in_b ^ 0xFF; visitor.visit_vector(v_a, bitmask_belongs_to_difference); - i += u16x8::LEN; + i += LANES; if i == st_a { break; } runningmask_a_found_in_b = 0; - v_a = load(&lhs[i..]); + v_a = load(simd, &lhs[i..]); } if b_max <= a_max { // in this code path, the current v_b has become useless - j += u16x8::LEN; + j += LANES; if j == st_b { break; } - v_b = load(&rhs[j..]); + v_b = load(simd, &rhs[j..]); } } @@ -336,16 +390,18 @@ pub fn sub(lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) // Ensure the buffer is filled with a value we should remove: we do not want to // end up trying to remove zero values which aren't actually in rhs buffer[remaining_rhs.len()..].fill(remaining_rhs[0]); - v_b = Simd::from_array(buffer); + v_b = load(simd, &buffer); let a_found_in_b: u8 = matrix_cmp_u16(v_a, v_b).to_bitmask() as u8; runningmask_a_found_in_b |= a_found_in_b; - let [.., max_va] = *v_a.as_array(); + // Read from `lhs` (which `v_a` was loaded from): reading a lane of `v_a` makes LLVM + // split `v_a` into pieces in the loop above + let max_va = lhs[i + LANES - 1]; let used_rhs = remaining_rhs.partition_point(|&b| b <= max_va); j += used_rhs; } let bitmask_belongs_to_difference: u8 = runningmask_a_found_in_b ^ 0xFF; visitor.visit_vector(v_a, bitmask_belongs_to_difference); - i += u16x8::LEN; + i += LANES; } } @@ -353,67 +409,22 @@ pub fn sub(lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) scalar::sub(&lhs[i..], &rhs[j..], visitor); } -/// compute the min for each lane in `a` and `b` -#[inline] -fn lanes_min_u16( - lhs: Simd, - rhs: Simd, -) -> Simd { - lhs.simd_le(rhs).select(lhs, rhs) -} - -/// compute the max for each lane in `a` and `b` -#[inline] -fn lanes_max_u16( - lhs: Simd, - rhs: Simd, -) -> Simd { - lhs.simd_gt(rhs).select(lhs, rhs) -} - -#[inline] -pub fn load(src: &[U]) -> Simd -where - U: SimdElement + PartialOrd, -{ - debug_assert!(src.len() >= LANES); - unsafe { load_unchecked(src) } -} - -/// write `v` to slice `out` without checking bounds +/// load the first `LANES` values of `src` /// -/// ### Safety -/// - The caller must ensure `LANES` does not exceed the allocation for `out` -#[inline] -pub unsafe fn load_unchecked(src: &[U]) -> Simd -where - U: SimdElement + PartialOrd, -{ - unsafe { core::ptr::read_unaligned(src as *const _ as *const Simd) } +/// ### Panics +/// - If `src` is shorter than `LANES` +#[inline(always)] +fn load(simd: S, src: &[u16]) -> u16x8 { + u16x8::from_slice(simd, &src[..LANES]) } -/// write `v` to slice `out` -#[inline] -pub fn store(v: Simd, out: &mut [U]) -where - U: SimdElement + PartialOrd, -{ - debug_assert!(out.len() >= LANES); - unsafe { - store_unchecked(v, out); - } -} - -/// write `v` to slice `out` without checking bounds +/// write `v` to the first `LANES` values of `out` /// -/// ### Safety -/// - The caller must ensure `LANES` does not exceed the allocation for `out` -#[inline] -unsafe fn store_unchecked(v: Simd, out: &mut [U]) -where - U: SimdElement + PartialOrd, -{ - unsafe { core::ptr::write_unaligned(out as *mut _ as *mut Simd, v) } +/// ### Panics +/// - If `out` is shorter than `LANES` +#[inline(always)] +fn store(v: u16x8, out: &mut [u16]) { + v.store_slice(&mut out[..LANES]) } /// Compare all lanes in `a` to all lanes in `b` @@ -422,15 +433,13 @@ where /// /// ### Example /// ```ignore -/// let a = Simd::from_array([1, 2, 3, 4, 32, 33, 34, 35]); -/// let b = Simd::from_array([2, 4, 6, 8, 10, 12, 14, 16]); +/// let a = u16x8::from_slice(simd, &[1, 2, 3, 4, 32, 33, 34, 35]); +/// let b = u16x8::from_slice(simd, &[2, 4, 6, 8, 10, 12, 14, 16]); /// let result = matrix_cmp_u16(a, b); -/// assert_eq!(result, Mask::from_array([false, true, false, true, false, false, false, false])); +/// assert_eq!(result.to_bitmask(), 0b0000_1010); /// ``` -#[inline] -// It would be nice to implement this for all supported lane counts -// However, we currently only support u16x8 so it's not really necessary -fn matrix_cmp_u16(a: Simd, b: Simd) -> Mask { +#[inline(always)] +fn matrix_cmp_u16(a: u16x8, b: u16x8) -> mask16x8 { a.simd_eq(b) | a.simd_eq(b.rotate_elements_left::<1>()) | a.simd_eq(b.rotate_elements_left::<2>()) @@ -441,39 +450,22 @@ fn matrix_cmp_u16(a: Simd, b: Simd) -> Mask { | a.simd_eq(b.rotate_elements_left::<7>()) } -use crate::bitmap::store::array_store::visitor::BinaryOperationVisitor; -use core::simd::Swizzle; - -/// Append to vectors to an imaginary 16 lane vector, shift the lanes right by 1, then -/// truncate to the low order 8 lanes -pub struct Shr1; -impl Swizzle<8> for Shr1 { - const INDEX: [usize; 8] = [15, 0, 1, 2, 3, 4, 5, 6]; -} - -/// Append to vectors to an imaginary 16 lane vector, shift the lanes right by 2, then -/// truncate to the low order 8 lanes -pub struct Shr2; -impl Swizzle<8> for Shr2 { - const INDEX: [usize; 8] = [14, 15, 0, 1, 2, 3, 4, 5]; -} - /// Assuming that a and b are sorted, returns an array of the sorted output. /// Developed originally for merge sort using SIMD instructions. /// Standard merge. See, e.g., Inoue and Taura, SIMD- and Cache-Friendly /// Algorithm for Sorting an Array of Structures -#[inline] -fn simd_merge_u16(a: Simd, b: Simd) -> [Simd; 2] { - let mut tmp: Simd = lanes_min_u16(a, b); - let mut max: Simd = lanes_max_u16(a, b); +#[inline(always)] +fn simd_merge_u16(a: u16x8, b: u16x8) -> [u16x8; 2] { + let mut tmp: u16x8 = a.min(b); + let mut max: u16x8 = a.max(b); tmp = tmp.rotate_elements_left::<1>(); - let mut min: Simd = lanes_min_u16(tmp, max); + let mut min: u16x8 = tmp.min(max); for _ in 0..6 { - max = lanes_max_u16(tmp, max); + max = tmp.max(max); tmp = min.rotate_elements_left::<1>(); - min = lanes_min_u16(tmp, max); + min = tmp.min(max); } - max = lanes_max_u16(tmp, max); + max = tmp.max(max); min = min.rotate_elements_left::<1>(); [min, max] } @@ -482,20 +474,12 @@ fn simd_merge_u16(a: Simd, b: Simd) -> [Simd; 2] { /// set to the front of the return vector, preserving their order. /// /// The values in the return vector after index bitmask.count_ones() is unspecified. -// Dynamic swizzle is only available for `u8`s. -// -// So we need to convert the `u16x8` to `u8x16`, and then swizzle it two lanes at a time. +// Dynamic swizzles operate on bytes, so the swizzle table moves the `u16` lanes two bytes at a time. // // e.g. if `bitmask` is `0b0101`, then swizzle the first two bytes (the first u16 lane) to the // first two positions, and the 5th and 6th bytes (the third u16 lane) to the next two positions. -// -// Note however: -// https://github.com/rust-lang/rust/blob/34097a38afc9efdedf776d3f1c84a190ff334886/library/portable-simd/crates/core_simd/src/swizzle_dyn.rs#L12-L15 -// > Note that the current implementation is selected during build-time -// > of the standard library, so `cargo build -Zbuild-std` may be necessary -// > to unlock better performance, especially for larger vectors. -// > A planned compiler improvement will enable using `#[target_feature]` instead. -pub fn swizzle_to_front(val: u16x8, bitmask: u8) -> u16x8 { +#[inline(always)] +pub fn swizzle_to_front(val: u16x8, bitmask: u8) -> u16x8 { static SWIZZLE_TABLE: [[u8; 16]; 256] = { let mut table = [[0; 16]; 256]; let mut n = 0usize; @@ -514,37 +498,94 @@ pub fn swizzle_to_front(val: u16x8, bitmask: u8) -> u16x8 { table }; - // Our swizzle table retains the order of the bytes in the 16 bit lanes, we can - // stick with native byte order as long as we convert back with native endianness too. - let val_convert: u8x16 = val.to_ne_bytes(); - let swizzle_idxs = u8x16::from_array(SWIZZLE_TABLE[bitmask as usize]); - - // Because the default `x86_64` target does not enable ssse3 (and without -Zbuild-std` - // std will not be compiled with it), use a manual swizzle with intrinsics so we can get - // reasonable performance without requiring the caller to use `-Zbuild-std`. - #[cfg(all(target_arch = "x86_64", any(target_feature = "ssse3", feature = "std")))] - { - let has_ssse3 = { - #[cfg(target_feature = "ssse3")] - { - true + // Our swizzle table retains the order of the bytes in the 16 bit lanes, + // so it works with either native byte order. + val.swizzle_dyn(SWIZZLE_TABLE[bitmask as usize]) +} + +#[cfg(test)] +mod test { + use super::*; + use crate::bitmap::store::array_store::visitor::VecWriter; + use alloc::vec::Vec; + use proptest::prelude::*; + + /// Checks the vectorized op produces the same result as the scalar op + fn check_op( + vector: impl Fn(&[u16], &[u16], &mut VecWriter), + scalar: impl Fn(&[u16], &[u16], &mut VecWriter), + lhs: &[u16], + rhs: &[u16], + ) { + let mut expected = VecWriter::new(lhs.len() + rhs.len()); + scalar(lhs, rhs, &mut expected); + let mut actual = VecWriter::new(lhs.len() + rhs.len()); + vector(lhs, rhs, &mut actual); + assert_eq!(actual.into_inner(), expected.into_inner()); + } + + /// Checks all vectorized ops produce the same results as the scalar ops + fn check_all(lhs: &[u16], rhs: &[u16]) { + check_op(or, scalar::or, lhs, rhs); + check_op(and, scalar::and, lhs, rhs); + check_op(xor, scalar::xor, lhs, rhs); + check_op(sub, scalar::sub, lhs, rhs); + } + + proptest! { + #[test] + fn vector_ops_match_scalar_dense( + lhs in prop::collection::btree_set(0u16..256, 0..200), + rhs in prop::collection::btree_set(0u16..256, 0..200), + ) { + check_all(&Vec::from_iter(lhs), &Vec::from_iter(rhs)); + } + + #[test] + fn vector_ops_match_scalar_sparse( + lhs in prop::collection::btree_set(any::(), 0..200), + rhs in prop::collection::btree_set(any::(), 0..200), + ) { + check_all(&Vec::from_iter(lhs), &Vec::from_iter(rhs)); + } + } + + #[test] + fn swizzle_to_front_keeps_masked_lanes_in_order() { + dispatch!(level(), simd => { + let values: [u16; 8] = [10, 11, 12, 13, 14, 15, 16, 17]; + let v = u16x8::from_slice(simd, &values); + for mask in 0..=255u8 { + let expected: Vec = + (0..8).filter(|i| mask & (1 << i) != 0).map(|i| values[i]).collect(); + let result = swizzle_to_front(v, mask); + assert_eq!(&result.as_slice()[..expected.len()], &expected[..]); } - #[cfg(not(target_feature = "ssse3"))] - { - // From above, `feature = std` must be true here, so we can do runtime detection - std::arch::is_x86_feature_detected!("ssse3") + }); + } + + #[test] + fn vector_ops_match_scalar_at_lane_boundaries() { + // Lengths around multiples of `LANES` exercise the scalar fallback, the vector loops, + // and the partial vector tails; the offsets give disjoint, interleaved, and equal inputs. + let lens = [0, 1, 7, 8, 9, 15, 16, 17, 24, 25]; + for len_l in lens { + for len_r in lens { + for offset in [0, 1, 4, 8, 64] { + let lhs: Vec = (0..len_l).map(|x| 2 * x).collect(); + let rhs: Vec = (0..len_r).map(|x| 2 * x + offset).collect(); + check_all(&lhs, &rhs); + } } - }; - if has_ssse3 { - use core::arch::x86_64::{__m128i, _mm_shuffle_epi8}; - let val_m128 = __m128i::from(val_convert); - let swizzle_m128 = __m128i::from(swizzle_idxs); - // SAFETY: We only are in this block if the target supports `ssse3` - let swizzled_m128 = unsafe { _mm_shuffle_epi8(val_m128, swizzle_m128) }; - return u16x8::from(swizzled_m128); } } - let swizzled: u8x16 = val_convert.swizzle_dyn(swizzle_idxs); - u16x8::from_ne_bytes(swizzled) + #[test] + fn matrix_cmp_finds_lanes_of_a_in_b() { + dispatch!(level(), simd => { + let a = u16x8::from_slice(simd, &[1, 2, 3, 4, 32, 33, 34, 35]); + let b = u16x8::from_slice(simd, &[2, 4, 6, 8, 10, 12, 14, 35]); + assert_eq!(matrix_cmp_u16(a, b).to_bitmask(), 0b1000_1010); + }); + } } diff --git a/roaring/src/bitmap/store/array_store/visitor.rs b/roaring/src/bitmap/store/array_store/visitor.rs index d7293d7b..5a1025eb 100644 --- a/roaring/src/bitmap/store/array_store/visitor.rs +++ b/roaring/src/bitmap/store/array_store/visitor.rs @@ -1,5 +1,9 @@ #[cfg(feature = "simd")] use crate::bitmap::store::array_store::vector::swizzle_to_front; +#[cfg(feature = "simd")] +use fearless_simd::{prelude::*, u16x8}; +#[cfg(feature = "simd")] +use fearless_simd_macros::simd; #[cfg(not(feature = "std"))] use alloc::vec::Vec; @@ -13,7 +17,7 @@ use alloc::vec::Vec; /// computing the cardinality of an operation without materializng a new bitmap. pub trait BinaryOperationVisitor { #[cfg(feature = "simd")] - fn visit_vector(&mut self, value: core::simd::u16x8, mask: u8); + fn visit_vector(&mut self, value: u16x8, mask: u8); fn visit_scalar(&mut self, value: u16); fn visit_slice(&mut self, values: &[u16]); } @@ -40,7 +44,8 @@ impl VecWriter { impl BinaryOperationVisitor for VecWriter { #[cfg(feature = "simd")] - fn visit_vector(&mut self, value: core::simd::u16x8, mask: u8) { + #[simd] + fn visit_vector(&mut self, value: u16x8, mask: u8) { let result = swizzle_to_front(value, mask); // This idiom is better than subslicing result, as it compiles down to an unaligned vector @@ -48,7 +53,7 @@ impl BinaryOperationVisitor for VecWriter { // A more straightforward, but unsafe way would be ptr::write_unaligned and Vec::set_len // Writing a vector at once is why the vectorized algorithms do not operate in place // first write the entire vector - self.vec.extend_from_slice(&result.as_array()[..]); + self.vec.extend_from_slice(result.as_slice()); // next truncate the masked out values self.vec.truncate(self.vec.len() - (result.len() - mask.count_ones() as usize)); } @@ -78,7 +83,7 @@ impl CardinalityCounter { impl BinaryOperationVisitor for CardinalityCounter { #[cfg(feature = "simd")] - fn visit_vector(&mut self, _value: core::simd::u16x8, mask: u8) { + fn visit_vector(&mut self, _value: u16x8, mask: u8) { self.count += mask.count_ones() as usize; } diff --git a/roaring/src/lib.rs b/roaring/src/lib.rs index d66f15f5..a01c421d 100644 --- a/roaring/src/lib.rs +++ b/roaring/src/lib.rs @@ -8,7 +8,6 @@ //! [roaring-paper]: https://arxiv.org/pdf/1402.6407v4 #![cfg_attr(not(feature = "std"), no_std)] -#![cfg_attr(feature = "simd", feature(portable_simd))] #![warn(missing_docs)] #![warn(unsafe_op_in_unsafe_fn)] #![warn(variant_size_differences)] From 6f473f2986d1ea5c2ae2a5bfb9b322a344b777df Mon Sep 17 00:00:00 2001 From: Zachary Dremann Date: Sat, 3 Oct 2026 14:46:48 +0000 Subject: [PATCH 3/4] Invert masks with bitwise not `255 - mask` and `mask ^ 0xFF` both invert a u8 bitmask; `!mask` says so directly. --- roaring/src/bitmap/store/array_store/vector.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/roaring/src/bitmap/store/array_store/vector.rs b/roaring/src/bitmap/store/array_store/vector.rs index 5710034b..93a1a704 100644 --- a/roaring/src/bitmap/store/array_store/vector.rs +++ b/roaring/src/bitmap/store/array_store/vector.rs @@ -66,7 +66,7 @@ fn or_impl(simd: S, lhs: &[u16], rhs: &[u16], visitor: &mut impl Binary fn handle_vector(old: u16x8, new: u16x8) -> (u16x8, u8) { // `[old[7], new[0], ..., new[6]]` let tmp: u16x8 = old.slide::<7>(new); - let mask = 255 - tmp.simd_eq(new).to_bitmask() as u8; + let mask = !tmp.simd_eq(new).to_bitmask() as u8; (new, mask) } @@ -225,8 +225,8 @@ fn xor_impl(simd: S, lhs: &[u16], rhs: &[u16], visitor: &mut impl Binar let eq_l: mask16x8 = tmp2.simd_eq(tmp1); let eq_r: mask16x8 = tmp2.simd_eq(new); let eq_l_or_r: mask16x8 = eq_l | eq_r; - let mask: u8 = eq_l_or_r.to_bitmask() as u8; - (tmp2, 255 - mask) + let mask: u8 = !(eq_l_or_r.to_bitmask() as u8); + (tmp2, mask) } if (lhs.len() < 8) || (rhs.len() < 8) { @@ -357,7 +357,7 @@ fn sub_impl(simd: S, lhs: &[u16], rhs: &[u16], visitor: &mut impl Binar // Ok. In this code path, we are ready to write our v_a // because there is no need to read more from B, they will // all be large values. - let bitmask_belongs_to_difference = runningmask_a_found_in_b ^ 0xFF; + let bitmask_belongs_to_difference = !runningmask_a_found_in_b; visitor.visit_vector(v_a, bitmask_belongs_to_difference); i += LANES; if i == st_a { @@ -399,7 +399,7 @@ fn sub_impl(simd: S, lhs: &[u16], rhs: &[u16], visitor: &mut impl Binar let used_rhs = remaining_rhs.partition_point(|&b| b <= max_va); j += used_rhs; } - let bitmask_belongs_to_difference: u8 = runningmask_a_found_in_b ^ 0xFF; + let bitmask_belongs_to_difference: u8 = !runningmask_a_found_in_b; visitor.visit_vector(v_a, bitmask_belongs_to_difference); i += LANES; } From 632d7c5c74ec5551775d5e883d38c707b4cc4244 Mon Sep 17 00:00:00 2001 From: Zachary Dremann Date: Thu, 22 Oct 2026 19:31:58 -0400 Subject: [PATCH 4/4] feat!: remove the simd feature, always use SIMD fearless_simd works on stable, so there's no reason to keep it behind a flag anymore. Drops the scalar fallbacks in ArrayStore and the simd-only CI jobs. BREAKING CHANGE: the `simd` feature no longer exists. --- .github/workflows/test.yml | 19 +------ README.md | 8 +-- benchmarks/Cargo.toml | 3 -- fuzz/Cargo.toml | 3 -- roaring/Cargo.toml | 9 ++-- roaring/src/bitmap/store/array_store/mod.rs | 51 +++---------------- .../src/bitmap/store/array_store/vector.rs | 2 - .../src/bitmap/store/array_store/visitor.rs | 6 --- 8 files changed, 15 insertions(+), 86 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7cb9e6fa..55a45bd8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -57,11 +57,6 @@ jobs: cargo clippy -p roaring --all-targets --no-default-features -- -D warnings cargo clippy -p roaring --all-targets --features serde -- -Dclippy::todo -D warnings - - name: Check SIMD - run: | - cargo clippy -p roaring --all-targets --all-features -- -D warnings - cargo clippy -p roaring --all-targets --no-default-features --features simd -- -D warnings - - name: Check formatting run: cargo fmt --all -- --check @@ -78,9 +73,6 @@ jobs: features: - default - no-std - include: - - rust: stable - features: simd env: RUSTFLAGS: "-C target-cpu=native -C opt-level=3" ROARINGRS_BENCH_OFFLINE: "true" @@ -124,12 +116,6 @@ jobs: if: matrix.features == 'no-std' run: cargo test -p roaring --no-default-features - - name: SIMD test - if: matrix.features == 'simd' - run: | - cargo test -p roaring --features simd - cargo test -p roaring --no-default-features --features simd - miri: runs-on: ubuntu-latest needs: build @@ -204,7 +190,4 @@ jobs: fuzz/corpus - name: Run Fuzzer vs croaring for 15 minutes - run: cargo fuzz run against_croaring -s none -- -timeout=5 -max_total_time=900 - - - name: Run Fuzzer (with simd) vs croaring for 15 minutes - run: cargo fuzz run --features=simd against_croaring -- -timeout=5 -max_total_time=900 + run: cargo fuzz run against_croaring -- -timeout=5 -max_total_time=900 diff --git a/README.md b/README.md index 29794096..6adb1344 100644 --- a/README.md +++ b/README.md @@ -68,10 +68,10 @@ https://github.com/RoaringBitmap/roaring-rs/actions/workflows/test.yml/badge.svg [real-world datasets]: https://github.com/RoaringBitmap/real-roaring-datasets [on the user guide]: https://bheisler.github.io/criterion.rs/book/user_guide/user_guide.html -## Experimental features +## SIMD -The `simd` feature is in active development. It is implemented with the [fearless_simd] crate and works on stable -Rust. When the `std` feature is enabled, the best available SIMD instruction set is detected at runtime on x86 and -x86-64; otherwise, it is determined by the target features enabled at compile time (e.g. with `-C target-cpu=native`). +Some operations are accelerated with SIMD, implemented with the [fearless_simd] crate. When the `std` feature is +enabled, the best available SIMD instruction set is detected at runtime on x86 and x86-64; otherwise, it is determined +by the target features enabled at compile time (e.g. with `-C target-cpu=native`). [fearless_simd]: https://docs.rs/fearless_simd diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml index 34e250ef..491d6d78 100644 --- a/benchmarks/Cargo.toml +++ b/benchmarks/Cargo.toml @@ -17,9 +17,6 @@ indicatif = { workspace = true } itertools = { workspace = true } zip = { workspace = true, default-features = false, features = ["deflate"] } -[features] -simd = ["roaring/simd"] - [[bench]] name = "lib" harness = false diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 0ecade04..8d9b3465 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -12,9 +12,6 @@ libfuzzer-sys = { version = "0.4.9", features = ["arbitrary-derive"] } roaring = { path = "../roaring" } croaring = "2.0" -[features] -simd = ["roaring/simd"] - [[bin]] name = "against_croaring" path = "fuzz_targets/against_croaring.rs" diff --git a/roaring/Cargo.toml b/roaring/Cargo.toml index 826194ef..cc2c65df 100644 --- a/roaring/Cargo.toml +++ b/roaring/Cargo.toml @@ -20,15 +20,14 @@ license = "MIT OR Apache-2.0" bytemuck = { workspace = true, optional = true } byteorder = { workspace = true, optional = true } serde = { workspace = true, optional = true } -# `libm` is needed for no_std builds; with `std` enabled, fearless_simd uses std instead -fearless_simd = { workspace = true, optional = true, features = ["libm"] } -fearless_simd_macros = { workspace = true, optional = true } +# At least one of std and libm is required; std overrides libm. +fearless_simd = { workspace = true, features = ["libm"] } +fearless_simd_macros = { workspace = true } [features] default = ["std"] serde = ["dep:serde", "std"] -simd = ["dep:fearless_simd", "dep:fearless_simd_macros"] -std = ["dep:bytemuck", "dep:byteorder", "fearless_simd?/std"] +std = ["dep:bytemuck", "dep:byteorder", "fearless_simd/std"] [dev-dependencies] proptest = { workspace = true } diff --git a/roaring/src/bitmap/store/array_store/mod.rs b/roaring/src/bitmap/store/array_store/mod.rs index ec3cbcf9..549f17c7 100644 --- a/roaring/src/bitmap/store/array_store/mod.rs +++ b/roaring/src/bitmap/store/array_store/mod.rs @@ -256,10 +256,7 @@ impl ArrayStore { pub fn intersection_len(&self, other: &Self) -> u64 { let mut visitor = CardinalityCounter::new(); - #[cfg(feature = "simd")] vector::and(self.as_slice(), other.as_slice(), &mut visitor); - #[cfg(not(feature = "simd"))] - scalar::and(self.as_slice(), other.as_slice(), &mut visitor); visitor.into_inner() } @@ -418,10 +415,7 @@ impl BitOr for &ArrayStore { #[allow(clippy::suspicious_arithmetic_impl)] let capacity = self.vec.len() + rhs.vec.len(); let mut visitor = VecWriter::new(capacity); - #[cfg(feature = "simd")] vector::or(self.as_slice(), rhs.as_slice(), &mut visitor); - #[cfg(not(feature = "simd"))] - scalar::or(self.as_slice(), rhs.as_slice(), &mut visitor); ArrayStore::from_vec_unchecked(visitor.into_inner()) } } @@ -431,31 +425,16 @@ impl BitAnd for &ArrayStore { fn bitand(self, rhs: Self) -> Self::Output { let mut visitor = VecWriter::new(self.vec.len().min(rhs.vec.len())); - #[cfg(feature = "simd")] vector::and(self.as_slice(), rhs.as_slice(), &mut visitor); - #[cfg(not(feature = "simd"))] - scalar::and(self.as_slice(), rhs.as_slice(), &mut visitor); ArrayStore::from_vec_unchecked(visitor.into_inner()) } } impl BitAndAssign<&Self> for ArrayStore { - #[allow(clippy::suspicious_op_assign_impl)] fn bitand_assign(&mut self, rhs: &Self) { - #[cfg(feature = "simd")] - { - let mut visitor = VecWriter::new(self.vec.len().min(rhs.vec.len())); - vector::and(self.as_slice(), rhs.as_slice(), &mut visitor); - self.vec = visitor.into_inner() - } - #[cfg(not(feature = "simd"))] - { - let mut i = 0; - self.retain(|x| { - i += rhs.iter().skip(i).position(|y| *y >= x).unwrap_or(rhs.vec.len()); - rhs.vec.get(i).is_some_and(|y| x == *y) - }); - } + let mut visitor = VecWriter::new(self.vec.len().min(rhs.vec.len())); + vector::and(self.as_slice(), rhs.as_slice(), &mut visitor); + self.vec = visitor.into_inner() } } @@ -470,31 +449,16 @@ impl Sub for &ArrayStore { fn sub(self, rhs: Self) -> Self::Output { let mut visitor = VecWriter::new(self.vec.len()); - #[cfg(feature = "simd")] vector::sub(self.as_slice(), rhs.as_slice(), &mut visitor); - #[cfg(not(feature = "simd"))] - scalar::sub(self.as_slice(), rhs.as_slice(), &mut visitor); ArrayStore::from_vec_unchecked(visitor.into_inner()) } } impl SubAssign<&Self> for ArrayStore { - #[allow(clippy::suspicious_op_assign_impl)] fn sub_assign(&mut self, rhs: &Self) { - #[cfg(feature = "simd")] - { - let mut visitor = VecWriter::new(self.vec.len().min(rhs.vec.len())); - vector::sub(self.as_slice(), rhs.as_slice(), &mut visitor); - self.vec = visitor.into_inner() - } - #[cfg(not(feature = "simd"))] - { - let mut i = 0; - self.retain(|x| { - i += rhs.iter().skip(i).position(|y| *y >= x).unwrap_or(rhs.vec.len()); - rhs.vec.get(i).is_none_or(|y| x != *y) - }); - } + let mut visitor = VecWriter::new(self.vec.len().min(rhs.vec.len())); + vector::sub(self.as_slice(), rhs.as_slice(), &mut visitor); + self.vec = visitor.into_inner() } } @@ -511,10 +475,7 @@ impl BitXor for &ArrayStore { #[allow(clippy::suspicious_arithmetic_impl)] let capacity = self.vec.len() + rhs.vec.len(); let mut visitor = VecWriter::new(capacity); - #[cfg(feature = "simd")] vector::xor(self.as_slice(), rhs.as_slice(), &mut visitor); - #[cfg(not(feature = "simd"))] - scalar::xor(self.as_slice(), rhs.as_slice(), &mut visitor); ArrayStore::from_vec_unchecked(visitor.into_inner()) } } diff --git a/roaring/src/bitmap/store/array_store/vector.rs b/roaring/src/bitmap/store/array_store/vector.rs index 93a1a704..bbfacbbc 100644 --- a/roaring/src/bitmap/store/array_store/vector.rs +++ b/roaring/src/bitmap/store/array_store/vector.rs @@ -15,8 +15,6 @@ //! features by being inlined, and `#[simd]` does not force inlining, which can leave them out of //! line in the hot loops. -#![cfg(feature = "simd")] - use super::scalar; use crate::bitmap::store::array_store::visitor::BinaryOperationVisitor; use fearless_simd::prelude::*; diff --git a/roaring/src/bitmap/store/array_store/visitor.rs b/roaring/src/bitmap/store/array_store/visitor.rs index 5a1025eb..c2cbf5c0 100644 --- a/roaring/src/bitmap/store/array_store/visitor.rs +++ b/roaring/src/bitmap/store/array_store/visitor.rs @@ -1,8 +1,5 @@ -#[cfg(feature = "simd")] use crate::bitmap::store::array_store::vector::swizzle_to_front; -#[cfg(feature = "simd")] use fearless_simd::{prelude::*, u16x8}; -#[cfg(feature = "simd")] use fearless_simd_macros::simd; #[cfg(not(feature = "std"))] @@ -16,7 +13,6 @@ use alloc::vec::Vec; /// their results. Future work can utilize the exiting algorithms to trivially implement /// computing the cardinality of an operation without materializng a new bitmap. pub trait BinaryOperationVisitor { - #[cfg(feature = "simd")] fn visit_vector(&mut self, value: u16x8, mask: u8); fn visit_scalar(&mut self, value: u16); fn visit_slice(&mut self, values: &[u16]); @@ -43,7 +39,6 @@ impl VecWriter { } impl BinaryOperationVisitor for VecWriter { - #[cfg(feature = "simd")] #[simd] fn visit_vector(&mut self, value: u16x8, mask: u8) { let result = swizzle_to_front(value, mask); @@ -82,7 +77,6 @@ impl CardinalityCounter { } impl BinaryOperationVisitor for CardinalityCounter { - #[cfg(feature = "simd")] fn visit_vector(&mut self, _value: u16x8, mask: u8) { self.count += mask.count_ones() as usize; }