diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d6a0b520..55a45bd8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,8 +35,17 @@ jobs: toolchain: ${{ matrix.rust }} components: rustfmt, clippy + # Builds use `-C target-cpu=native`, so cached artifacts (including proc macros, which run in + # the compiler) only work on runners with the same target features + - name: Identify native target features + id: native + run: | + echo "features=$(rustc --print cfg -C target-cpu=native | grep '^target_feature=' | sha256sum | cut -c1-12)" >> "$GITHUB_OUTPUT" + - name: Caching uses: Swatinem/rust-cache@v2 + with: + key: native-${{ steps.native.outputs.features }} - name: Check # clippy will also do a build check @@ -48,10 +57,6 @@ jobs: cargo clippy -p roaring --all-targets --no-default-features -- -D warnings cargo clippy -p roaring --all-targets --features serde -- -Dclippy::todo -D warnings - - name: Check SIMD - if: matrix.rust == 'nightly' - run: cargo clippy -p roaring --all-targets --all-features -- -D warnings - - name: Check formatting run: cargo fmt --all -- --check @@ -68,9 +73,6 @@ jobs: features: - default - no-std - include: - - rust: nightly - features: simd env: RUSTFLAGS: "-C target-cpu=native -C opt-level=3" ROARINGRS_BENCH_OFFLINE: "true" @@ -85,8 +87,17 @@ jobs: repository: "RoaringBitmap/real-roaring-datasets" path: "benchmarks/real-roaring-datasets" + # Builds use `-C target-cpu=native`, so cached artifacts (including proc macros, which run in + # the compiler) only work on runners with the same target features + - name: Identify native target features + id: native + run: | + echo "features=$(rustc --print cfg -C target-cpu=native | grep '^target_feature=' | sha256sum | cut -c1-12)" >> "$GITHUB_OUTPUT" + - name: Caching uses: Swatinem/rust-cache@v2 + with: + key: native-${{ steps.native.outputs.features }} - name: Initialize rust toolchain uses: dtolnay/rust-toolchain@master @@ -105,10 +116,6 @@ jobs: if: matrix.features == 'no-std' run: cargo test -p roaring --no-default-features - - name: SIMD test - if: matrix.rust == 'nightly' && matrix.features == 'simd' - run: cargo +nightly test -p roaring --features simd - miri: runs-on: ubuntu-latest needs: build @@ -126,8 +133,17 @@ jobs: toolchain: nightly components: miri + # Builds use `-C target-cpu=native`, so cached artifacts (including proc macros, which run in + # the compiler) only work on runners with the same target features + - name: Identify native target features + id: native + run: | + echo "features=$(rustc --print cfg -C target-cpu=native | grep '^target_feature=' | sha256sum | cut -c1-12)" >> "$GITHUB_OUTPUT" + - name: Caching uses: Swatinem/rust-cache@v2 + with: + key: native-${{ steps.native.outputs.features }} - name: Setup miri run: cargo miri setup @@ -150,8 +166,17 @@ jobs: with: toolchain: nightly + # Builds use `-C target-cpu=native`, so cached artifacts (including proc macros, which run in + # the compiler) only work on runners with the same target features + - name: Identify native target features + id: native + run: | + echo "features=$(rustc --print cfg -C target-cpu=native | grep '^target_feature=' | sha256sum | cut -c1-12)" >> "$GITHUB_OUTPUT" + - name: Caching uses: Swatinem/rust-cache@v2 + with: + key: native-${{ steps.native.outputs.features }} - name: Install cargo fuzz run: cargo install cargo-fuzz @@ -165,7 +190,4 @@ jobs: fuzz/corpus - name: Run Fuzzer vs croaring for 15 minutes - run: cargo fuzz run against_croaring -s none -- -timeout=5 -max_total_time=900 - - - name: Run Fuzzer (with simd) vs croaring for 15 minutes - run: cargo fuzz run --features=simd against_croaring -- -timeout=5 -max_total_time=900 + run: cargo fuzz run against_croaring -- -timeout=5 -max_total_time=900 diff --git a/Cargo.toml b/Cargo.toml index ff3f9007..8ff74a02 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,8 @@ roaring = { path = "roaring" } bytemuck = "1.25.2" byteorder = "1.5.0" criterion = "0.5" +fearless_simd = { version = "1.0.0", default-features = false } +fearless_simd_macros = "0.1.0" git2 = { version = "0.21", default-features = false } indicatif = "0.17" itertools = "0.15" diff --git a/README.md b/README.md index 132733ac..6adb1344 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,10 @@ https://github.com/RoaringBitmap/roaring-rs/actions/workflows/test.yml/badge.svg [real-world datasets]: https://github.com/RoaringBitmap/real-roaring-datasets [on the user guide]: https://bheisler.github.io/criterion.rs/book/user_guide/user_guide.html -## Experimental features +## SIMD -The `simd` feature is in active development. It has not been tested. If you would like to build with `simd` note that -`std::simd` is only available in Rust nightly. +Some operations are accelerated with SIMD, implemented with the [fearless_simd] crate. When the `std` feature is +enabled, the best available SIMD instruction set is detected at runtime on x86 and x86-64; otherwise, it is determined +by the target features enabled at compile time (e.g. with `-C target-cpu=native`). + +[fearless_simd]: https://docs.rs/fearless_simd diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml index 34e250ef..491d6d78 100644 --- a/benchmarks/Cargo.toml +++ b/benchmarks/Cargo.toml @@ -17,9 +17,6 @@ indicatif = { workspace = true } itertools = { workspace = true } zip = { workspace = true, default-features = false, features = ["deflate"] } -[features] -simd = ["roaring/simd"] - [[bench]] name = "lib" harness = false diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index 8545b3f4..c0aef096 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -19,9 +19,9 @@ checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" [[package]] name = "bytemuck" -version = "1.23.0" +version = "1.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9134a6ef01ce4b366b50689c94f82c14bc72bc5d0386829828a2e2752ef7958c" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" [[package]] name = "byteorder" @@ -72,7 +72,27 @@ checksum = "30542c1ad912e0e3d22a1935c290e12e8a29d704a420177a31faad4a601a0800" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.101", +] + +[[package]] +name = "fearless_simd" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3772b63c40606beea8fe1f7b06a60de27ceb9bfc2d9dad3427368ca41dc7e62" +dependencies = [ + "libm", +] + +[[package]] +name = "fearless_simd_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e91d94c6885390b81782d7ee60fd6583b33907333a8b340de25ccf468353315" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", ] [[package]] @@ -113,6 +133,12 @@ dependencies = [ "cc", ] +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + [[package]] name = "proc-macro2" version = "1.0.95" @@ -139,10 +165,12 @@ checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" [[package]] name = "roaring" -version = "0.11.4" +version = "0.11.5" dependencies = [ "bytemuck", "byteorder", + "fearless_simd", + "fearless_simd_macros", ] [[package]] @@ -171,6 +199,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "unicode-ident" version = "1.0.18" diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 0ecade04..8d9b3465 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -12,9 +12,6 @@ libfuzzer-sys = { version = "0.4.9", features = ["arbitrary-derive"] } roaring = { path = "../roaring" } croaring = "2.0" -[features] -simd = ["roaring/simd"] - [[bin]] name = "against_croaring" path = "fuzz_targets/against_croaring.rs" diff --git a/roaring/Cargo.toml b/roaring/Cargo.toml index b3c50c01..cc2c65df 100644 --- a/roaring/Cargo.toml +++ b/roaring/Cargo.toml @@ -20,12 +20,14 @@ license = "MIT OR Apache-2.0" bytemuck = { workspace = true, optional = true } byteorder = { workspace = true, optional = true } serde = { workspace = true, optional = true } +# At least one of std and libm is required; std overrides libm. +fearless_simd = { workspace = true, features = ["libm"] } +fearless_simd_macros = { workspace = true } [features] default = ["std"] serde = ["dep:serde", "std"] -simd = [] -std = ["dep:bytemuck", "dep:byteorder"] +std = ["dep:bytemuck", "dep:byteorder", "fearless_simd/std"] [dev-dependencies] proptest = { workspace = true } diff --git a/roaring/src/bitmap/store/array_store/mod.rs b/roaring/src/bitmap/store/array_store/mod.rs index ec3cbcf9..549f17c7 100644 --- a/roaring/src/bitmap/store/array_store/mod.rs +++ b/roaring/src/bitmap/store/array_store/mod.rs @@ -256,10 +256,7 @@ impl ArrayStore { pub fn intersection_len(&self, other: &Self) -> u64 { let mut visitor = CardinalityCounter::new(); - #[cfg(feature = "simd")] vector::and(self.as_slice(), other.as_slice(), &mut visitor); - #[cfg(not(feature = "simd"))] - scalar::and(self.as_slice(), other.as_slice(), &mut visitor); visitor.into_inner() } @@ -418,10 +415,7 @@ impl BitOr for &ArrayStore { #[allow(clippy::suspicious_arithmetic_impl)] let capacity = self.vec.len() + rhs.vec.len(); let mut visitor = VecWriter::new(capacity); - #[cfg(feature = "simd")] vector::or(self.as_slice(), rhs.as_slice(), &mut visitor); - #[cfg(not(feature = "simd"))] - scalar::or(self.as_slice(), rhs.as_slice(), &mut visitor); ArrayStore::from_vec_unchecked(visitor.into_inner()) } } @@ -431,31 +425,16 @@ impl BitAnd for &ArrayStore { fn bitand(self, rhs: Self) -> Self::Output { let mut visitor = VecWriter::new(self.vec.len().min(rhs.vec.len())); - #[cfg(feature = "simd")] vector::and(self.as_slice(), rhs.as_slice(), &mut visitor); - #[cfg(not(feature = "simd"))] - scalar::and(self.as_slice(), rhs.as_slice(), &mut visitor); ArrayStore::from_vec_unchecked(visitor.into_inner()) } } impl BitAndAssign<&Self> for ArrayStore { - #[allow(clippy::suspicious_op_assign_impl)] fn bitand_assign(&mut self, rhs: &Self) { - #[cfg(feature = "simd")] - { - let mut visitor = VecWriter::new(self.vec.len().min(rhs.vec.len())); - vector::and(self.as_slice(), rhs.as_slice(), &mut visitor); - self.vec = visitor.into_inner() - } - #[cfg(not(feature = "simd"))] - { - let mut i = 0; - self.retain(|x| { - i += rhs.iter().skip(i).position(|y| *y >= x).unwrap_or(rhs.vec.len()); - rhs.vec.get(i).is_some_and(|y| x == *y) - }); - } + let mut visitor = VecWriter::new(self.vec.len().min(rhs.vec.len())); + vector::and(self.as_slice(), rhs.as_slice(), &mut visitor); + self.vec = visitor.into_inner() } } @@ -470,31 +449,16 @@ impl Sub for &ArrayStore { fn sub(self, rhs: Self) -> Self::Output { let mut visitor = VecWriter::new(self.vec.len()); - #[cfg(feature = "simd")] vector::sub(self.as_slice(), rhs.as_slice(), &mut visitor); - #[cfg(not(feature = "simd"))] - scalar::sub(self.as_slice(), rhs.as_slice(), &mut visitor); ArrayStore::from_vec_unchecked(visitor.into_inner()) } } impl SubAssign<&Self> for ArrayStore { - #[allow(clippy::suspicious_op_assign_impl)] fn sub_assign(&mut self, rhs: &Self) { - #[cfg(feature = "simd")] - { - let mut visitor = VecWriter::new(self.vec.len().min(rhs.vec.len())); - vector::sub(self.as_slice(), rhs.as_slice(), &mut visitor); - self.vec = visitor.into_inner() - } - #[cfg(not(feature = "simd"))] - { - let mut i = 0; - self.retain(|x| { - i += rhs.iter().skip(i).position(|y| *y >= x).unwrap_or(rhs.vec.len()); - rhs.vec.get(i).is_none_or(|y| x != *y) - }); - } + let mut visitor = VecWriter::new(self.vec.len().min(rhs.vec.len())); + vector::sub(self.as_slice(), rhs.as_slice(), &mut visitor); + self.vec = visitor.into_inner() } } @@ -511,10 +475,7 @@ impl BitXor for &ArrayStore { #[allow(clippy::suspicious_arithmetic_impl)] let capacity = self.vec.len() + rhs.vec.len(); let mut visitor = VecWriter::new(capacity); - #[cfg(feature = "simd")] vector::xor(self.as_slice(), rhs.as_slice(), &mut visitor); - #[cfg(not(feature = "simd"))] - scalar::xor(self.as_slice(), rhs.as_slice(), &mut visitor); ArrayStore::from_vec_unchecked(visitor.into_inner()) } } diff --git a/roaring/src/bitmap/store/array_store/vector.rs b/roaring/src/bitmap/store/array_store/vector.rs index 6fc14a9a..bbfacbbc 100644 --- a/roaring/src/bitmap/store/array_store/vector.rs +++ b/roaring/src/bitmap/store/array_store/vector.rs @@ -4,18 +4,45 @@ //! Prior work: Schlegel et al., Fast Sorted-Set Intersection using SIMD Instructions //! //! Rust port notes: -//! The x86 PCMPESTRM instruction has been replaced with a simple vector or-shift -//! While several more instructions, this is what is available through LLVM intrinsics -//! and is portable. - -#![cfg(feature = "simd")] +//! The x86 PCMPESTRM instruction has been replaced with a portable all-pairs comparison +//! (see `matrix_cmp_u16`), which takes more instructions but is available on every SIMD level. +//! +//! The public functions select a SIMD level at runtime with `fearless_simd::dispatch!`, and call +//! implementations generic over the SIMD level. Those are annotated with `#[simd]`, so that they +//! are compiled with the matching target features enabled. +//! +//! The small helpers they call are `#[inline(always)]` rather than `#[simd]`: they get the target +//! features by being inlined, and `#[simd]` does not force inlining, which can leave them out of +//! line in the hot loops. use super::scalar; -use core::simd::cmp::{SimdPartialEq, SimdPartialOrd}; -use core::simd::{mask16x8, u16x8, u8x16, Mask, Select as _, Simd, SimdElement, ToBytes}; +use crate::bitmap::store::array_store::visitor::BinaryOperationVisitor; +use fearless_simd::prelude::*; +use fearless_simd::{dispatch, mask16x8, u16x8, Level}; +use fearless_simd_macros::simd; -// a one-pass SSE union algorithm +/// The number of lanes in a `u16x8` +const LANES: usize = 8; + +/// The SIMD level to use. +/// +/// With `std`, this is detected at runtime on x86/x86-64 (and cached by `fearless_simd`). +/// Otherwise, it is determined by the target features enabled at compile time. +#[inline] +fn level() -> Level { + #[cfg(feature = "std")] + return Level::new(); + #[cfg(not(feature = "std"))] + return Level::baseline(); +} + +// a one-pass union algorithm pub fn or(lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) { + dispatch!(level(), simd => or_impl(simd, lhs, rhs, visitor)) +} + +#[simd] +fn or_impl(simd: S, lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) { // De-duplicates `slice` in place // Returns the end index of the deduplicated slice. // elements after the return value are not guaranteed to be unique or in order @@ -31,11 +58,14 @@ pub fn or(lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) { pos } - #[inline] - fn handle_vector(old: u16x8, new: u16x8, f: impl FnOnce(u16x8, u8)) { - let tmp: u16x8 = Shr1::concat_swizzle(new, old); - let mask = 255 - tmp.simd_eq(new).to_bitmask() as u8; - f(new, mask); + // returns `new`, with a mask of its values which were not already written, + // assuming that the previously written vector was `old` + #[inline(always)] + fn handle_vector(old: u16x8, new: u16x8) -> (u16x8, u8) { + // `[old[7], new[0], ..., new[6]]` + let tmp: u16x8 = old.slide::<7>(new); + let mask = !tmp.simd_eq(new).to_bitmask() as u8; + (new, mask) } if (lhs.len() < 8) || (rhs.len() < 8) { @@ -46,21 +76,22 @@ pub fn or(lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) { let len1: usize = lhs.len() / 8; let len2: usize = rhs.len() / 8; - let v_a: u16x8 = load(lhs); - let v_b: u16x8 = load(rhs); + let v_a: u16x8 = load(simd, lhs); + let v_b: u16x8 = load(simd, rhs); let [mut v_min, mut v_max] = simd_merge_u16(v_a, v_b); let mut i = 1; let mut j = 1; - handle_vector(Simd::splat(u16::MAX), v_min, |v, m| visitor.visit_vector(v, m)); - let mut v_prev: u16x8 = v_min; + let (v, m) = handle_vector(u16x8::splat(simd, u16::MAX), v_min); + visitor.visit_vector(v, m); + let mut v_prev: u16x8 = v_min; if (i < len1) && (j < len2) { - let mut v: u16x8; + let mut v: u16x8; let mut cur_a: u16 = lhs[8 * i]; let mut cur_b: u16 = rhs[8 * j]; loop { if cur_a <= cur_b { - v = load(&lhs[8 * i..]); + v = load(simd, &lhs[8 * i..]); i += 1; if i < len1 { cur_a = lhs[8 * i]; @@ -68,7 +99,7 @@ pub fn or(lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) { break; } } else { - v = load(&rhs[8 * j..]); + v = load(simd, &rhs[8 * j..]); j += 1; if j < len2 { cur_b = rhs[8 * j]; @@ -77,11 +108,13 @@ pub fn or(lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) { } } [v_min, v_max] = simd_merge_u16(v, v_max); - handle_vector(v_prev, v_min, |v, m| visitor.visit_vector(v, m)); + let (v, m) = handle_vector(v_prev, v_min); + visitor.visit_vector(v, m); v_prev = v_min; } [v_min, v_max] = simd_merge_u16(v, v_max); - handle_vector(v_prev, v_min, |v, m| visitor.visit_vector(v, m)); + let (v, m) = handle_vector(v_prev, v_min); + visitor.visit_vector(v, m); v_prev = v_min; } @@ -92,11 +125,9 @@ pub fn or(lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) { // // copy the small end on a tmp buffer let mut buffer: [u16; 16] = [0; 16]; - let mut rem = 0; - handle_vector(v_prev, v_max, |v, m| { - store(swizzle_to_front(v, m), buffer.as_mut_slice()); - rem = m.count_ones() as usize; - }); + let (v, m) = handle_vector(v_prev, v_max); + store(swizzle_to_front(v, m), buffer.as_mut_slice()); + let mut rem = m.count_ones() as usize; let (tail_a, tail_b, tail_len) = if i == len1 { (&lhs[8 * i..], &rhs[8 * j..], lhs.len() - 8 * len1) @@ -117,33 +148,38 @@ pub fn or(lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) { } pub fn and(lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) { - let st_a = (lhs.len() / u16x8::LEN) * u16x8::LEN; - let st_b = (rhs.len() / u16x8::LEN) * u16x8::LEN; + dispatch!(level(), simd => and_impl(simd, lhs, rhs, visitor)) +} + +#[simd] +fn and_impl(simd: S, lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) { + let st_a = (lhs.len() / LANES) * LANES; + let st_b = (rhs.len() / LANES) * LANES; let mut i: usize = 0; let mut j: usize = 0; if (i < st_a) && (j < st_b) { - let mut v_a: u16x8 = load(&lhs[i..]); - let mut v_b: u16x8 = load(&rhs[j..]); + let mut v_a: u16x8 = load(simd, &lhs[i..]); + let mut v_b: u16x8 = load(simd, &rhs[j..]); loop { let mask = matrix_cmp_u16(v_a, v_b).to_bitmask() as u8; visitor.visit_vector(v_a, mask); - let a_max: u16 = lhs[i + u16x8::LEN - 1]; - let b_max: u16 = rhs[j + u16x8::LEN - 1]; + let a_max: u16 = lhs[i + LANES - 1]; + let b_max: u16 = rhs[j + LANES - 1]; if a_max <= b_max { - i += u16x8::LEN; + i += LANES; if i == st_a { break; } - v_a = load(&lhs[i..]); + v_a = load(simd, &lhs[i..]); } if b_max <= a_max { - j += u16x8::LEN; + j += LANES; if j == st_b { break; } - v_b = load(&rhs[j..]); + v_b = load(simd, &rhs[j..]); } } } @@ -152,8 +188,13 @@ pub fn and(lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) scalar::and(&lhs[i..], &rhs[j..], visitor); } -// a one-pass SSE xor algorithm +// a one-pass xor algorithm pub fn xor(lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) { + dispatch!(level(), simd => xor_impl(simd, lhs, rhs, visitor)) +} + +#[simd] +fn xor_impl(simd: S, lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) { /// De-duplicates `slice` in place, removing _both_ duplicates /// Returns the end index of the xor-ed slice. /// elements after the return value are not guaranteed to be unique or in order @@ -171,17 +212,19 @@ pub fn xor(lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) pos } - // write vector new, while omitting repeated values assuming that previously - // written vector was "old" - #[inline] - fn handle_vector(old: u16x8, new: u16x8, f: impl FnOnce(u16x8, u8)) { - let tmp1: u16x8 = Shr2::concat_swizzle(new, old); - let tmp2: u16x8 = Shr1::concat_swizzle(new, old); - let eq_l: mask16x8 = tmp2.simd_eq(tmp1); - let eq_r: mask16x8 = tmp2.simd_eq(new); - let eq_l_or_r: mask16x8 = eq_l | eq_r; - let mask: u8 = eq_l_or_r.to_bitmask() as u8; - f(tmp2, 255 - mask); + // returns the vector to write, and a mask of the values to write from it, + // omitting repeated values assuming that previously written vector was "old" + #[inline(always)] + fn handle_vector(old: u16x8, new: u16x8) -> (u16x8, u8) { + // `[old[6], old[7], new[0], ..., new[5]]` + let tmp1: u16x8 = old.slide::<6>(new); + // `[old[7], new[0], ..., new[6]]` + let tmp2: u16x8 = old.slide::<7>(new); + let eq_l: mask16x8 = tmp2.simd_eq(tmp1); + let eq_r: mask16x8 = tmp2.simd_eq(new); + let eq_l_or_r: mask16x8 = eq_l | eq_r; + let mask: u8 = !(eq_l_or_r.to_bitmask() as u8); + (tmp2, mask) } if (lhs.len() < 8) || (rhs.len() < 8) { @@ -192,21 +235,22 @@ pub fn xor(lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) let len1: usize = lhs.len() / 8; let len2: usize = rhs.len() / 8; - let v_a: u16x8 = load(lhs); - let v_b: u16x8 = load(rhs); + let v_a: u16x8 = load(simd, lhs); + let v_b: u16x8 = load(simd, rhs); let [mut v_min, mut v_max] = simd_merge_u16(v_a, v_b); let mut i = 1; let mut j = 1; - handle_vector(Simd::splat(u16::MAX), v_min, |v, m| visitor.visit_vector(v, m)); - let mut v_prev: u16x8 = v_min; + let (v, m) = handle_vector(u16x8::splat(simd, u16::MAX), v_min); + visitor.visit_vector(v, m); + let mut v_prev: u16x8 = v_min; if (i < len1) && (j < len2) { - let mut v: u16x8; + let mut v: u16x8; let mut cur_a: u16 = lhs[8 * i]; let mut cur_b: u16 = rhs[8 * j]; loop { if cur_a <= cur_b { - v = load(&lhs[8 * i..]); + v = load(simd, &lhs[8 * i..]); i += 1; if i < len1 { cur_a = lhs[8 * i]; @@ -214,7 +258,7 @@ pub fn xor(lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) break; } } else { - v = load(&rhs[8 * j..]); + v = load(simd, &rhs[8 * j..]); j += 1; if j < len2 { cur_b = rhs[8 * j]; @@ -223,11 +267,13 @@ pub fn xor(lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) } } [v_min, v_max] = simd_merge_u16(v, v_max); - handle_vector(v_prev, v_min, |v, m| visitor.visit_vector(v, m)); + let (v, m) = handle_vector(v_prev, v_min); + visitor.visit_vector(v, m); v_prev = v_min; } [v_min, v_max] = simd_merge_u16(v, v_max); - handle_vector(v_prev, v_min, |v, m| visitor.visit_vector(v, m)); + let (v, m) = handle_vector(v_prev, v_min); + visitor.visit_vector(v, m); v_prev = v_min; } @@ -239,13 +285,14 @@ pub fn xor(lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) // last value of vecMax, let mut buffer: [u16; 17] = [0; 17]; // remaining size - let mut rem = 0; - handle_vector(v_prev, v_max, |v, m| { - store(swizzle_to_front(v, m), buffer.as_mut_slice()); - rem = m.count_ones() as usize; - }); - - let arr_max = v_max.as_array(); + let (v, m) = handle_vector(v_prev, v_max); + store(swizzle_to_front(v, m), buffer.as_mut_slice()); + let mut rem = m.count_ones() as usize; + + // Store `v_max` rather than reading its lanes with indexing, which makes LLVM split + // `v_max` into pieces in the merge loop above + let mut arr_max = [0u16; LANES]; + v_max.store_array(&mut arr_max); let vec7 = arr_max[7]; let vec6 = arr_max[6]; if vec6 != vec7 { @@ -272,6 +319,11 @@ pub fn xor(lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) } pub fn sub(lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) { + dispatch!(level(), simd => sub_impl(simd, lhs, rhs, visitor)) +} + +#[simd] +fn sub_impl(simd: S, lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) { // we handle the degenerate cases if lhs.is_empty() { return; @@ -280,14 +332,14 @@ pub fn sub(lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) return; } - let st_a = (lhs.len() / u16x8::LEN) * u16x8::LEN; - let st_b = (rhs.len() / u16x8::LEN) * u16x8::LEN; + let st_a = (lhs.len() / LANES) * LANES; + let st_b = (rhs.len() / LANES) * LANES; let mut i = 0; let mut j = 0; if (i < st_a) && (j < st_b) { - let mut v_a: u16x8 = load(&lhs[i..]); - let mut v_b: u16x8 = load(&rhs[j..]); + let mut v_a: u16x8 = load(simd, &lhs[i..]); + let mut v_b: u16x8 = load(simd, &rhs[j..]); // we have a running mask which indicates which values from a have been // spotted in b, these don't get written out. let mut runningmask_a_found_in_b: u8 = 0; @@ -297,28 +349,28 @@ pub fn sub(lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) let a_found_in_b: u8 = matrix_cmp_u16(v_a, v_b).to_bitmask() as u8; runningmask_a_found_in_b |= a_found_in_b; // we always compare the last values of A and B - let a_max: u16 = lhs[i + u16x8::LEN - 1]; - let b_max: u16 = rhs[j + u16x8::LEN - 1]; + let a_max: u16 = lhs[i + LANES - 1]; + let b_max: u16 = rhs[j + LANES - 1]; if a_max <= b_max { // Ok. In this code path, we are ready to write our v_a // because there is no need to read more from B, they will // all be large values. - let bitmask_belongs_to_difference = runningmask_a_found_in_b ^ 0xFF; + let bitmask_belongs_to_difference = !runningmask_a_found_in_b; visitor.visit_vector(v_a, bitmask_belongs_to_difference); - i += u16x8::LEN; + i += LANES; if i == st_a { break; } runningmask_a_found_in_b = 0; - v_a = load(&lhs[i..]); + v_a = load(simd, &lhs[i..]); } if b_max <= a_max { // in this code path, the current v_b has become useless - j += u16x8::LEN; + j += LANES; if j == st_b { break; } - v_b = load(&rhs[j..]); + v_b = load(simd, &rhs[j..]); } } @@ -336,16 +388,18 @@ pub fn sub(lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) // Ensure the buffer is filled with a value we should remove: we do not want to // end up trying to remove zero values which aren't actually in rhs buffer[remaining_rhs.len()..].fill(remaining_rhs[0]); - v_b = Simd::from_array(buffer); + v_b = load(simd, &buffer); let a_found_in_b: u8 = matrix_cmp_u16(v_a, v_b).to_bitmask() as u8; runningmask_a_found_in_b |= a_found_in_b; - let [.., max_va] = *v_a.as_array(); + // Read from `lhs` (which `v_a` was loaded from): reading a lane of `v_a` makes LLVM + // split `v_a` into pieces in the loop above + let max_va = lhs[i + LANES - 1]; let used_rhs = remaining_rhs.partition_point(|&b| b <= max_va); j += used_rhs; } - let bitmask_belongs_to_difference: u8 = runningmask_a_found_in_b ^ 0xFF; + let bitmask_belongs_to_difference: u8 = !runningmask_a_found_in_b; visitor.visit_vector(v_a, bitmask_belongs_to_difference); - i += u16x8::LEN; + i += LANES; } } @@ -353,67 +407,22 @@ pub fn sub(lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) scalar::sub(&lhs[i..], &rhs[j..], visitor); } -/// compute the min for each lane in `a` and `b` -#[inline] -fn lanes_min_u16( - lhs: Simd, - rhs: Simd, -) -> Simd { - lhs.simd_le(rhs).select(lhs, rhs) -} - -/// compute the max for each lane in `a` and `b` -#[inline] -fn lanes_max_u16( - lhs: Simd, - rhs: Simd, -) -> Simd { - lhs.simd_gt(rhs).select(lhs, rhs) -} - -#[inline] -pub fn load(src: &[U]) -> Simd -where - U: SimdElement + PartialOrd, -{ - debug_assert!(src.len() >= LANES); - unsafe { load_unchecked(src) } -} - -/// write `v` to slice `out` without checking bounds +/// load the first `LANES` values of `src` /// -/// ### Safety -/// - The caller must ensure `LANES` does not exceed the allocation for `out` -#[inline] -pub unsafe fn load_unchecked(src: &[U]) -> Simd -where - U: SimdElement + PartialOrd, -{ - unsafe { core::ptr::read_unaligned(src as *const _ as *const Simd) } +/// ### Panics +/// - If `src` is shorter than `LANES` +#[inline(always)] +fn load(simd: S, src: &[u16]) -> u16x8 { + u16x8::from_slice(simd, &src[..LANES]) } -/// write `v` to slice `out` -#[inline] -pub fn store(v: Simd, out: &mut [U]) -where - U: SimdElement + PartialOrd, -{ - debug_assert!(out.len() >= LANES); - unsafe { - store_unchecked(v, out); - } -} - -/// write `v` to slice `out` without checking bounds +/// write `v` to the first `LANES` values of `out` /// -/// ### Safety -/// - The caller must ensure `LANES` does not exceed the allocation for `out` -#[inline] -unsafe fn store_unchecked(v: Simd, out: &mut [U]) -where - U: SimdElement + PartialOrd, -{ - unsafe { core::ptr::write_unaligned(out as *mut _ as *mut Simd, v) } +/// ### Panics +/// - If `out` is shorter than `LANES` +#[inline(always)] +fn store(v: u16x8, out: &mut [u16]) { + v.store_slice(&mut out[..LANES]) } /// Compare all lanes in `a` to all lanes in `b` @@ -422,15 +431,13 @@ where /// /// ### Example /// ```ignore -/// let a = Simd::from_array([1, 2, 3, 4, 32, 33, 34, 35]); -/// let b = Simd::from_array([2, 4, 6, 8, 10, 12, 14, 16]); +/// let a = u16x8::from_slice(simd, &[1, 2, 3, 4, 32, 33, 34, 35]); +/// let b = u16x8::from_slice(simd, &[2, 4, 6, 8, 10, 12, 14, 16]); /// let result = matrix_cmp_u16(a, b); -/// assert_eq!(result, Mask::from_array([false, true, false, true, false, false, false, false])); +/// assert_eq!(result.to_bitmask(), 0b0000_1010); /// ``` -#[inline] -// It would be nice to implement this for all supported lane counts -// However, we currently only support u16x8 so it's not really necessary -fn matrix_cmp_u16(a: Simd, b: Simd) -> Mask { +#[inline(always)] +fn matrix_cmp_u16(a: u16x8, b: u16x8) -> mask16x8 { a.simd_eq(b) | a.simd_eq(b.rotate_elements_left::<1>()) | a.simd_eq(b.rotate_elements_left::<2>()) @@ -441,39 +448,22 @@ fn matrix_cmp_u16(a: Simd, b: Simd) -> Mask { | a.simd_eq(b.rotate_elements_left::<7>()) } -use crate::bitmap::store::array_store::visitor::BinaryOperationVisitor; -use core::simd::Swizzle; - -/// Append to vectors to an imaginary 16 lane vector, shift the lanes right by 1, then -/// truncate to the low order 8 lanes -pub struct Shr1; -impl Swizzle<8> for Shr1 { - const INDEX: [usize; 8] = [15, 0, 1, 2, 3, 4, 5, 6]; -} - -/// Append to vectors to an imaginary 16 lane vector, shift the lanes right by 2, then -/// truncate to the low order 8 lanes -pub struct Shr2; -impl Swizzle<8> for Shr2 { - const INDEX: [usize; 8] = [14, 15, 0, 1, 2, 3, 4, 5]; -} - /// Assuming that a and b are sorted, returns an array of the sorted output. /// Developed originally for merge sort using SIMD instructions. /// Standard merge. See, e.g., Inoue and Taura, SIMD- and Cache-Friendly /// Algorithm for Sorting an Array of Structures -#[inline] -fn simd_merge_u16(a: Simd, b: Simd) -> [Simd; 2] { - let mut tmp: Simd = lanes_min_u16(a, b); - let mut max: Simd = lanes_max_u16(a, b); +#[inline(always)] +fn simd_merge_u16(a: u16x8, b: u16x8) -> [u16x8; 2] { + let mut tmp: u16x8 = a.min(b); + let mut max: u16x8 = a.max(b); tmp = tmp.rotate_elements_left::<1>(); - let mut min: Simd = lanes_min_u16(tmp, max); + let mut min: u16x8 = tmp.min(max); for _ in 0..6 { - max = lanes_max_u16(tmp, max); + max = tmp.max(max); tmp = min.rotate_elements_left::<1>(); - min = lanes_min_u16(tmp, max); + min = tmp.min(max); } - max = lanes_max_u16(tmp, max); + max = tmp.max(max); min = min.rotate_elements_left::<1>(); [min, max] } @@ -482,20 +472,12 @@ fn simd_merge_u16(a: Simd, b: Simd) -> [Simd; 2] { /// set to the front of the return vector, preserving their order. /// /// The values in the return vector after index bitmask.count_ones() is unspecified. -// Dynamic swizzle is only available for `u8`s. -// -// So we need to convert the `u16x8` to `u8x16`, and then swizzle it two lanes at a time. +// Dynamic swizzles operate on bytes, so the swizzle table moves the `u16` lanes two bytes at a time. // // e.g. if `bitmask` is `0b0101`, then swizzle the first two bytes (the first u16 lane) to the // first two positions, and the 5th and 6th bytes (the third u16 lane) to the next two positions. -// -// Note however: -// https://github.com/rust-lang/rust/blob/34097a38afc9efdedf776d3f1c84a190ff334886/library/portable-simd/crates/core_simd/src/swizzle_dyn.rs#L12-L15 -// > Note that the current implementation is selected during build-time -// > of the standard library, so `cargo build -Zbuild-std` may be necessary -// > to unlock better performance, especially for larger vectors. -// > A planned compiler improvement will enable using `#[target_feature]` instead. -pub fn swizzle_to_front(val: u16x8, bitmask: u8) -> u16x8 { +#[inline(always)] +pub fn swizzle_to_front(val: u16x8, bitmask: u8) -> u16x8 { static SWIZZLE_TABLE: [[u8; 16]; 256] = { let mut table = [[0; 16]; 256]; let mut n = 0usize; @@ -514,37 +496,94 @@ pub fn swizzle_to_front(val: u16x8, bitmask: u8) -> u16x8 { table }; - // Our swizzle table retains the order of the bytes in the 16 bit lanes, we can - // stick with native byte order as long as we convert back with native endianness too. - let val_convert: u8x16 = val.to_ne_bytes(); - let swizzle_idxs = u8x16::from_array(SWIZZLE_TABLE[bitmask as usize]); - - // Because the default `x86_64` target does not enable ssse3 (and without -Zbuild-std` - // std will not be compiled with it), use a manual swizzle with intrinsics so we can get - // reasonable performance without requiring the caller to use `-Zbuild-std`. - #[cfg(all(target_arch = "x86_64", any(target_feature = "ssse3", feature = "std")))] - { - let has_ssse3 = { - #[cfg(target_feature = "ssse3")] - { - true + // Our swizzle table retains the order of the bytes in the 16 bit lanes, + // so it works with either native byte order. + val.swizzle_dyn(SWIZZLE_TABLE[bitmask as usize]) +} + +#[cfg(test)] +mod test { + use super::*; + use crate::bitmap::store::array_store::visitor::VecWriter; + use alloc::vec::Vec; + use proptest::prelude::*; + + /// Checks the vectorized op produces the same result as the scalar op + fn check_op( + vector: impl Fn(&[u16], &[u16], &mut VecWriter), + scalar: impl Fn(&[u16], &[u16], &mut VecWriter), + lhs: &[u16], + rhs: &[u16], + ) { + let mut expected = VecWriter::new(lhs.len() + rhs.len()); + scalar(lhs, rhs, &mut expected); + let mut actual = VecWriter::new(lhs.len() + rhs.len()); + vector(lhs, rhs, &mut actual); + assert_eq!(actual.into_inner(), expected.into_inner()); + } + + /// Checks all vectorized ops produce the same results as the scalar ops + fn check_all(lhs: &[u16], rhs: &[u16]) { + check_op(or, scalar::or, lhs, rhs); + check_op(and, scalar::and, lhs, rhs); + check_op(xor, scalar::xor, lhs, rhs); + check_op(sub, scalar::sub, lhs, rhs); + } + + proptest! { + #[test] + fn vector_ops_match_scalar_dense( + lhs in prop::collection::btree_set(0u16..256, 0..200), + rhs in prop::collection::btree_set(0u16..256, 0..200), + ) { + check_all(&Vec::from_iter(lhs), &Vec::from_iter(rhs)); + } + + #[test] + fn vector_ops_match_scalar_sparse( + lhs in prop::collection::btree_set(any::(), 0..200), + rhs in prop::collection::btree_set(any::(), 0..200), + ) { + check_all(&Vec::from_iter(lhs), &Vec::from_iter(rhs)); + } + } + + #[test] + fn swizzle_to_front_keeps_masked_lanes_in_order() { + dispatch!(level(), simd => { + let values: [u16; 8] = [10, 11, 12, 13, 14, 15, 16, 17]; + let v = u16x8::from_slice(simd, &values); + for mask in 0..=255u8 { + let expected: Vec = + (0..8).filter(|i| mask & (1 << i) != 0).map(|i| values[i]).collect(); + let result = swizzle_to_front(v, mask); + assert_eq!(&result.as_slice()[..expected.len()], &expected[..]); } - #[cfg(not(target_feature = "ssse3"))] - { - // From above, `feature = std` must be true here, so we can do runtime detection - std::arch::is_x86_feature_detected!("ssse3") + }); + } + + #[test] + fn vector_ops_match_scalar_at_lane_boundaries() { + // Lengths around multiples of `LANES` exercise the scalar fallback, the vector loops, + // and the partial vector tails; the offsets give disjoint, interleaved, and equal inputs. + let lens = [0, 1, 7, 8, 9, 15, 16, 17, 24, 25]; + for len_l in lens { + for len_r in lens { + for offset in [0, 1, 4, 8, 64] { + let lhs: Vec = (0..len_l).map(|x| 2 * x).collect(); + let rhs: Vec = (0..len_r).map(|x| 2 * x + offset).collect(); + check_all(&lhs, &rhs); + } } - }; - if has_ssse3 { - use core::arch::x86_64::{__m128i, _mm_shuffle_epi8}; - let val_m128 = __m128i::from(val_convert); - let swizzle_m128 = __m128i::from(swizzle_idxs); - // SAFETY: We only are in this block if the target supports `ssse3` - let swizzled_m128 = unsafe { _mm_shuffle_epi8(val_m128, swizzle_m128) }; - return u16x8::from(swizzled_m128); } } - let swizzled: u8x16 = val_convert.swizzle_dyn(swizzle_idxs); - u16x8::from_ne_bytes(swizzled) + #[test] + fn matrix_cmp_finds_lanes_of_a_in_b() { + dispatch!(level(), simd => { + let a = u16x8::from_slice(simd, &[1, 2, 3, 4, 32, 33, 34, 35]); + let b = u16x8::from_slice(simd, &[2, 4, 6, 8, 10, 12, 14, 35]); + assert_eq!(matrix_cmp_u16(a, b).to_bitmask(), 0b1000_1010); + }); + } } diff --git a/roaring/src/bitmap/store/array_store/visitor.rs b/roaring/src/bitmap/store/array_store/visitor.rs index d7293d7b..c2cbf5c0 100644 --- a/roaring/src/bitmap/store/array_store/visitor.rs +++ b/roaring/src/bitmap/store/array_store/visitor.rs @@ -1,5 +1,6 @@ -#[cfg(feature = "simd")] use crate::bitmap::store::array_store::vector::swizzle_to_front; +use fearless_simd::{prelude::*, u16x8}; +use fearless_simd_macros::simd; #[cfg(not(feature = "std"))] use alloc::vec::Vec; @@ -12,8 +13,7 @@ use alloc::vec::Vec; /// their results. Future work can utilize the exiting algorithms to trivially implement /// computing the cardinality of an operation without materializng a new bitmap. pub trait BinaryOperationVisitor { - #[cfg(feature = "simd")] - fn visit_vector(&mut self, value: core::simd::u16x8, mask: u8); + fn visit_vector(&mut self, value: u16x8, mask: u8); fn visit_scalar(&mut self, value: u16); fn visit_slice(&mut self, values: &[u16]); } @@ -39,8 +39,8 @@ impl VecWriter { } impl BinaryOperationVisitor for VecWriter { - #[cfg(feature = "simd")] - fn visit_vector(&mut self, value: core::simd::u16x8, mask: u8) { + #[simd] + fn visit_vector(&mut self, value: u16x8, mask: u8) { let result = swizzle_to_front(value, mask); // This idiom is better than subslicing result, as it compiles down to an unaligned vector @@ -48,7 +48,7 @@ impl BinaryOperationVisitor for VecWriter { // A more straightforward, but unsafe way would be ptr::write_unaligned and Vec::set_len // Writing a vector at once is why the vectorized algorithms do not operate in place // first write the entire vector - self.vec.extend_from_slice(&result.as_array()[..]); + self.vec.extend_from_slice(result.as_slice()); // next truncate the masked out values self.vec.truncate(self.vec.len() - (result.len() - mask.count_ones() as usize)); } @@ -77,8 +77,7 @@ impl CardinalityCounter { } impl BinaryOperationVisitor for CardinalityCounter { - #[cfg(feature = "simd")] - fn visit_vector(&mut self, _value: core::simd::u16x8, mask: u8) { + fn visit_vector(&mut self, _value: u16x8, mask: u8) { self.count += mask.count_ones() as usize; } diff --git a/roaring/src/lib.rs b/roaring/src/lib.rs index d66f15f5..a01c421d 100644 --- a/roaring/src/lib.rs +++ b/roaring/src/lib.rs @@ -8,7 +8,6 @@ //! [roaring-paper]: https://arxiv.org/pdf/1402.6407v4 #![cfg_attr(not(feature = "std"), no_std)] -#![cfg_attr(feature = "simd", feature(portable_simd))] #![warn(missing_docs)] #![warn(unsafe_op_in_unsafe_fn)] #![warn(variant_size_differences)]