simd_brotli/enc/
vectorization.rs

1//! Fixed-width vector storage for the encoder.
2//!
3//! The types here are plain arrays: `Default` + `Copy`, so they can live in the
4//! encoder's allocator-backed slices and be handed to `Allocator<T>`. They carry no
5//! arithmetic of their own. Math is done on [`fearless_simd`] vectors instead: inside a
6//! `dispatch!` region, [`Mem256f::to_simd`] (and friends) loads a register and
7//! [`Mem256f::from_simd`] stores it back.
8
9use core::ops::{Index, IndexMut};
10use core::slice::SliceIndex;
11
12use fearless_simd::{Level, Simd, SimdBase, SimdInto, f32x8, i16x16, i32x8, u32x8};
13
14/// The instruction set the vectorized encoder paths run on.
15///
16/// Detected at runtime where the platform allows it (`std` builds, wasm), otherwise the
17/// best level this crate was compiled for. The `std` answer is cached: probing costs a
18/// dozen feature tests, and callers such as [`crate::enc::bit_cost::BrotliPopulationCost`]
19/// dispatch once per histogram, deep inside the clustering loops.
20#[cfg(feature = "std")]
21#[inline]
22pub fn detect_level() -> Level {
23    static LEVEL: std::sync::LazyLock<Level> = std::sync::LazyLock::new(|| Level::try_detect().unwrap_or_else(Level::baseline));
24    *LEVEL
25}
26
27/// See the `std` variant above; without `std` there is nothing to cache, as
28/// `try_detect` cannot probe the CPU and always resolves to the compiled-for level.
29#[cfg(not(feature = "std"))]
30#[inline]
31pub fn detect_level() -> Level {
32    Level::try_detect().unwrap_or_else(Level::baseline)
33}
34
35/// The smallest lane of `v`, folded in `log2(8)` steps.
36#[inline(always)]
37pub fn min_lane_f32x8<S: Simd>(v: f32x8<S>) -> f32 {
38    let v = v.min(v.slide::<4>(v));
39    let v = v.min(v.slide::<2>(v));
40    let v = v.min(v.slide::<1>(v));
41    v[0]
42}
43
44/// The smallest lane of `v`, folded in `log2(8)` steps.
45#[inline(always)]
46pub fn min_lane_u32x8<S: Simd>(v: u32x8<S>) -> u32 {
47    let v = v.min(v.slide::<4>(v));
48    let v = v.min(v.slide::<2>(v));
49    let v = v.min(v.slide::<1>(v));
50    v[0]
51}
52
53macro_rules! define_vector {
54    ($(#[$attr:meta])* $name:ident, $elem:ty, $lanes:literal, $simd:ident) => {
55        $(#[$attr])*
56        #[derive(Default, Copy, Clone, Debug)]
57        pub struct $name([$elem; $lanes]);
58
59        impl $name {
60            /// Load the lanes into a SIMD register.
61            #[inline(always)]
62            pub fn to_simd<S: Simd>(self, simd: S) -> $simd<S> {
63                self.0.simd_into(simd)
64            }
65
66            /// Store a SIMD register back into plain memory.
67            #[inline(always)]
68            pub fn from_simd<S: Simd>(value: $simd<S>) -> Self {
69                Self(value.into())
70            }
71        }
72
73        impl From<[$elem; $lanes]> for $name {
74            #[inline(always)]
75            fn from(value: [$elem; $lanes]) -> Self {
76                Self(value)
77            }
78        }
79
80        impl<I: SliceIndex<[$elem]>> Index<I> for $name {
81            type Output = I::Output;
82
83            #[inline(always)]
84            fn index(&self, index: I) -> &Self::Output {
85                &self.0[index]
86            }
87        }
88
89        impl<I: SliceIndex<[$elem]>> IndexMut<I> for $name {
90            #[inline(always)]
91            fn index_mut(&mut self, index: I) -> &mut Self::Output {
92                &mut self.0[index]
93            }
94        }
95    };
96}
97
98define_vector!(Mem256f, f32, 8, f32x8);
99define_vector!(Mem256i, i32, 8, i32x8);
100define_vector!(Mem16x16, i16, 16, i16x16);
101define_vector!(
102    /// A 16-bucket probability distribution.
103    ///
104    /// Same shape as [`Mem16x16`], but deliberately a separate type: `BrotliAlloc`
105    /// requires `Allocator<PDF>` and `Allocator<s16>` as distinct bounds, so the two
106    /// cannot be aliases of each other. Re-exported as [`crate::enc::pdf::PDF`].
107    PDF,
108    i16,
109    16,
110    i16x16
111);
112
113pub type v256 = Mem256f;
114pub type v256i = Mem256i;
simd_brotli/enc/vectorization.rs

simd_brotli/enc/
vectorization.rs
