simd_brotli/enc/
vectorization.rs

1//! Fixed-width vector storage for the encoder.
2//!
3//! The types here are plain arrays: `Default` + `Copy`, so they can live in the
4//! encoder's allocator-backed slices and be handed to `Allocator<T>`. They carry no
5//! arithmetic of their own. Math is done on [`fearless_simd`] vectors instead: inside a
6//! `dispatch!` region, [`Mem256f::to_simd`] (and friends) loads a register and
7//! [`Mem256f::from_simd`] stores it back.
8
9use core::ops::{Index, IndexMut};
10use core::slice::SliceIndex;
11
12use fearless_simd::{
13    Level, Simd, SimdBase, SimdFloat, SimdInt, SimdInto, f32x8, i16x16, i32x8, u32x8,
14};
15
16/// The instruction set the vectorized encoder paths run on.
17///
18/// Detected at runtime where the platform allows it (`std` builds, wasm), otherwise the
19/// best level this crate was compiled for. The `std` answer is cached: probing costs a
20/// dozen feature tests, and callers such as [`crate::enc::bit_cost::BrotliPopulationCost`]
21/// dispatch once per histogram, deep inside the clustering loops.
22#[cfg(feature = "std")]
23#[inline]
24pub fn detect_level() -> Level {
25    static LEVEL: std::sync::OnceLock<Level> = std::sync::OnceLock::new();
26    *LEVEL.get_or_init(|| Level::try_detect().unwrap_or_else(Level::baseline))
27}
28
29/// See the `std` variant above; without `std` there is nothing to cache, as
30/// `try_detect` cannot probe the CPU and always resolves to the compiled-for level.
31#[cfg(not(feature = "std"))]
32#[inline]
33pub fn detect_level() -> Level {
34    Level::try_detect().unwrap_or_else(Level::baseline)
35}
36
37/// The smallest lane of `v`, folded in `log2(8)` steps.
38#[inline(always)]
39pub fn min_lane_f32x8<S: Simd>(v: f32x8<S>) -> f32 {
40    let v = v.min(v.slide::<4>(v));
41    let v = v.min(v.slide::<2>(v));
42    let v = v.min(v.slide::<1>(v));
43    v[0]
44}
45
46/// The smallest lane of `v`, folded in `log2(8)` steps.
47#[inline(always)]
48pub fn min_lane_u32x8<S: Simd>(v: u32x8<S>) -> u32 {
49    let v = v.min(v.slide::<4>(v));
50    let v = v.min(v.slide::<2>(v));
51    let v = v.min(v.slide::<1>(v));
52    v[0]
53}
54
55macro_rules! define_vector {
56    ($(#[$attr:meta])* $name:ident, $elem:ty, $lanes:literal, $simd:ident) => {
57        $(#[$attr])*
58        #[derive(Default, Copy, Clone, Debug)]
59        pub struct $name([$elem; $lanes]);
60
61        impl $name {
62            /// Load the lanes into a SIMD register.
63            #[inline(always)]
64            pub fn to_simd<S: Simd>(self, simd: S) -> $simd<S> {
65                self.0.simd_into(simd)
66            }
67
68            /// Store a SIMD register back into plain memory.
69            #[inline(always)]
70            pub fn from_simd<S: Simd>(value: $simd<S>) -> Self {
71                Self(value.into())
72            }
73        }
74
75        impl From<[$elem; $lanes]> for $name {
76            #[inline(always)]
77            fn from(value: [$elem; $lanes]) -> Self {
78                Self(value)
79            }
80        }
81
82        impl<I: SliceIndex<[$elem]>> Index<I> for $name {
83            type Output = I::Output;
84
85            #[inline(always)]
86            fn index(&self, index: I) -> &Self::Output {
87                &self.0[index]
88            }
89        }
90
91        impl<I: SliceIndex<[$elem]>> IndexMut<I> for $name {
92            #[inline(always)]
93            fn index_mut(&mut self, index: I) -> &mut Self::Output {
94                &mut self.0[index]
95            }
96        }
97    };
98}
99
100define_vector!(Mem256f, f32, 8, f32x8);
101define_vector!(Mem256i, i32, 8, i32x8);
102define_vector!(Mem16x16, i16, 16, i16x16);
103define_vector!(
104    /// A 16-bucket probability distribution.
105    ///
106    /// Same shape as [`Mem16x16`], but deliberately a separate type: `BrotliAlloc`
107    /// requires `Allocator<PDF>` and `Allocator<s16>` as distinct bounds, so the two
108    /// cannot be aliases of each other. Re-exported as [`crate::enc::pdf::PDF`].
109    PDF,
110    i16,
111    16,
112    i16x16
113);
114
115pub type v256 = Mem256f;
116pub type v256i = Mem256i;
simd_brotli/enc/vectorization.rs

simd_brotli/enc/
vectorization.rs
