rustfs_erasure_codec/galois_8/
profile.rs

1#[cfg(feature = "std")]
2use core::sync::atomic::{AtomicUsize, Ordering};
3#[cfg(feature = "std")]
4use std::sync::OnceLock;
5
6#[cfg(feature = "std")]
7#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
8pub struct RustNeonProfileStats {
9    pub mul_calls: usize,
10    pub mul_xor_calls: usize,
11    pub total_bytes: usize,
12    pub vector_64b_chunks: usize,
13    pub vector_16b_chunks: usize,
14    pub tail_bytes: usize,
15    pub tail_calls: usize,
16    pub table_lookups: usize,
17}
18
19#[cfg(feature = "std")]
20impl RustNeonProfileStats {
21    pub fn saturating_sub(self, baseline: Self) -> Self {
22        Self {
23            mul_calls: self.mul_calls.saturating_sub(baseline.mul_calls),
24            mul_xor_calls: self.mul_xor_calls.saturating_sub(baseline.mul_xor_calls),
25            total_bytes: self.total_bytes.saturating_sub(baseline.total_bytes),
26            vector_64b_chunks: self
27                .vector_64b_chunks
28                .saturating_sub(baseline.vector_64b_chunks),
29            vector_16b_chunks: self
30                .vector_16b_chunks
31                .saturating_sub(baseline.vector_16b_chunks),
32            tail_bytes: self.tail_bytes.saturating_sub(baseline.tail_bytes),
33            tail_calls: self.tail_calls.saturating_sub(baseline.tail_calls),
34            table_lookups: self.table_lookups.saturating_sub(baseline.table_lookups),
35        }
36    }
37}
38
39#[cfg(feature = "std")]
40#[derive(Debug, Default)]
41pub(crate) struct RustNeonProfileMetrics {
42    mul_calls: AtomicUsize,
43    mul_xor_calls: AtomicUsize,
44    total_bytes: AtomicUsize,
45    vector_64b_chunks: AtomicUsize,
46    vector_16b_chunks: AtomicUsize,
47    tail_bytes: AtomicUsize,
48    tail_calls: AtomicUsize,
49    table_lookups: AtomicUsize,
50}
51
52#[cfg(feature = "std")]
53impl RustNeonProfileMetrics {
54    fn snapshot(&self) -> RustNeonProfileStats {
55        RustNeonProfileStats {
56            mul_calls: self.mul_calls.load(Ordering::Relaxed),
57            mul_xor_calls: self.mul_xor_calls.load(Ordering::Relaxed),
58            total_bytes: self.total_bytes.load(Ordering::Relaxed),
59            vector_64b_chunks: self.vector_64b_chunks.load(Ordering::Relaxed),
60            vector_16b_chunks: self.vector_16b_chunks.load(Ordering::Relaxed),
61            tail_bytes: self.tail_bytes.load(Ordering::Relaxed),
62            tail_calls: self.tail_calls.load(Ordering::Relaxed),
63            table_lookups: self.table_lookups.load(Ordering::Relaxed),
64        }
65    }
66
67    fn reset(&self) {
68        self.mul_calls.store(0, Ordering::Relaxed);
69        self.mul_xor_calls.store(0, Ordering::Relaxed);
70        self.total_bytes.store(0, Ordering::Relaxed);
71        self.vector_64b_chunks.store(0, Ordering::Relaxed);
72        self.vector_16b_chunks.store(0, Ordering::Relaxed);
73        self.tail_bytes.store(0, Ordering::Relaxed);
74        self.tail_calls.store(0, Ordering::Relaxed);
75        self.table_lookups.store(0, Ordering::Relaxed);
76    }
77
78    #[cfg(all(
79        feature = "simd-neon",
80        target_arch = "aarch64",
81        not(target_env = "msvc"),
82        not(any(target_os = "android", target_os = "ios"))
83    ))]
84    pub(crate) fn record_call(
85        &self,
86        is_xor: bool,
87        input_len: usize,
88        vector_64b_chunks: usize,
89        vector_16b_chunks: usize,
90        tail_bytes: usize,
91    ) {
92        if is_xor {
93            self.mul_xor_calls.fetch_add(1, Ordering::Relaxed);
94        } else {
95            self.mul_calls.fetch_add(1, Ordering::Relaxed);
96        }
97        self.total_bytes.fetch_add(input_len, Ordering::Relaxed);
98        self.vector_64b_chunks
99            .fetch_add(vector_64b_chunks, Ordering::Relaxed);
100        self.vector_16b_chunks
101            .fetch_add(vector_16b_chunks, Ordering::Relaxed);
102        if tail_bytes > 0 {
103            self.tail_calls.fetch_add(1, Ordering::Relaxed);
104            self.tail_bytes.fetch_add(tail_bytes, Ordering::Relaxed);
105        }
106        let lookups = vector_64b_chunks
107            .saturating_mul(8)
108            .saturating_add(vector_16b_chunks.saturating_mul(2));
109        self.table_lookups.fetch_add(lookups, Ordering::Relaxed);
110
111        // Test-only per-thread mirror of the increment (compiled out of every
112        // non-test build). Lets a test observe exactly its own thread's calls,
113        // immune to concurrent tests hitting this shared global aggregate.
114        #[cfg(test)]
115        capture::record(
116            is_xor,
117            input_len,
118            vector_64b_chunks,
119            vector_16b_chunks,
120            tail_bytes,
121        );
122    }
123}
124
125/// Test-only per-thread capture of the NEON profile increments.
126///
127/// The production metrics ([`RUST_NEON_PROFILE_METRICS`]) are a single global
128/// aggregate — correct for benchmarking, but exact-delta assertions on it are
129/// racy because every NEON `mul_slice` on any thread (e.g. concurrent codec
130/// tests) increments it. In test builds `record_call` additionally feeds this
131/// thread-local accumulator, so a test reads only its own thread's calls.
132#[cfg(all(
133    test,
134    feature = "simd-neon",
135    target_arch = "aarch64",
136    not(target_env = "msvc"),
137    not(any(target_os = "android", target_os = "ios"))
138))]
139pub(crate) mod capture {
140    use super::RustNeonProfileStats;
141    use core::cell::Cell;
142
143    const ZERO: RustNeonProfileStats = RustNeonProfileStats {
144        mul_calls: 0,
145        mul_xor_calls: 0,
146        total_bytes: 0,
147        vector_64b_chunks: 0,
148        vector_16b_chunks: 0,
149        tail_bytes: 0,
150        tail_calls: 0,
151        table_lookups: 0,
152    };
153
154    std::thread_local! {
155        static CAPTURE: Cell<RustNeonProfileStats> = const { Cell::new(ZERO) };
156    }
157
158    pub(crate) fn record(
159        is_xor: bool,
160        input_len: usize,
161        vector_64b_chunks: usize,
162        vector_16b_chunks: usize,
163        tail_bytes: usize,
164    ) {
165        CAPTURE.with(|cell| {
166            let mut s = cell.get();
167            if is_xor {
168                s.mul_xor_calls += 1;
169            } else {
170                s.mul_calls += 1;
171            }
172            s.total_bytes += input_len;
173            s.vector_64b_chunks += vector_64b_chunks;
174            s.vector_16b_chunks += vector_16b_chunks;
175            if tail_bytes > 0 {
176                s.tail_calls += 1;
177                s.tail_bytes += tail_bytes;
178            }
179            s.table_lookups += vector_64b_chunks
180                .saturating_mul(8)
181                .saturating_add(vector_16b_chunks.saturating_mul(2));
182            cell.set(s);
183        });
184    }
185
186    /// Reset this thread's accumulator to zero.
187    pub(crate) fn reset() {
188        CAPTURE.with(|cell| cell.set(ZERO));
189    }
190
191    /// This thread's accumulated increments since the last [`reset`].
192    pub(crate) fn snapshot() -> RustNeonProfileStats {
193        CAPTURE.with(|cell| cell.get())
194    }
195}
196
197#[cfg(feature = "std")]
198pub(crate) static RUST_NEON_PROFILE_METRICS: RustNeonProfileMetrics = RustNeonProfileMetrics {
199    mul_calls: AtomicUsize::new(0),
200    mul_xor_calls: AtomicUsize::new(0),
201    total_bytes: AtomicUsize::new(0),
202    vector_64b_chunks: AtomicUsize::new(0),
203    vector_16b_chunks: AtomicUsize::new(0),
204    tail_bytes: AtomicUsize::new(0),
205    tail_calls: AtomicUsize::new(0),
206    table_lookups: AtomicUsize::new(0),
207};
208
209#[cfg(feature = "std")]
210const RS_NEON_MUL_SLICE_XOR_UNROLL_ENV: &str = "RS_NEON_MUL_SLICE_XOR_UNROLL";
211#[cfg(feature = "std")]
212pub(crate) const RS_NEON_MUL_SLICE_XOR_SCHEDULE_ENV: &str = "RS_NEON_MUL_SLICE_XOR_SCHEDULE";
213
214#[cfg(feature = "std")]
215pub(crate) fn parse_rust_neon_xor_unroll(value: &str) -> Option<usize> {
216    match value {
217        "2" => Some(2),
218        "4" => Some(4),
219        _ => None,
220    }
221}
222
223#[cfg(feature = "std")]
224pub(crate) fn rust_neon_mul_slice_xor_unroll() -> usize {
225    static UNROLL: OnceLock<usize> = OnceLock::new();
226    *UNROLL.get_or_init(|| {
227        std::env::var(RS_NEON_MUL_SLICE_XOR_UNROLL_ENV)
228            .ok()
229            .as_deref()
230            .and_then(parse_rust_neon_xor_unroll)
231            .unwrap_or(4)
232    })
233}
234
235#[cfg(feature = "std")]
236pub(crate) fn rust_neon_mul_slice_xor_schedule_split() -> bool {
237    static SPLIT: OnceLock<bool> = OnceLock::new();
238    *SPLIT.get_or_init(|| {
239        std::env::var(RS_NEON_MUL_SLICE_XOR_SCHEDULE_ENV)
240            .ok()
241            .is_some_and(|value| value == "split")
242    })
243}
244
245#[cfg(feature = "std")]
246pub fn rust_neon_profile_stats() -> RustNeonProfileStats {
247    RUST_NEON_PROFILE_METRICS.snapshot()
248}
249
250#[cfg(feature = "std")]
251pub fn reset_rust_neon_profile_stats() {
252    RUST_NEON_PROFILE_METRICS.reset();
253}
rustfs_erasure_codec/galois_8/profile.rs

rustfs_erasure_codec/galois_8/
profile.rs
