simd-normalizer 0.1.1

SIMD-accelerated Unicode normalization (NFC, NFD, NFKC, NFKD)
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140

//! Cross-backend consistency tests: verify that SSE4.2, AVX2, and AVX-512BW
//! produce identical bitmasks for the same input.

#[cfg(test)]
mod tests {
    use crate::simd::x86_64::{avx2, avx512, sse42};

    /// Generate a deterministic but non-trivial 64-byte test vector.
    fn make_test_data(seed: u8) -> [u8; 64] {
        let mut data = [0u8; 64];
        for (i, byte) in data.iter_mut().enumerate() {
            *byte = seed.wrapping_mul(i as u8).wrapping_add(i as u8 ^ seed);
        }
        data
    }

    #[test]
    fn sse42_vs_avx2_consistency() {
        if !std::is_x86_feature_detected!("sse4.2") || !std::is_x86_feature_detected!("avx2") {
            return;
        }
        for seed in 0..=255u8 {
            let data = make_test_data(seed);
            for bound in [0x00, 0x40, 0x80, 0xC0, 0xFF] {
                let mask_sse = unsafe { sse42::scan_chunk(data.as_ptr(), bound) };
                let mask_avx = unsafe { avx2::scan_chunk(data.as_ptr(), bound) };
                assert_eq!(
                    mask_sse, mask_avx,
                    "SSE4.2 vs AVX2 mismatch: seed={seed}, bound=0x{bound:02X}"
                );
            }
        }
    }

    #[test]
    fn sse42_vs_avx512_consistency() {
        if !std::is_x86_feature_detected!("sse4.2") || !std::is_x86_feature_detected!("avx512bw") {
            return;
        }
        for seed in 0..=255u8 {
            let data = make_test_data(seed);
            for bound in [0x00, 0x40, 0x80, 0xC0, 0xFF] {
                let mask_sse = unsafe { sse42::scan_chunk(data.as_ptr(), bound) };
                let mask_512 = unsafe { avx512::scan_chunk(data.as_ptr(), bound) };
                assert_eq!(
                    mask_sse, mask_512,
                    "SSE4.2 vs AVX-512BW mismatch: seed={seed}, bound=0x{bound:02X}"
                );
            }
        }
    }

    #[test]
    fn prefetch_vs_plain_sse42() {
        if !std::is_x86_feature_detected!("sse4.2") {
            return;
        }
        for seed in 0..=255u8 {
            let data = make_test_data(seed);
            let dummy = data.as_ptr();
            let plain = unsafe { sse42::scan_chunk(data.as_ptr(), 0x80) };
            let pf = unsafe { sse42::scan_and_prefetch(data.as_ptr(), dummy, dummy, 0x80) };
            assert_eq!(plain, pf, "SSE4.2 prefetch mismatch at seed={seed}");
        }
    }

    #[test]
    fn prefetch_vs_plain_avx2() {
        if !std::is_x86_feature_detected!("avx2") {
            return;
        }
        for seed in 0..=255u8 {
            let data = make_test_data(seed);
            let dummy = data.as_ptr();
            let plain = unsafe { avx2::scan_chunk(data.as_ptr(), 0x80) };
            let pf = unsafe { avx2::scan_and_prefetch(data.as_ptr(), dummy, dummy, 0x80) };
            assert_eq!(plain, pf, "AVX2 prefetch mismatch at seed={seed}");
        }
    }

    #[test]
    fn prefetch_vs_plain_avx512() {
        if !std::is_x86_feature_detected!("avx512bw") {
            return;
        }
        for seed in 0..=255u8 {
            let data = make_test_data(seed);
            let dummy = data.as_ptr();
            let plain = unsafe { avx512::scan_chunk(data.as_ptr(), 0x80) };
            let pf = unsafe { avx512::scan_and_prefetch(data.as_ptr(), dummy, dummy, 0x80) };
            assert_eq!(plain, pf, "AVX-512BW prefetch mismatch at seed={seed}");
        }
    }

    #[test]
    fn all_backends_against_scalar_reference() {
        if !std::is_x86_feature_detected!("sse4.2") {
            return;
        }

        fn scalar_scan(data: &[u8; 64], bound: u8) -> u64 {
            let mut mask = 0u64;
            for (i, &byte) in data.iter().enumerate() {
                if byte >= bound {
                    mask |= 1u64 << i;
                }
            }
            mask
        }

        for seed in 0..=255u8 {
            let data = make_test_data(seed);
            for bound in [0x00, 0x01, 0x7F, 0x80, 0xBF, 0xC0, 0xFE, 0xFF] {
                let expected = scalar_scan(&data, bound);

                let sse = unsafe { sse42::scan_chunk(data.as_ptr(), bound) };
                assert_eq!(
                    sse, expected,
                    "SSE4.2 vs scalar: seed={seed}, bound=0x{bound:02X}"
                );

                if std::is_x86_feature_detected!("avx2") {
                    let avx = unsafe { avx2::scan_chunk(data.as_ptr(), bound) };
                    assert_eq!(
                        avx, expected,
                        "AVX2 vs scalar: seed={seed}, bound=0x{bound:02X}"
                    );
                }

                if std::is_x86_feature_detected!("avx512bw") {
                    let a512 = unsafe { avx512::scan_chunk(data.as_ptr(), bound) };
                    assert_eq!(
                        a512, expected,
                        "AVX-512BW vs scalar: seed={seed}, bound=0x{bound:02X}"
                    );
                }
            }
        }
    }
}