oxide_regexp/unicode/
property.rs

1//! Checksum-pinned Unicode 17 property sets used by RegExp property escapes.
2//!
3//! The generated compact range and code-point sequence streams are
4//! materialized from the pinned QuickJS `libunicode.c` implementation. Product
5//! builds consume only these Rust arrays; the C helper under
6//! `apps/cli/tests/fixtures/` is a regeneration oracle.
7
8mod tables {
9    include!("generated/unicode/unicode_property_tables.rs");
10}
11
12/// Index of the alias group containing `name` in a generated name table.
13fn find(names: &str, name: &str) -> Option<usize> {
14    names
15        .split(';')
16        .position(|group| group.split(',').any(|alias| alias == name))
17}
18
19fn read_varint(data: &[u8], position: &mut usize) -> u32 {
20    let mut value = 0;
21    let mut shift = 0;
22    loop {
23        let byte = data[*position];
24        *position += 1;
25        value |= u32::from(byte & 0x7f) << shift;
26        if byte < 0x80 {
27            return value;
28        }
29        shift += 7;
30    }
31}
32
33/// Half-open `(start, end)` code-point ranges of the union of the selected
34/// generated sets. Each set is ascending and disjoint; a union of several
35/// sets is neither sorted nor merged.
36#[derive(Clone, Debug)]
37pub struct PropertyRanges {
38    sets: &'static [u32],
39    mask: u64,
40    position: usize,
41    remaining: u32,
42    end: u32,
43}
44
45impl PropertyRanges {
46    fn new(sets: &'static [u32], mask: u64) -> Self {
47        Self {
48            sets,
49            mask,
50            position: 0,
51            remaining: 0,
52            end: 0,
53        }
54    }
55}
56
57impl Iterator for PropertyRanges {
58    type Item = (u32, u32);
59
60    fn next(&mut self) -> Option<(u32, u32)> {
61        while self.remaining == 0 {
62            if self.mask == 0 {
63                return None;
64            }
65            let set = self.mask.trailing_zeros() as usize;
66            self.mask &= self.mask - 1;
67            self.position = self.sets[set] as usize;
68            self.remaining = read_varint(&tables::RANGE_DATA, &mut self.position);
69            self.end = 0;
70        }
71        self.remaining -= 1;
72        let start = self.end + read_varint(&tables::RANGE_DATA, &mut self.position);
73        self.end = start + read_varint(&tables::RANGE_DATA, &mut self.position);
74        Some((start, self.end))
75    }
76}
77
78pub fn general_category(name: &str) -> Option<PropertyRanges> {
79    let mask = tables::GENERAL_CATEGORY_MASKS[find(tables::GENERAL_CATEGORY_NAMES, name)?];
80    Some(PropertyRanges::new(
81        &tables::GENERAL_CATEGORY_SETS,
82        u64::from(mask),
83    ))
84}
85
86pub fn script(name: &str, extensions: bool) -> Option<PropertyRanges> {
87    let sets = if extensions {
88        &tables::SCRIPT_EXTENSIONS_SETS
89    } else {
90        &tables::SCRIPT_SETS
91    };
92    let index = find(tables::SCRIPT_NAMES, name)?;
93    Some(PropertyRanges::new(std::slice::from_ref(&sets[index]), 1))
94}
95
96pub fn binary_property(name: &str) -> Option<PropertyRanges> {
97    let index = find(tables::BINARY_PROPERTY_NAMES, name)?;
98    Some(PropertyRanges::new(
99        std::slice::from_ref(&tables::BINARY_PROPERTY_SETS[index]),
100        1,
101    ))
102}
103
104/// Visit every code-point sequence of a sequence property; `None` when the
105/// name is unknown. Sequences of a union are visited property by property.
106pub fn sequence_property(name: &str, mut visit: impl FnMut(&[u32])) -> Option<()> {
107    let index = find(tables::SEQUENCE_PROPERTY_NAMES, name)?;
108    // RGI_Emoji, the one name past the stored sets, is the union of them all.
109    let sets = tables::SEQUENCE_PROPERTY_SETS
110        .get(index)
111        .map_or(&tables::SEQUENCE_PROPERTY_SETS[..], std::slice::from_ref);
112    let data = &tables::SEQUENCE_DATA;
113    for &offset in sets {
114        let mut position = offset as usize;
115        let mut sequence = [0_u32; 16];
116        let mut previous_len = 0;
117        for _ in 0..read_varint(data, &mut position) {
118            let len = usize::from(data[position]);
119            position += 1;
120            for index in 0..len {
121                let base = match index {
122                    _ if index < previous_len => sequence[index],
123                    0 => 0,
124                    _ => sequence[index - 1],
125                };
126                let delta = read_varint(data, &mut position);
127                let delta = (delta >> 1) as i32 ^ -((delta & 1) as i32);
128                sequence[index] = base.wrapping_add_signed(delta);
129            }
130            previous_len = len;
131            visit(&sequence[..len]);
132        }
133    }
134    Some(())
135}
136
137#[cfg(test)]
138mod tests {
139    use std::collections::HashSet;
140
141    use super::{
142        PropertyRanges, binary_property, general_category, script, sequence_property, tables,
143    };
144
145    fn sequences(name: &str) -> Vec<Vec<u32>> {
146        let mut sequences = Vec::new();
147        sequence_property(name, |sequence| sequences.push(sequence.to_vec())).unwrap();
148        sequences
149    }
150
151    fn assert_valid_set(ranges: PropertyRanges) {
152        let mut previous_end = 0;
153        for (index, (start, end)) in ranges.enumerate() {
154            assert!(start < end && end <= 0x11_0000);
155            assert!(index == 0 || start >= previous_end);
156            previous_end = end;
157        }
158    }
159
160    fn names(table: &str) -> Vec<&str> {
161        table
162            .split(';')
163            .flat_map(|group| group.split(','))
164            .collect()
165    }
166
167    fn sorted(ranges: PropertyRanges) -> Vec<(u32, u32)> {
168        merged(ranges.collect())
169    }
170
171    fn merged(mut ranges: Vec<(u32, u32)>) -> Vec<(u32, u32)> {
172        let mut merged: Vec<(u32, u32)> = Vec::new();
173        ranges.sort_unstable();
174        for (start, end) in ranges {
175            match merged.last_mut() {
176                Some(last) if start <= last.1 => last.1 = last.1.max(end),
177                _ => merged.push((start, end)),
178            }
179        }
180        merged
181    }
182
183    #[test]
184    fn generated_property_catalog_matches_pinned_quickjs_shape() {
185        assert_eq!(tables::GENERAL_CATEGORY_NAMES.split(';').count(), 38);
186        assert_eq!(tables::SCRIPT_NAMES.split(';').count(), 176);
187        assert_eq!(tables::BINARY_PROPERTY_NAMES.split(';').count(), 55);
188        assert_eq!(names(tables::GENERAL_CATEGORY_NAMES).len(), 80);
189        assert_eq!(names(tables::SCRIPT_NAMES).len(), 354);
190        assert_eq!(names(tables::BINARY_PROPERTY_NAMES).len(), 102);
191
192        for name in names(tables::GENERAL_CATEGORY_NAMES) {
193            let ranges = general_category(name).unwrap();
194            if ranges.mask.count_ones() == 1 {
195                assert_valid_set(ranges);
196            }
197        }
198        for name in names(tables::SCRIPT_NAMES) {
199            assert_valid_set(script(name, false).unwrap());
200            assert_valid_set(script(name, true).unwrap());
201        }
202        for name in names(tables::BINARY_PROPERTY_NAMES) {
203            assert_valid_set(binary_property(name).unwrap());
204        }
205        // The base general categories partition the code space.
206        assert_eq!(sorted(binary_property("Any").unwrap()), [(0, 0x11_0000)]);
207        let all = PropertyRanges::new(&tables::GENERAL_CATEGORY_SETS, (1 << 30) - 1);
208        assert_eq!(sorted(all.clone()), [(0, 0x11_0000)]);
209        assert_eq!(all.map(|(start, end)| end - start).sum::<u32>(), 0x11_0000);
210    }
211
212    #[test]
213    fn generated_sequence_property_catalog_matches_pinned_quickjs_shape() {
214        let names = names(tables::SEQUENCE_PROPERTY_NAMES);
215        assert_eq!(
216            names,
217            [
218                "Basic_Emoji",
219                "Emoji_Keycap_Sequence",
220                "RGI_Emoji_Modifier_Sequence",
221                "RGI_Emoji_Flag_Sequence",
222                "RGI_Emoji_Tag_Sequence",
223                "RGI_Emoji_ZWJ_Sequence",
224                "RGI_Emoji",
225            ]
226        );
227        let counts = names
228            .iter()
229            .map(|name| sequences(name).len())
230            .collect::<Vec<_>>();
231        assert_eq!(counts, [1400, 12, 670, 259, 3, 1614, 3958]);
232
233        let all = sequences("RGI_Emoji");
234        let mut unique = HashSet::with_capacity(all.len());
235        for sequence in &all {
236            assert!(!sequence.is_empty());
237            assert!(sequence.iter().all(|code_point| *code_point < 0x11_0000));
238            assert!(unique.insert(sequence.clone()), "duplicate {sequence:x?}");
239        }
240        assert_eq!(
241            sequences("Emoji_Keycap_Sequence")[0],
242            [0x23, 0xfe0f, 0x20e3]
243        );
244        assert!(sequence_property("basic_emoji", |_| {}).is_none());
245    }
246
247    #[test]
248    fn aliases_are_exact_and_share_the_expected_sets() {
249        let gc = |name| general_category(name).map(sorted);
250        assert_eq!(gc("Letter"), gc("L"));
251        assert_eq!(gc("Uppercase_Letter"), gc("Lu"));
252        assert!(gc("letter").is_none());
253        assert_eq!(gc("Nd").unwrap()[0], (0x30, 0x3a));
254
255        let sc = |name, extensions| script(name, extensions).map(sorted);
256        assert_eq!(sc("Latin", false), sc("Latn", false));
257        assert_eq!(sc("Hiragana", true), sc("Hira", true));
258        assert_ne!(sc("Hiragana", false), sc("Hiragana", true));
259
260        let binary = |name| binary_property(name).map(sorted);
261        assert_eq!(binary("ASCII_Hex_Digit"), binary("AHex"));
262        assert_eq!(binary("Emoji_Presentation"), binary("EPres"));
263        assert_eq!(binary("ASCII").unwrap(), [(0, 0x80)]);
264        assert!(binary("RGI_Emoji").is_none());
265        assert!(binary("ID_Compat_Math_Start").is_none());
266    }
267}
oxide_regexp/unicode/property.rs

oxide_regexp/unicode/
property.rs
