servo_base/
text.rs

1/* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
4
5use std::iter::Sum;
6use std::ops::{Add, AddAssign, Range, Sub, SubAssign};
7
8use malloc_size_of_derive::MallocSizeOf;
9
10pub use crate::unicode_block::{UnicodeBlock, UnicodeBlockMethod};
11
12pub fn is_bidi_control(c: char) -> bool {
13    matches!(c, '\u{202A}'..='\u{202E}' | '\u{2066}'..='\u{2069}' | '\u{200E}' | '\u{200F}' | '\u{061C}')
14}
15
16pub fn unicode_plane(codepoint: char) -> u32 {
17    (codepoint as u32) >> 16
18}
19
20pub fn is_cjk(codepoint: char) -> bool {
21    if let Some(
22        UnicodeBlock::CJKRadicalsSupplement |
23        UnicodeBlock::KangxiRadicals |
24        UnicodeBlock::IdeographicDescriptionCharacters |
25        UnicodeBlock::CJKSymbolsandPunctuation |
26        UnicodeBlock::Hiragana |
27        UnicodeBlock::Katakana |
28        UnicodeBlock::Bopomofo |
29        UnicodeBlock::HangulCompatibilityJamo |
30        UnicodeBlock::Kanbun |
31        UnicodeBlock::BopomofoExtended |
32        UnicodeBlock::CJKStrokes |
33        UnicodeBlock::KatakanaPhoneticExtensions |
34        UnicodeBlock::EnclosedCJKLettersandMonths |
35        UnicodeBlock::CJKCompatibility |
36        UnicodeBlock::CJKUnifiedIdeographsExtensionA |
37        UnicodeBlock::YijingHexagramSymbols |
38        UnicodeBlock::CJKUnifiedIdeographs |
39        UnicodeBlock::CJKCompatibilityIdeographs |
40        UnicodeBlock::CJKCompatibilityForms |
41        UnicodeBlock::HalfwidthandFullwidthForms,
42    ) = codepoint.block()
43    {
44        return true;
45    }
46
47    // https://en.wikipedia.org/wiki/Plane_(Unicode)#Supplementary_Ideographic_Plane
48    // https://en.wikipedia.org/wiki/Plane_(Unicode)#Tertiary_Ideographic_Plane
49    unicode_plane(codepoint) == 2 || unicode_plane(codepoint) == 3
50}
51
52/// Equivalent to either `Range`, `RangeTo`, `RangeFrom`, or `RangeFull`
53#[derive(Clone, Copy)]
54pub struct RangeAny<T> {
55    /// `None` means zero
56    pub start: Option<T>,
57    /// `None` means the full available length
58    pub end: Option<T>,
59}
60
61impl<T> RangeAny<T> {
62    /// Apply `Option::map` to each bound of this range
63    pub fn map<U>(self, f: impl Fn(T) -> U + Copy) -> RangeAny<U> {
64        let Self { start, end } = self;
65        RangeAny {
66            start: start.map(f),
67            end: end.map(f),
68        }
69    }
70
71    /// Returns the intersection of two ranges, if it is non-empty
72    pub fn intersect(self, other: Self) -> Option<Self>
73    where
74        T: Ord,
75    {
76        // TODO: https://github.com/rust-lang/rust/issues/144273
77        // let start = a.start.reduce(b.start, std::cmp::max);
78        // let end = a.end.reduce(b.end, std::cmp::min);
79        let start = match (self.start, other.start) {
80            (None, None) => None,
81            (None, Some(b)) => Some(b),
82            (Some(a), None) => Some(a),
83            (Some(a), Some(b)) => Some(a.max(b)),
84        };
85        let end = match (self.end, other.end) {
86            (None, None) => None,
87            (None, Some(b)) => Some(b),
88            (Some(a), None) => Some(a),
89            (Some(a), Some(b)) => Some(a.min(b)),
90        };
91        if start
92            .as_ref()
93            .is_none_or(|start| end.as_ref().is_none_or(|end| start < end))
94        {
95            Some(RangeAny { start, end })
96        } else {
97            // `max()..min()` producing a "backwards" range means the intersection is empty
98            None
99        }
100    }
101}
102
103impl<T> From<Range<T>> for RangeAny<T> {
104    fn from(value: Range<T>) -> Self {
105        Self {
106            start: Some(value.start),
107            end: Some(value.end),
108        }
109    }
110}
111
112macro_rules! unicode_length_type {
113    ($( #[$doc:meta] )+ $type_name:ident) => {
114        $( #[$doc] )+
115        #[derive(Clone, Copy, Debug, Default, Eq, MallocSizeOf, Ord, PartialEq, PartialOrd)]
116        pub struct $type_name(pub usize);
117
118        impl $type_name {
119            pub fn zero() -> Self {
120                Self(0)
121            }
122
123            pub fn one() -> Self {
124                Self(1)
125            }
126
127            pub fn saturating_sub(self, value: Self) -> Self {
128                Self(self.0.saturating_sub(value.0))
129            }
130        }
131
132        impl From<u32> for $type_name {
133            fn from(value: u32) -> Self {
134                Self(value as usize)
135            }
136        }
137
138        impl From<isize> for $type_name {
139            fn from(value: isize) -> Self {
140                Self(value as usize)
141            }
142        }
143
144        impl Add for $type_name {
145            type Output = Self;
146            fn add(self, other: Self) -> Self {
147                Self(self.0 + other.0)
148            }
149        }
150
151        impl AddAssign for $type_name {
152            fn add_assign(&mut self, other: Self) {
153                *self = Self(self.0 + other.0)
154            }
155        }
156
157        impl Sub for $type_name {
158            type Output = Self;
159            fn sub(self, value: Self) -> Self {
160                Self(self.0 - value.0)
161            }
162        }
163
164        impl SubAssign for $type_name {
165            fn sub_assign(&mut self, other: Self) {
166                *self = Self(self.0 - other.0)
167            }
168        }
169
170        impl Sum for $type_name {
171            fn sum<I: Iterator<Item = Self>>(iter: I) -> Self {
172                iter.fold(Self::zero(), |a, b| Self(a.0 + b.0))
173            }
174        }
175    };
176}
177
178unicode_length_type! {
179    /// A length or offset counted in 8-bit code units (bytes) in an UTF-8 string.
180    /// This type is used to more reliable work with lengths or offsets in different encodings.
181    Utf8CodeUnits
182}
183
184unicode_length_type! {
185    /// A length or offset counted in 16-bit code units in an UTF-16 string.
186    /// This type is used to more reliable work with lengths or offsets in different encodings.
187    Utf16CodeUnits
188}
189
190unicode_length_type! {
191    /// A length or offset counted in 32-bit code units in UTF-32.
192    /// This is the same as counting Rust `char`s, Unicode scalar values, or Unicode code points.
193    /// This type is used to more reliable work with lengths or offsets in different encodings.
194    Utf32CodeUnits
195}
196
197unicode_length_type! {
198    /// A length or offset counted in 32-bit code units in UTF-32 or a node offset in a container
199    /// node counted in previous siblings.
200    Utf32CodeUnitsOrNodeOffset
201}
202
203impl Utf16CodeUnits {
204    pub fn length_of(string: &str) -> Self {
205        Self(string.bytes().map(len_utf16_for_utf8_byte).sum())
206
207        // TODO: after upgrading to a Rust version (1.99?) that includes that PR,
208        // replace the above with:
209
210        // // `EncodeUtf16::count` is optimized in https://github.com/rust-lang/rust/pull/159467
211        // Self(string.encode_utf16().count())
212    }
213
214    pub fn to_utf32_code_units_in(self, string: &str) -> Utf32CodeUnits {
215        let mut current_utf16_offset = Utf16CodeUnits(0);
216        let mut current_utf32_offset = Utf32CodeUnits(0);
217        for utf8_byte in string.bytes() {
218            if current_utf16_offset >= self {
219                break;
220            }
221            increment_offsets_for_utf8_byte(
222                utf8_byte,
223                &mut current_utf16_offset,
224                &mut current_utf32_offset,
225            );
226        }
227        current_utf32_offset
228    }
229}
230
231fn len_utf16_for_utf8_byte(byte: u8) -> usize {
232    if byte < 0b1000_0000 {
233        // 0b0xxx_xxxx: ASCII-compatible U+0000 to U+007F
234        1
235    } else if byte < 0b1100_0000 {
236        // 0b10xx_xxxx: UTF-8 continuation byte, already accounted for by its non-continuation byte
237        0
238    } else if byte < 0b1111_0000 {
239        // 0b110x_xxxx: start of a 2-byte UTF-8 sequence for U+0080 to U+07FF
240        // 0b1110_xxxx: start of a 3-byte UTF-8 sequence for U+0800 to U+FFFF
241        1
242    } else {
243        // 0b1111_0xxx: start of a 4-byte UTF-8 sequence for U+010000 to U+10FFFF
244        // This is exactly the range encoded as a surrogate pair in UTF-16
245        //
246        // 0b1111_1xxx: would fall here but never occurs in valid UTF-8
247        2
248    }
249}
250
251fn increment_offsets_for_utf8_byte(
252    utf8_byte: u8,
253    utf16_offset: &mut Utf16CodeUnits,
254    utf32_offset: &mut Utf32CodeUnits,
255) {
256    let len_utf16 = len_utf16_for_utf8_byte(utf8_byte);
257    utf16_offset.0 += len_utf16;
258    // `len_utf16 != 0` means this byte is the first byte of the UTF-8 byte sequence
259    // for one `char` /  UTF-32 code unit
260    utf32_offset.0 += (len_utf16 != 0) as usize;
261}
262
263impl Utf32CodeUnits {
264    pub fn length_of(string: &str) -> Self {
265        // `std::str::Chars::count` is optimized in:
266        // https://github.com/rust-lang/rust/blob/main/library/core/src/str/count.rs
267        Self(string.chars().count())
268    }
269
270    pub fn to_utf8_code_units_in(self, string: &str) -> Utf8CodeUnits {
271        let mut current_utf32_offset = Utf32CodeUnits(0);
272        for (current_utf8_offset, utf8_byte) in string.bytes().enumerate() {
273            if (utf8_byte & 0b1100_0000) == 0b1000_0000 {
274                // UTF-8 continuation byte
275                continue;
276            }
277            if current_utf32_offset >= self {
278                return Utf8CodeUnits(current_utf8_offset);
279            }
280            current_utf32_offset.0 += 1;
281        }
282        Utf8CodeUnits(string.len())
283    }
284
285    pub fn to_utf16_code_units_in(self, string: &str) -> Utf16CodeUnits {
286        let mut current_utf32_offset = Utf32CodeUnits(0);
287        let mut current_utf16_offset = Utf16CodeUnits(0);
288        for utf8_byte in string.bytes() {
289            if current_utf32_offset >= self {
290                break;
291            }
292            increment_offsets_for_utf8_byte(
293                utf8_byte,
294                &mut current_utf16_offset,
295                &mut current_utf32_offset,
296            );
297        }
298        current_utf16_offset
299    }
300}
301
302impl Utf32CodeUnitsOrNodeOffset {
303    pub fn to_utf16_code_units_in(self, string: &str) -> Utf16CodeUnits {
304        Utf32CodeUnits(self.0).to_utf16_code_units_in(string)
305    }
306}
307
308#[cfg(test)]
309mod test {
310    use super::*;
311
312    #[test]
313    fn test_is_cjk() {
314        // Test characters from different CJK blocks
315        assert_eq!(is_cjk('〇'), true);
316        assert_eq!(is_cjk('㐀'), true);
317        assert_eq!(is_cjk('あ'), true);
318        assert_eq!(is_cjk('ア'), true);
319        assert_eq!(is_cjk('㆒'), true);
320        assert_eq!(is_cjk('ㆣ'), true);
321        assert_eq!(is_cjk('龥'), true);
322        assert_eq!(is_cjk('𰾑'), true);
323        assert_eq!(is_cjk('𰻝'), true);
324
325        // Test characters from outside CJK blocks
326        assert_eq!(is_cjk('a'), false);
327        assert_eq!(is_cjk('🙂'), false);
328        assert_eq!(is_cjk('©'), false);
329    }
330
331    #[test]
332    fn test_utf16_length() {
333        assert_eq!(Utf16CodeUnits::length_of(""), Utf16CodeUnits(0));
334        assert_eq!(Utf16CodeUnits::length_of("a"), Utf16CodeUnits(1));
335        assert_eq!(Utf16CodeUnits::length_of("é"), Utf16CodeUnits(1));
336        assert_eq!(Utf16CodeUnits::length_of("字"), Utf16CodeUnits(1));
337        assert_eq!(Utf16CodeUnits::length_of("\u{1F4A9}"), Utf16CodeUnits(2));
338        assert_eq!(
339            Utf16CodeUnits::length_of("\u{1F4A9}字éa"),
340            Utf16CodeUnits(5)
341        );
342    }
343
344    #[test]
345    fn test_utf16_to_utf32() {
346        let s = "aé字\u{1F4A9}";
347        assert_eq!(
348            Utf16CodeUnits(0).to_utf32_code_units_in(s),
349            Utf32CodeUnits(0)
350        );
351        assert_eq!(
352            Utf16CodeUnits(1).to_utf32_code_units_in(s),
353            Utf32CodeUnits(1)
354        );
355        assert_eq!(
356            Utf16CodeUnits(2).to_utf32_code_units_in(s),
357            Utf32CodeUnits(2)
358        );
359        assert_eq!(
360            Utf16CodeUnits(3).to_utf32_code_units_in(s),
361            Utf32CodeUnits(3)
362        );
363
364        // This 16-bit offset splits the would-be surrogate pair. We return the 32-bit position
365        // after the whole pair. Should this be an error instead?
366        assert_eq!(
367            Utf16CodeUnits(4).to_utf32_code_units_in(s),
368            Utf32CodeUnits(4)
369        );
370
371        assert_eq!(
372            Utf16CodeUnits(5).to_utf32_code_units_in(s),
373            Utf32CodeUnits(4)
374        );
375
376        // This 16-bit offset is out of bounds. We clamp to the nearest valid 32-bit offset,
377        // a.k.a the UTF-32 length. Should this be an error instead?
378        assert_eq!(
379            Utf16CodeUnits(6).to_utf32_code_units_in(s),
380            Utf32CodeUnits(4)
381        );
382        assert_eq!(
383            Utf16CodeUnits(7).to_utf32_code_units_in(s),
384            Utf32CodeUnits(4)
385        );
386    }
387
388    #[test]
389    fn test_utf32_to_utf16() {
390        let string = "aé字\u{1F4A9}";
391        assert_eq!(
392            Utf32CodeUnits(0).to_utf16_code_units_in(string),
393            Utf16CodeUnits(0),
394        );
395        assert_eq!(
396            Utf32CodeUnits(1).to_utf16_code_units_in(string),
397            Utf16CodeUnits(1),
398        );
399        assert_eq!(
400            Utf32CodeUnits(2).to_utf16_code_units_in(string),
401            Utf16CodeUnits(2),
402        );
403        assert_eq!(
404            Utf32CodeUnits(3).to_utf16_code_units_in(string),
405            Utf16CodeUnits(3),
406        );
407
408        assert_eq!(
409            Utf32CodeUnits(4).to_utf16_code_units_in(string),
410            Utf16CodeUnits(5),
411        );
412
413        // This 32-bit offset is out of bounds. We clamp to the nearest valid 16-bit offset,
414        // a.k.a the UTF-16 length. Should this be an error instead?
415        assert_eq!(
416            Utf32CodeUnits(6).to_utf16_code_units_in(string),
417            Utf16CodeUnits(5),
418        );
419        assert_eq!(
420            Utf32CodeUnits(1000).to_utf16_code_units_in(string),
421            Utf16CodeUnits(5),
422        );
423    }
424}
servo_base/text.rs

servo_base/
text.rs
