oxc_markdown_parser/syntax/
unicode.rs

1//! Unicode punctuation (general categories P and S),
2//! as CommonMark's flanking rules define it.
3//!
4//! micromark tests `/\p{P}|\p{S}/u`, i.e. the JS runtime's Unicode tables;
5//! this table is that set pinned at Unicode 17.0 (Node 26).
6//! Regenerate when the runtime moves:
7//!
8//! ```js
9//! const re = /\p{P}|\p{S}/u, cps = [];
10//! for (let c = 0x80; c <= 0x10ffff; c++)
11//!   if ((c < 0xd800 || c > 0xdfff) && re.test(String.fromCodePoint(c))) cps.push(c);
12//! // then collapse `cps` into inclusive ranges
13//! ```
14//!
15//! Astral characters included: micromark tests UTF-16 code units,
16//! so it sees them as surrogates and never as punctuation.
17//! That is a documented divergence (`DIVERGENCES.md`, "Astral punctuation"), not parity.
18
19/// micromark's `unicodeWhitespace`
20/// (JavaScript `\s` = Unicode `White_Space` minus U+0085, plus U+FEFF).
21pub fn is_whitespace(c: char) -> bool {
22    (c.is_whitespace() && c != '\u{85}') || c == '\u{FEFF}'
23}
24
25/// Whether `c` is a Unicode punctuation character (general category P or S).
26pub fn is_punctuation(c: char) -> bool {
27    let cp = u32::from(c);
28    if cp < 0x80 {
29        return c.is_ascii_punctuation();
30    }
31    PUNCTUATION_RANGES
32        .binary_search_by(|&(lo, hi)| {
33            if cp < lo {
34                std::cmp::Ordering::Greater
35            } else if cp > hi {
36                std::cmp::Ordering::Less
37            } else {
38                std::cmp::Ordering::Equal
39            }
40        })
41        .is_ok()
42}
43
44#[rustfmt::skip]
45static PUNCTUATION_RANGES: &[(u32, u32)] = &[
46    (0xA1, 0xA9), (0xAB, 0xAC), (0xAE, 0xB1), (0xB4, 0xB4), (0xB6, 0xB8), (0xBB, 0xBB),
47    (0xBF, 0xBF), (0xD7, 0xD7), (0xF7, 0xF7), (0x2C2, 0x2C5), (0x2D2, 0x2DF), (0x2E5, 0x2EB),
48    (0x2ED, 0x2ED), (0x2EF, 0x2FF), (0x375, 0x375), (0x37E, 0x37E), (0x384, 0x385), (0x387, 0x387),
49    (0x3F6, 0x3F6), (0x482, 0x482), (0x55A, 0x55F), (0x589, 0x58A), (0x58D, 0x58F), (0x5BE, 0x5BE),
50    (0x5C0, 0x5C0), (0x5C3, 0x5C3), (0x5C6, 0x5C6), (0x5F3, 0x5F4), (0x606, 0x60F), (0x61B, 0x61B),
51    (0x61D, 0x61F), (0x66A, 0x66D), (0x6D4, 0x6D4), (0x6DE, 0x6DE), (0x6E9, 0x6E9), (0x6FD, 0x6FE),
52    (0x700, 0x70D), (0x7F6, 0x7F9), (0x7FE, 0x7FF), (0x830, 0x83E), (0x85E, 0x85E), (0x888, 0x888),
53    (0x964, 0x965), (0x970, 0x970), (0x9F2, 0x9F3), (0x9FA, 0x9FB), (0x9FD, 0x9FD), (0xA76, 0xA76),
54    (0xAF0, 0xAF1), (0xB70, 0xB70), (0xBF3, 0xBFA), (0xC77, 0xC77), (0xC7F, 0xC7F), (0xC84, 0xC84),
55    (0xD4F, 0xD4F), (0xD79, 0xD79), (0xDF4, 0xDF4), (0xE3F, 0xE3F), (0xE4F, 0xE4F), (0xE5A, 0xE5B),
56    (0xF01, 0xF17), (0xF1A, 0xF1F), (0xF34, 0xF34), (0xF36, 0xF36), (0xF38, 0xF38), (0xF3A, 0xF3D),
57    (0xF85, 0xF85), (0xFBE, 0xFC5), (0xFC7, 0xFCC), (0xFCE, 0xFDA), (0x104A, 0x104F),
58    (0x109E, 0x109F), (0x10FB, 0x10FB), (0x1360, 0x1368), (0x1390, 0x1399), (0x1400, 0x1400),
59    (0x166D, 0x166E), (0x169B, 0x169C), (0x16EB, 0x16ED), (0x1735, 0x1736), (0x17D4, 0x17D6),
60    (0x17D8, 0x17DB), (0x1800, 0x180A), (0x1940, 0x1940), (0x1944, 0x1945), (0x19DE, 0x19FF),
61    (0x1A1E, 0x1A1F), (0x1AA0, 0x1AA6), (0x1AA8, 0x1AAD), (0x1B4E, 0x1B4F), (0x1B5A, 0x1B6A),
62    (0x1B74, 0x1B7F), (0x1BFC, 0x1BFF), (0x1C3B, 0x1C3F), (0x1C7E, 0x1C7F), (0x1CC0, 0x1CC7),
63    (0x1CD3, 0x1CD3), (0x1FBD, 0x1FBD), (0x1FBF, 0x1FC1), (0x1FCD, 0x1FCF), (0x1FDD, 0x1FDF),
64    (0x1FED, 0x1FEF), (0x1FFD, 0x1FFE), (0x2010, 0x2027), (0x2030, 0x205E), (0x207A, 0x207E),
65    (0x208A, 0x208E), (0x20A0, 0x20C1), (0x2100, 0x2101), (0x2103, 0x2106), (0x2108, 0x2109),
66    (0x2114, 0x2114), (0x2116, 0x2118), (0x211E, 0x2123), (0x2125, 0x2125), (0x2127, 0x2127),
67    (0x2129, 0x2129), (0x212E, 0x212E), (0x213A, 0x213B), (0x2140, 0x2144), (0x214A, 0x214D),
68    (0x214F, 0x214F), (0x218A, 0x218B), (0x2190, 0x2429), (0x2440, 0x244A), (0x249C, 0x24E9),
69    (0x2500, 0x2775), (0x2794, 0x2B73), (0x2B76, 0x2BFF), (0x2CE5, 0x2CEA), (0x2CF9, 0x2CFC),
70    (0x2CFE, 0x2CFF), (0x2D70, 0x2D70), (0x2E00, 0x2E2E), (0x2E30, 0x2E5D), (0x2E80, 0x2E99),
71    (0x2E9B, 0x2EF3), (0x2F00, 0x2FD5), (0x2FF0, 0x2FFF), (0x3001, 0x3004), (0x3008, 0x3020),
72    (0x3030, 0x3030), (0x3036, 0x3037), (0x303D, 0x303F), (0x309B, 0x309C), (0x30A0, 0x30A0),
73    (0x30FB, 0x30FB), (0x3190, 0x3191), (0x3196, 0x319F), (0x31C0, 0x31E5), (0x31EF, 0x31EF),
74    (0x3200, 0x321E), (0x322A, 0x3247), (0x3250, 0x3250), (0x3260, 0x327F), (0x328A, 0x32B0),
75    (0x32C0, 0x33FF), (0x4DC0, 0x4DFF), (0xA490, 0xA4C6), (0xA4FE, 0xA4FF), (0xA60D, 0xA60F),
76    (0xA673, 0xA673), (0xA67E, 0xA67E), (0xA6F2, 0xA6F7), (0xA700, 0xA716), (0xA720, 0xA721),
77    (0xA789, 0xA78A), (0xA828, 0xA82B), (0xA836, 0xA839), (0xA874, 0xA877), (0xA8CE, 0xA8CF),
78    (0xA8F8, 0xA8FA), (0xA8FC, 0xA8FC), (0xA92E, 0xA92F), (0xA95F, 0xA95F), (0xA9C1, 0xA9CD),
79    (0xA9DE, 0xA9DF), (0xAA5C, 0xAA5F), (0xAA77, 0xAA79), (0xAADE, 0xAADF), (0xAAF0, 0xAAF1),
80    (0xAB5B, 0xAB5B), (0xAB6A, 0xAB6B), (0xABEB, 0xABEB), (0xFB29, 0xFB29), (0xFBB2, 0xFBD2),
81    (0xFD3E, 0xFD4F), (0xFD90, 0xFD91), (0xFDC8, 0xFDCF), (0xFDFC, 0xFDFF), (0xFE10, 0xFE19),
82    (0xFE30, 0xFE52), (0xFE54, 0xFE66), (0xFE68, 0xFE6B), (0xFF01, 0xFF0F), (0xFF1A, 0xFF20),
83    (0xFF3B, 0xFF40), (0xFF5B, 0xFF65), (0xFFE0, 0xFFE6), (0xFFE8, 0xFFEE), (0xFFFC, 0xFFFD),
84    (0x10100, 0x10102), (0x10137, 0x1013F), (0x10179, 0x10189), (0x1018C, 0x1018E), (0x10190, 0x1019C),
85    (0x101A0, 0x101A0), (0x101D0, 0x101FC), (0x1039F, 0x1039F), (0x103D0, 0x103D0), (0x1056F, 0x1056F),
86    (0x10857, 0x10857), (0x10877, 0x10878), (0x1091F, 0x1091F), (0x1093F, 0x1093F), (0x10A50, 0x10A58),
87    (0x10A7F, 0x10A7F), (0x10AC8, 0x10AC8), (0x10AF0, 0x10AF6), (0x10B39, 0x10B3F), (0x10B99, 0x10B9C),
88    (0x10D6E, 0x10D6E), (0x10D8E, 0x10D8F), (0x10EAD, 0x10EAD), (0x10ED0, 0x10ED8), (0x10F55, 0x10F59),
89    (0x10F86, 0x10F89), (0x11047, 0x1104D), (0x110BB, 0x110BC), (0x110BE, 0x110C1), (0x11140, 0x11143),
90    (0x11174, 0x11175), (0x111C5, 0x111C8), (0x111CD, 0x111CD), (0x111DB, 0x111DB), (0x111DD, 0x111DF),
91    (0x11238, 0x1123D), (0x112A9, 0x112A9), (0x113D4, 0x113D5), (0x113D7, 0x113D8), (0x1144B, 0x1144F),
92    (0x1145A, 0x1145B), (0x1145D, 0x1145D), (0x114C6, 0x114C6), (0x115C1, 0x115D7), (0x11641, 0x11643),
93    (0x11660, 0x1166C), (0x116B9, 0x116B9), (0x1173C, 0x1173F), (0x1183B, 0x1183B), (0x11944, 0x11946),
94    (0x119E2, 0x119E2), (0x11A3F, 0x11A46), (0x11A9A, 0x11A9C), (0x11A9E, 0x11AA2), (0x11B00, 0x11B09),
95    (0x11BE1, 0x11BE1), (0x11C41, 0x11C45), (0x11C70, 0x11C71), (0x11EF7, 0x11EF8), (0x11F43, 0x11F4F),
96    (0x11FD5, 0x11FF1), (0x11FFF, 0x11FFF), (0x12470, 0x12474), (0x12FF1, 0x12FF2), (0x16A6E, 0x16A6F),
97    (0x16AF5, 0x16AF5), (0x16B37, 0x16B3F), (0x16B44, 0x16B45), (0x16D6D, 0x16D6F), (0x16E97, 0x16E9A),
98    (0x16FE2, 0x16FE2), (0x1BC9C, 0x1BC9C), (0x1BC9F, 0x1BC9F), (0x1CC00, 0x1CCEF), (0x1CCFA, 0x1CCFC),
99    (0x1CD00, 0x1CEB3), (0x1CEBA, 0x1CED0), (0x1CEE0, 0x1CEF0), (0x1CF50, 0x1CFC3), (0x1D000, 0x1D0F5),
100    (0x1D100, 0x1D126), (0x1D129, 0x1D164), (0x1D16A, 0x1D16C), (0x1D183, 0x1D184), (0x1D18C, 0x1D1A9),
101    (0x1D1AE, 0x1D1EA), (0x1D200, 0x1D241), (0x1D245, 0x1D245), (0x1D300, 0x1D356), (0x1D6C1, 0x1D6C1),
102    (0x1D6DB, 0x1D6DB), (0x1D6FB, 0x1D6FB), (0x1D715, 0x1D715), (0x1D735, 0x1D735), (0x1D74F, 0x1D74F),
103    (0x1D76F, 0x1D76F), (0x1D789, 0x1D789), (0x1D7A9, 0x1D7A9), (0x1D7C3, 0x1D7C3), (0x1D800, 0x1D9FF),
104    (0x1DA37, 0x1DA3A), (0x1DA6D, 0x1DA74), (0x1DA76, 0x1DA83), (0x1DA85, 0x1DA8B), (0x1E14F, 0x1E14F),
105    (0x1E2FF, 0x1E2FF), (0x1E5FF, 0x1E5FF), (0x1E95E, 0x1E95F), (0x1ECAC, 0x1ECAC), (0x1ECB0, 0x1ECB0),
106    (0x1ED2E, 0x1ED2E), (0x1EEF0, 0x1EEF1), (0x1F000, 0x1F02B), (0x1F030, 0x1F093), (0x1F0A0, 0x1F0AE),
107    (0x1F0B1, 0x1F0BF), (0x1F0C1, 0x1F0CF), (0x1F0D1, 0x1F0F5), (0x1F10D, 0x1F1AD), (0x1F1E6, 0x1F202),
108    (0x1F210, 0x1F23B), (0x1F240, 0x1F248), (0x1F250, 0x1F251), (0x1F260, 0x1F265), (0x1F300, 0x1F6D8),
109    (0x1F6DC, 0x1F6EC), (0x1F6F0, 0x1F6FC), (0x1F700, 0x1F7D9), (0x1F7E0, 0x1F7EB), (0x1F7F0, 0x1F7F0),
110    (0x1F800, 0x1F80B), (0x1F810, 0x1F847), (0x1F850, 0x1F859), (0x1F860, 0x1F887), (0x1F890, 0x1F8AD),
111    (0x1F8B0, 0x1F8BB), (0x1F8C0, 0x1F8C1), (0x1F8D0, 0x1F8D8), (0x1F900, 0x1FA57), (0x1FA60, 0x1FA6D),
112    (0x1FA70, 0x1FA7C), (0x1FA80, 0x1FA8A), (0x1FA8E, 0x1FAC6), (0x1FAC8, 0x1FAC8), (0x1FACD, 0x1FADC),
113    (0x1FADF, 0x1FAEA), (0x1FAEF, 0x1FAF8), (0x1FB00, 0x1FB92), (0x1FB94, 0x1FBEF), (0x1FBFA, 0x1FBFA),
114];
115
116#[cfg(test)]
117mod tests {
118    use super::*;
119
120    #[test]
121    fn whitespace_is_js_backslash_s() {
122        assert!(is_whitespace(' ') && is_whitespace('\t') && is_whitespace('\n'));
123        assert!(is_whitespace('\u{A0}') && is_whitespace('\u{3000}'));
124        assert!(is_whitespace('\u{FEFF}'), "BOM is `\\s` in JS but not `White_Space`");
125        assert!(!is_whitespace('\u{85}'), "NEL is `White_Space` but not `\\s` in JS");
126        assert!(!is_whitespace('a'));
127    }
128
129    #[test]
130    fn punctuation_covers_categories_p_and_s() {
131        assert!(is_punctuation('*') && is_punctuation('$'));
132        assert!(!is_punctuation('a') && !is_punctuation(' '));
133        assert!(is_punctuation('«') && is_punctuation('。') && is_punctuation('→'));
134        assert!(is_punctuation('©'), "symbols (S) count, not only punctuation (P)");
135        assert!(!is_punctuation('é') && !is_punctuation('日'));
136        // Range boundaries are inclusive
137        assert!(is_punctuation('\u{A1}') && is_punctuation('\u{A9}'));
138        assert!(!is_punctuation('\u{AA}'));
139        assert!(is_punctuation('😀'), "astral symbols count (spec), unlike micromark");
140        assert!(!is_punctuation('\u{10000}'), "astral letters (Lo) stay non-punctuation");
141    }
142}
oxc_markdown_parser/syntax/unicode.rs

oxc_markdown_parser/syntax/
unicode.rs
