solar_parse/lexer/
unicode_chars.rs

1//! Characters and their corresponding confusables were collected from
2//! <https://www.unicode.org/Public/security/10.0.0/confusables.txt>
3
4#![allow(unused_imports, dead_code)]
5
6use super::Lexer;
7use solar_ast::token::{BinOpToken, Delimiter, TokenKind};
8use solar_interface::{BytePos, Span, sym};
9
10#[rustfmt::skip] // for line breaks
11pub(crate) const UNICODE_ARRAY: &[(char, &str, &str)] = &[
12    (' ', "Line Separator", " "),
13    (' ', "Paragraph Separator", " "),
14    (' ', "Ogham Space mark", " "),
15    (' ', "En Quad", " "),
16    (' ', "Em Quad", " "),
17    (' ', "En Space", " "),
18    (' ', "Em Space", " "),
19    (' ', "Three-Per-Em Space", " "),
20    (' ', "Four-Per-Em Space", " "),
21    (' ', "Six-Per-Em Space", " "),
22    (' ', "Punctuation Space", " "),
23    (' ', "Thin Space", " "),
24    (' ', "Hair Space", " "),
25    (' ', "Medium Mathematical Space", " "),
26    (' ', "No-Break Space", " "),
27    (' ', "Figure Space", " "),
28    (' ', "Narrow No-Break Space", " "),
29    ('　', "Ideographic Space", " "),
30
31    ('ߺ', "Nko Lajanyalan", "_"),
32    ('﹍', "Dashed Low Line", "_"),
33    ('﹎', "Centreline Low Line", "_"),
34    ('﹏', "Wavy Low Line", "_"),
35    ('＿', "Fullwidth Low Line", "_"),
36
37    ('‐', "Hyphen", "-"),
38    ('‑', "Non-Breaking Hyphen", "-"),
39    ('‒', "Figure Dash", "-"),
40    ('–', "En Dash", "-"),
41    ('—', "Em Dash", "-"),
42    ('﹘', "Small Em Dash", "-"),
43    ('۔', "Arabic Full Stop", "-"),
44    ('⁃', "Hyphen Bullet", "-"),
45    ('˗', "Modifier Letter Minus Sign", "-"),
46    ('−', "Minus Sign", "-"),
47    ('➖', "Heavy Minus Sign", "-"),
48    ('Ⲻ', "Coptic Letter Dialect-P Ni", "-"),
49    ('ー', "Katakana-Hiragana Prolonged Sound Mark", "-"),
50    ('－', "Fullwidth Hyphen-Minus", "-"),
51    ('―', "Horizontal Bar", "-"),
52    ('─', "Box Drawings Light Horizontal", "-"),
53    ('━', "Box Drawings Heavy Horizontal", "-"),
54    ('㇐', "CJK Stroke H", "-"),
55    ('ꟷ', "Latin Epigraphic Letter Sideways I", "-"),
56    ('ᅳ', "Hangul Jungseong Eu", "-"),
57    ('ㅡ', "Hangul Letter Eu", "-"),
58    ('一', "CJK Unified Ideograph-4E00", "-"),
59    ('⼀', "Kangxi Radical One", "-"),
60
61    ('؍', "Arabic Date Separator", ","),
62    ('٫', "Arabic Decimal Separator", ","),
63    ('‚', "Single Low-9 Quotation Mark", ","),
64    ('¸', "Cedilla", ","),
65    ('ꓹ', "Lisu Letter Tone Na Po", ","),
66    ('，', "Fullwidth Comma", ","),
67
68    (';', "Greek Question Mark", ";"),
69    ('；', "Fullwidth Semicolon", ";"),
70    ('︔', "Presentation Form For Vertical Semicolon", ";"),
71
72    ('ः', "Devanagari Sign Visarga", ":"),
73    ('ઃ', "Gujarati Sign Visarga", ":"),
74    ('：', "Fullwidth Colon", ":"),
75    ('։', "Armenian Full Stop", ":"),
76    ('܃', "Syriac Supralinear Colon", ":"),
77    ('܄', "Syriac Sublinear Colon", ":"),
78    ('᛬', "Runic Multiple Punctuation", ":"),
79    ('︰', "Presentation Form For Vertical Two Dot Leader", ":"),
80    ('᠃', "Mongolian Full Stop", ":"),
81    ('᠉', "Mongolian Manchu Full Stop", ":"),
82    ('⁚', "Two Dot Punctuation", ":"),
83    ('׃', "Hebrew Punctuation Sof Pasuq", ":"),
84    ('˸', "Modifier Letter Raised Colon", ":"),
85    ('꞉', "Modifier Letter Colon", ":"),
86    ('∶', "Ratio", ":"),
87    ('ː', "Modifier Letter Triangular Colon", ":"),
88    ('ꓽ', "Lisu Letter Tone Mya Jeu", ":"),
89    ('︓', "Presentation Form For Vertical Colon", ":"),
90
91    ('！', "Fullwidth Exclamation Mark", "!"),
92    ('ǃ', "Latin Letter Retroflex Click", "!"),
93    ('ⵑ', "Tifinagh Letter Tuareg Yang", "!"),
94    ('︕', "Presentation Form For Vertical Exclamation Mark", "!"),
95
96    ('ʔ', "Latin Letter Glottal Stop", "?"),
97    ('Ɂ', "Latin Capital Letter Glottal Stop", "?"),
98    ('ॽ', "Devanagari Letter Glottal Stop", "?"),
99    ('Ꭾ', "Cherokee Letter He", "?"),
100    ('ꛫ', "Bamum Letter Ntuu", "?"),
101    ('？', "Fullwidth Question Mark", "?"),
102    ('︖', "Presentation Form For Vertical Question Mark", "?"),
103
104    ('𝅭', "Musical Symbol Combining Augmentation Dot", "."),
105    ('․', "One Dot Leader", "."),
106    ('܁', "Syriac Supralinear Full Stop", "."),
107    ('܂', "Syriac Sublinear Full Stop", "."),
108    ('꘎', "Vai Full Stop", "."),
109    ('𐩐', "Kharoshthi Punctuation Dot", "."),
110    ('٠', "Arabic-Indic Digit Zero", "."),
111    ('۰', "Extended Arabic-Indic Digit Zero", "."),
112    ('ꓸ', "Lisu Letter Tone Mya Ti", "."),
113    ('·', "Middle Dot", "."),
114    ('・', "Katakana Middle Dot", "."),
115    ('･', "Halfwidth Katakana Middle Dot", "."),
116    ('᛫', "Runic Single Punctuation", "."),
117    ('·', "Greek Ano Teleia", "."),
118    ('⸱', "Word Separator Middle Dot", "."),
119    ('𐄁', "Aegean Word Separator Dot", "."),
120    ('•', "Bullet", "."),
121    ('‧', "Hyphenation Point", "."),
122    ('∙', "Bullet Operator", "."),
123    ('⋅', "Dot Operator", "."),
124    ('ꞏ', "Latin Letter Sinological Dot", "."),
125    ('ᐧ', "Canadian Syllabics Final Middle Dot", "."),
126    ('ᐧ', "Canadian Syllabics Final Middle Dot", "."),
127    ('．', "Fullwidth Full Stop", "."),
128    ('。', "Ideographic Full Stop", "."),
129    ('︒', "Presentation Form For Vertical Ideographic Full Stop", "."),
130
131    ('՝', "Armenian Comma", "\'"),
132    ('＇', "Fullwidth Apostrophe", "\'"),
133    ('‘', "Left Single Quotation Mark", "\'"),
134    ('’', "Right Single Quotation Mark", "\'"),
135    ('‛', "Single High-Reversed-9 Quotation Mark", "\'"),
136    ('′', "Prime", "\'"),
137    ('‵', "Reversed Prime", "\'"),
138    ('՚', "Armenian Apostrophe", "\'"),
139    ('׳', "Hebrew Punctuation Geresh", "\'"),
140    ('`', "Grave Accent", "\'"),
141    ('`', "Greek Varia", "\'"),
142    ('｀', "Fullwidth Grave Accent", "\'"),
143    ('´', "Acute Accent", "\'"),
144    ('΄', "Greek Tonos", "\'"),
145    ('´', "Greek Oxia", "\'"),
146    ('᾽', "Greek Koronis", "\'"),
147    ('᾿', "Greek Psili", "\'"),
148    ('῾', "Greek Dasia", "\'"),
149    ('ʹ', "Modifier Letter Prime", "\'"),
150    ('ʹ', "Greek Numeral Sign", "\'"),
151    ('ˈ', "Modifier Letter Vertical Line", "\'"),
152    ('ˊ', "Modifier Letter Acute Accent", "\'"),
153    ('ˋ', "Modifier Letter Grave Accent", "\'"),
154    ('˴', "Modifier Letter Middle Grave Accent", "\'"),
155    ('ʻ', "Modifier Letter Turned Comma", "\'"),
156    ('ʽ', "Modifier Letter Reversed Comma", "\'"),
157    ('ʼ', "Modifier Letter Apostrophe", "\'"),
158    ('ʾ', "Modifier Letter Right Half Ring", "\'"),
159    ('ꞌ', "Latin Small Letter Saltillo", "\'"),
160    ('י', "Hebrew Letter Yod", "\'"),
161    ('ߴ', "Nko High Tone Apostrophe", "\'"),
162    ('ߵ', "Nko Low Tone Apostrophe", "\'"),
163    ('ᑊ', "Canadian Syllabics West-Cree P", "\'"),
164    ('ᛌ', "Runic Letter Short-Twig-Sol S", "\'"),
165    ('𖽑', "Miao Sign Aspiration", "\'"),
166    ('𖽒', "Miao Sign Reformed Voicing", "\'"),
167
168    ('᳓', "Vedic Sign Nihshvasa", "\""),
169    ('＂', "Fullwidth Quotation Mark", "\""),
170    ('“', "Left Double Quotation Mark", "\""),
171    ('”', "Right Double Quotation Mark", "\""),
172    ('‟', "Double High-Reversed-9 Quotation Mark", "\""),
173    ('″', "Double Prime", "\""),
174    ('‶', "Reversed Double Prime", "\""),
175    ('〃', "Ditto Mark", "\""),
176    ('״', "Hebrew Punctuation Gershayim", "\""),
177    ('˝', "Double Acute Accent", "\""),
178    ('ʺ', "Modifier Letter Double Prime", "\""),
179    ('˶', "Modifier Letter Middle Double Acute Accent", "\""),
180    ('˵', "Modifier Letter Middle Double Grave Accent", "\""),
181    ('ˮ', "Modifier Letter Double Apostrophe", "\""),
182    ('ײ', "Hebrew Ligature Yiddish Double Yod", "\""),
183    ('❞', "Heavy Double Comma Quotation Mark Ornament", "\""),
184    ('❝', "Heavy Double Turned Comma Quotation Mark Ornament", "\""),
185
186    ('（', "Fullwidth Left Parenthesis", "("),
187    ('❨', "Medium Left Parenthesis Ornament", "("),
188    ('﴾', "Ornate Left Parenthesis", "("),
189
190    ('）', "Fullwidth Right Parenthesis", ")"),
191    ('❩', "Medium Right Parenthesis Ornament", ")"),
192    ('﴿', "Ornate Right Parenthesis", ")"),
193
194    ('［', "Fullwidth Left Square Bracket", "["),
195    ('❲', "Light Left Tortoise Shell Bracket Ornament", "["),
196    ('「', "Left Corner Bracket", "["),
197    ('『', "Left White Corner Bracket", "["),
198    ('【', "Left Black Lenticular Bracket", "["),
199    ('〔', "Left Tortoise Shell Bracket", "["),
200    ('〖', "Left White Lenticular Bracket", "["),
201    ('〘', "Left White Tortoise Shell Bracket", "["),
202    ('〚', "Left White Square Bracket", "["),
203
204    ('］', "Fullwidth Right Square Bracket", "]"),
205    ('❳', "Light Right Tortoise Shell Bracket Ornament", "]"),
206    ('」', "Right Corner Bracket", "]"),
207    ('』', "Right White Corner Bracket", "]"),
208    ('】', "Right Black Lenticular Bracket", "]"),
209    ('〕', "Right Tortoise Shell Bracket", "]"),
210    ('〗', "Right White Lenticular Bracket", "]"),
211    ('〙', "Right White Tortoise Shell Bracket", "]"),
212    ('〛', "Right White Square Bracket", "]"),
213
214    ('❴', "Medium Left Curly Bracket Ornament", "{"),
215    ('𝄔', "Musical Symbol Brace", "{"),
216    ('｛', "Fullwidth Left Curly Bracket", "{"),
217
218    ('❵', "Medium Right Curly Bracket Ornament", "}"),
219    ('｝', "Fullwidth Right Curly Bracket", "}"),
220
221    ('⁎', "Low Asterisk", "*"),
222    ('٭', "Arabic Five Pointed Star", "*"),
223    ('∗', "Asterisk Operator", "*"),
224    ('𐌟', "Old Italic Letter Ess", "*"),
225    ('＊', "Fullwidth Asterisk", "*"),
226
227    ('᜵', "Philippine Single Punctuation", "/"),
228    ('⁁', "Caret Insertion Point", "/"),
229    ('∕', "Division Slash", "/"),
230    ('⁄', "Fraction Slash", "/"),
231    ('╱', "Box Drawings Light Diagonal Upper Right To Lower Left", "/"),
232    ('⟋', "Mathematical Rising Diagonal", "/"),
233    ('⧸', "Big Solidus", "/"),
234    ('𝈺', "Greek Instrumental Notation Symbol-47", "/"),
235    ('㇓', "CJK Stroke Sp", "/"),
236    ('〳', "Vertical Kana Repeat Mark Upper Half", "/"),
237    ('Ⳇ', "Coptic Capital Letter Old Coptic Esh", "/"),
238    ('ノ', "Katakana Letter No", "/"),
239    ('丿', "CJK Unified Ideograph-4E3F", "/"),
240    ('⼃', "Kangxi Radical Slash", "/"),
241    ('／', "Fullwidth Solidus", "/"),
242
243    ('＼', "Fullwidth Reverse Solidus", "\\"),
244    ('﹨', "Small Reverse Solidus", "\\"),
245    ('∖', "Set Minus", "\\"),
246    ('⟍', "Mathematical Falling Diagonal", "\\"),
247    ('⧵', "Reverse Solidus Operator", "\\"),
248    ('⧹', "Big Reverse Solidus", "\\"),
249    ('⧹', "Greek Vocal Notation Symbol-16", "\\"),
250    ('⧹', "Greek Instrumental Symbol-48", "\\"),
251    ('㇔', "CJK Stroke D", "\\"),
252    ('丶', "CJK Unified Ideograph-4E36", "\\"),
253    ('⼂', "Kangxi Radical Dot", "\\"),
254    ('、', "Ideographic Comma", "\\"),
255    ('ヽ', "Katakana Iteration Mark", "\\"),
256
257    ('ꝸ', "Latin Small Letter Um", "&"),
258    ('＆', "Fullwidth Ampersand", "&"),
259
260    ('᛭', "Runic Cross Punctuation", "+"),
261    ('➕', "Heavy Plus Sign", "+"),
262    ('𐊛', "Lycian Letter H", "+"),
263    ('﬩', "Hebrew Letter Alternative Plus Sign", "+"),
264    ('＋', "Fullwidth Plus Sign", "+"),
265
266    ('‹', "Single Left-Pointing Angle Quotation Mark", "<"),
267    ('❮', "Heavy Left-Pointing Angle Quotation Mark Ornament", "<"),
268    ('˂', "Modifier Letter Left Arrowhead", "<"),
269    ('𝈶', "Greek Instrumental Symbol-40", "<"),
270    ('ᐸ', "Canadian Syllabics Pa", "<"),
271    ('ᚲ', "Runic Letter Kauna", "<"),
272    ('❬', "Medium Left-Pointing Angle Bracket Ornament", "<"),
273    ('⟨', "Mathematical Left Angle Bracket", "<"),
274    ('〈', "Left-Pointing Angle Bracket", "<"),
275    ('〈', "Left Angle Bracket", "<"),
276    ('㇛', "CJK Stroke Pd", "<"),
277    ('く', "Hiragana Letter Ku", "<"),
278    ('𡿨', "CJK Unified Ideograph-21FE8", "<"),
279    ('《', "Left Double Angle Bracket", "<"),
280    ('＜', "Fullwidth Less-Than Sign", "<"),
281
282    ('᐀', "Canadian Syllabics Hyphen", "="),
283    ('⹀', "Double Hyphen", "="),
284    ('゠', "Katakana-Hiragana Double Hyphen", "="),
285    ('꓿', "Lisu Punctuation Full Stop", "="),
286    ('＝', "Fullwidth Equals Sign", "="),
287
288    ('›', "Single Right-Pointing Angle Quotation Mark", ">"),
289    ('❯', "Heavy Right-Pointing Angle Quotation Mark Ornament", ">"),
290    ('˃', "Modifier Letter Right Arrowhead", ">"),
291    ('𝈷', "Greek Instrumental Symbol-42", ">"),
292    ('ᐳ', "Canadian Syllabics Po", ">"),
293    ('𖼿', "Miao Letter Archaic Zza", ">"),
294    ('❭', "Medium Right-Pointing Angle Bracket Ornament", ">"),
295    ('⟩', "Mathematical Right Angle Bracket", ">"),
296    ('〉', "Right-Pointing Angle Bracket", ">"),
297    ('〉', "Right Angle Bracket", ">"),
298    ('》', "Right Double Angle Bracket", ">"),
299    ('＞', "Fullwidth Greater-Than Sign", ">"),
300    ('⩵', "Two Consecutive Equals Signs", "==")
301];
302
303const ASCII_ARRAY: &[(&str, &str, Option<TokenKind>)] = &[
304    (" ", "Space", None),
305    ("_", "Underscore", Some(TokenKind::Ident(sym::underscore))),
306    ("-", "Minus/Hyphen", Some(TokenKind::BinOp(BinOpToken::Minus))),
307    (",", "Comma", Some(TokenKind::Comma)),
308    (";", "Semicolon", Some(TokenKind::Semi)),
309    (":", "Colon", Some(TokenKind::Colon)),
310    ("!", "Exclamation Mark", Some(TokenKind::Not)),
311    ("?", "Question Mark", Some(TokenKind::Question)),
312    (".", "Period", Some(TokenKind::Dot)),
313    ("(", "Left Parenthesis", Some(TokenKind::OpenDelim(Delimiter::Parenthesis))),
314    (")", "Right Parenthesis", Some(TokenKind::CloseDelim(Delimiter::Parenthesis))),
315    ("[", "Left Square Bracket", Some(TokenKind::OpenDelim(Delimiter::Bracket))),
316    ("]", "Right Square Bracket", Some(TokenKind::CloseDelim(Delimiter::Bracket))),
317    ("{", "Left Curly Brace", Some(TokenKind::OpenDelim(Delimiter::Brace))),
318    ("}", "Right Curly Brace", Some(TokenKind::CloseDelim(Delimiter::Brace))),
319    ("*", "Asterisk", Some(TokenKind::BinOp(BinOpToken::Star))),
320    ("/", "Slash", Some(TokenKind::BinOp(BinOpToken::Slash))),
321    ("\\", "Backslash", None),
322    ("&", "Ampersand", Some(TokenKind::BinOp(BinOpToken::And))),
323    ("+", "Plus Sign", Some(TokenKind::BinOp(BinOpToken::Plus))),
324    ("<", "Less-Than Sign", Some(TokenKind::Lt)),
325    ("=", "Equals Sign", Some(TokenKind::Eq)),
326    ("==", "Double Equals Sign", Some(TokenKind::EqEq)),
327    (">", "Greater-Than Sign", Some(TokenKind::Gt)),
328    // FIXME: Literals are already lexed by this point, so we can't recover gracefully just by
329    // spitting the correct token out.
330    ("\'", "Single Quote", None),
331    ("\"", "Quotation Mark", None),
332];
333
334pub(super) enum TokenSubstitution {
335    DirectedQuotes {
336        span: Span,
337        suggestion: String,
338        ascii_str: &'static str,
339        ascii_name: &'static str,
340    },
341    Other {
342        span: Span,
343        suggestion: String,
344        ch: String,
345        u_name: &'static str,
346        ascii_str: &'static str,
347        ascii_name: &'static str,
348    },
349}
350
351pub(super) fn check_for_substitution(
352    reader: &Lexer<'_, '_>,
353    pos: BytePos,
354    ch: char,
355    count: usize,
356) -> (Option<TokenKind>, Option<TokenSubstitution>) {
357    let Some(&(_, u_name, ascii_str)) = UNICODE_ARRAY.iter().find(|&&(c, _, _)| c == ch) else {
358        return (None, None);
359    };
360
361    let span = Span::new(pos, pos + BytePos::from_usize(ch.len_utf8() * count));
362
363    let Some((_, ascii_name, token)) = ASCII_ARRAY.iter().find(|&&(s, _, _)| s == ascii_str) else {
364        panic!("substitution character not found for {ch:?} ({ascii_str:?})");
365    };
366
367    // special help suggestion for "directed" double quotes
368    let sugg = if let Some(s) = peek_delimited(&reader.src[reader.src_index(pos)..], '“', '”') {
369        let c = '”'.len_utf8();
370        let span = Span::new(pos, pos + BytePos::from_usize(c + s.len() + c));
371        Some(TokenSubstitution::DirectedQuotes {
372            span,
373            suggestion: format!("\"{s}\""),
374            ascii_str,
375            ascii_name,
376        })
377    } else {
378        let suggestion = ascii_str.to_string().repeat(count);
379        Some(TokenSubstitution::Other {
380            span,
381            suggestion,
382            ch: ch.to_string(),
383            u_name,
384            ascii_str,
385            ascii_name,
386        })
387    };
388    (*token, sugg)
389}
390
391/// Extract string if found at current position with given delimiters
392fn peek_delimited(text: &str, from_ch: char, to_ch: char) -> Option<&str> {
393    let mut chars = text.chars();
394    let first_char = chars.next()?;
395    if first_char != from_ch {
396        return None;
397    }
398    let last_char_idx = chars.as_str().find(to_ch)?;
399    Some(&chars.as_str()[..last_char_idx])
400}
solar_parse/lexer/unicode_chars.rs

solar_parse/lexer/
unicode_chars.rs
