ling/lexer/
mod.rs

1// src/lexer/mod.rs — hand-written polyglot lexer
2mod cursor;
3mod token;
4mod unicode;
5
6pub use cursor::Cursor;
7pub use token::Token;
8
9pub struct Lexer<'a> {
10    source: &'a str,
11    pos: usize,
12}
13
14impl<'a> Lexer<'a> {
15    pub fn new(source: &'a str) -> Self {
16        Self { source, pos: 0 }
17    }
18
19    fn peek(&self) -> Option<char> {
20        self.source[self.pos..].chars().next()
21    }
22
23    fn peek_nth(&self, n: usize) -> Option<char> {
24        self.source[self.pos..].chars().nth(n)
25    }
26
27    fn advance(&mut self) -> Option<char> {
28        let ch = self.source[self.pos..].chars().next()?;
29        self.pos += ch.len_utf8();
30        Some(ch)
31    }
32
33    fn rest(&self) -> &str {
34        &self.source[self.pos..]
35    }
36
37    fn skip_whitespace_and_comments(&mut self) {
38        loop {
39            while matches!(
40                self.peek(),
41                Some(' ') | Some('\t') | Some('\n') | Some('\r') | Some('\x0C')
42            ) {
43                self.advance();
44            }
45            if self.peek() == Some('/') && self.peek_nth(1) == Some('/') {
46                while self.peek().is_some_and(|c| c != '\n') {
47                    self.advance();
48                }
49                continue;
50            }
51            // Shebang or # comment
52            if self.peek() == Some('#') {
53                while self.peek().is_some_and(|c| c != '\n') {
54                    self.advance();
55                }
56                continue;
57            }
58            break;
59        }
60    }
61
62    fn lex_string(&mut self) -> Token {
63        self.advance(); // opening "
64        let mut s = std::string::String::new();
65        loop {
66            match self.advance() {
67                None => break,
68                Some('"') => break,
69                Some('\\') => match self.advance() {
70                    Some('n') => s.push('\n'),
71                    Some('t') => s.push('\t'),
72                    Some('r') => s.push('\r'),
73                    Some('"') => s.push('"'),
74                    Some('\\') => s.push('\\'),
75                    Some('0') => s.push('\0'),
76                    Some(c) => {
77                        s.push('\\');
78                        s.push(c);
79                    },
80                    None => break,
81                },
82                Some(c) => s.push(c),
83            }
84        }
85        Token::String(s)
86    }
87
88    fn lex_number(&mut self) -> Token {
89        let start = self.pos;
90        let mut has_dot = false;
91        loop {
92            match self.peek() {
93                Some('0'..='9') => {
94                    self.advance();
95                },
96                Some('.') if !has_dot && matches!(self.peek_nth(1), Some('0'..='9')) => {
97                    has_dot = true;
98                    self.advance();
99                },
100                _ => break,
101            }
102        }
103        Token::Number(self.source[start..self.pos].to_string())
104    }
105
106    fn lex_word(&mut self) -> Token {
107        let start = self.pos;
108        // Scan contiguous word characters: letters, digits, underscore, and Unicode
109        // combining marks (tone marks, vowel signs, diacritics) that Rust's
110        // is_alphabetic() misses because they lack the Other_Alphabetic property.
111        while let Some(c) = self.peek() {
112            if c.is_alphanumeric() || c == '_' || is_unicode_combining(c) {
113                self.advance();
114            } else {
115                break;
116            }
117        }
118        let word = &self.source[start..self.pos];
119        Self::classify_word(word)
120    }
121
122    /// Map a word (ASCII or Unicode) to its canonical token.
123    /// Chinese/Japanese/Korean/Russian keywords resolve here so there is
124    /// no pre-processing that could corrupt variable names or string contents.
125    fn classify_word(word: &str) -> Token {
126        match word {
127            // ── English keywords ────────────────────────────────────────────
128            "bind" | "令" | "灵符" => Token::Bind,
129            "do" | "执" => Token::Do,
130            "fn" | "函" => Token::Fn,
131            "mod" | "核" => Token::Mod,
132            "type" | "符" | "型" | "타입" | "ชนิด" => Token::Type,
133            // Module imports: use / 载 (zh) / 使う (ja) / 사용 (ko) / ใช้ / นำเข้า (th)
134            "use" | "载" | "引" | "사용" | "使う" | "ใช้" | "นำเข้า" | "importar" | "nutzen"
135            | "utiliser" => Token::Use,
136            "if" | "若" | "如" => Token::If,
137            "else" | "否则" | "否" => Token::Else,
138            "while" | "循" | "当" => Token::While,
139            "for" | "历" => Token::For,
140            "in" | "于" => Token::In,
141            "match" | "配" => Token::Match,
142            "return" | "归" => Token::Return,
143            "own" | "拥有" | "独" | "所有" | "소유" | "เป็นเจ้าของ" => {
144                Token::Own
145            },
146            "lend" | "借" | "貸す" | "빌려" | "ให้ยืม" => Token::Lend,
147            "share" | "共享" | "共" | "共有" | "공유" | "แบ่งปัน" => {
148                Token::Share
149            },
150            "move" | "移动" | "移" | "移動" | "이동" | "ย้าย" => Token::Move,
151            "copy" | "复制" | "复" | "複製" | "복사" | "คัดลอก" => Token::Copy,
152            "async" | "异步" | "异" => Token::Async,
153            "wait" | "等待" | "待" => Token::Wait,
154            "as" | "为" | "として" | "로서" | "เป็น" => Token::As,
155            "where" | "条件" | "但し" | "단" | "โดยที่" => Token::Where,
156            "post" | "发布" | "出" | "投稿" | "게시" | "ส่ง" => Token::Post,
157            "give" | "给" | "予" | "渡す" | "전달" | "ให้" => Token::Give,
158            "fit" | "适合" | "適合" | "적합" | "เหมาะสม" => Token::Fit,
159            // `form` — record/struct definition (EN · ZH · JA · KO · TH)
160            "form" | "形式" | "形" | "構造" | "구조" | "โครงสร้าง" => {
161                Token::Form
162            },
163            // `choose` — sum type / enum definition (EN · ZH · JA · KO · TH)
164            "choose" | "选择" | "选" | "選択" | "선택" | "เลือกแบบ" => {
165                Token::Choose
166            },
167            "can" | "能" | "できる" | "가능" | "สามารถ" => Token::Can,
168            "change" | "改变" | "变" | "変える" | "변경" | "เปลี่ยนแปลง" => {
169                Token::Change
170            },
171            // stop/again/try/spawn's ja/ko forms live in the Japanese/Korean
172            // sections below — only th was missing here.
173            "stop" | "停止" | "止" | "หยุด" => Token::Stop,
174            "again" | "继续" | "ทำอีก" => Token::Again,
175            "try" | "尝试" | "试" | "ลอง" => Token::Try,
176            "sure" | "确定" | "确" | "確か" | "확실" | "แน่นอน" => Token::Sure,
177            "maybe" | "可能" | "或" | "多分" | "아마도" | "อาจจะ" => Token::Maybe,
178            "pure" | "纯" | "純粋" | "순수" | "บริสุทธิ์" => Token::Pure,
179            "spawn" | "生成" | "启" | "สร้าง" => Token::Spawn,
180            "asm" => Token::Asm,
181            "ok" | "好" | "可" | "良い" | "좋아" | "ตกลง" => Token::Ok,
182            "bad" | "坏" | "误" | "悪い" | "나쁨" | "ผิดพลาด" => Token::Bad,
183            "none" | "无" | "なし" | "없음" | "ไม่มี" => Token::None,
184            // Japanese — full keyword set with short kanji shorthands
185            "束縛" | "バ" => Token::Bind,
186            "実行" | "執" => Token::Do,
187            "関数" | "関" => Token::Fn,
188            "モジュール" | "模" => Token::Mod,
189            "もし" => Token::If,
190            "他" => Token::Else,
191            "間" | "一方" => Token::While,
192            "繰" | "ために" => Token::For,
193            "の中" => Token::In,
194            "一致" => Token::Match,
195            "戻る" | "帰る" => Token::Return,
196            "試す" => Token::Try,
197            "待つ" => Token::Wait,
198            "非同期" => Token::Async,
199            "起動" => Token::Spawn,
200            "止まれ" | "停め" => Token::Stop,
201            "継続" => Token::Again,
202            // Booleans — shared across Chinese, Japanese, Korean, Thai (already in Thai section)
203            "true" | "真" => Token::Bool(true),
204            "false" | "假" | "偽" => Token::Bool(false),
205            // Korean — full keyword set with short Hangul forms
206            "바인드" | "묶" => Token::Bind,
207            "실행" => Token::Do,
208            "함수" => Token::Fn,
209            "모듈" => Token::Mod,
210            "만약" | "조건" => Token::If,
211            "아니면" => Token::Else,
212            "동안" | "반복" => Token::While,
213            "위해" => Token::For,
214            "안에" => Token::In,
215            "매치" => Token::Match,
216            "반환" | "귀환" => Token::Return,
217            "시도" => Token::Try,
218            "기다려" => Token::Wait,
219            "비동기" => Token::Async,
220            "생성" => Token::Spawn,
221            "멈춤" => Token::Stop,
222            "계속" => Token::Again,
223            "참" => Token::Bool(true),
224            "거짓" => Token::Bool(false),
225            // Russian (русский) — full keyword set
226            "связать" => Token::Bind,
227            "сделать" => Token::Do,
228            "функция" => Token::Fn,
229            "модуль" => Token::Mod,
230            "тип" => Token::Type,
231            "использовать" => Token::Use,
232            "если" => Token::If,
233            "иначе" => Token::Else,
234            "пока" => Token::While,
235            "для" => Token::For,
236            "в" => Token::In,
237            "сопоставить" => Token::Match,
238            "вернуть" => Token::Return,
239            "владеть" => Token::Own,
240            "одолжить" => Token::Lend,
241            "делиться" => Token::Share,
242            "переместить" => Token::Move,
243            "копировать" => Token::Copy,
244            "асинхронно" => Token::Async,
245            "ждать" => Token::Wait,
246            "как" => Token::As,
247            "где" => Token::Where,
248            "опубликовать" => Token::Post,
249            "дать" => Token::Give,
250            "подходит" => Token::Fit,
251            "форма" => Token::Form,
252            "выбрать" => Token::Choose,
253            "может" => Token::Can,
254            "изменить" => Token::Change,
255            "стоп" => Token::Stop,
256            "снова" => Token::Again,
257            "пробовать" => Token::Try,
258            "уверен" => Token::Sure,
259            "возможно" => Token::Maybe,
260            "чистый" => Token::Pure,
261            "создать" => Token::Spawn,
262            "хорошо" => Token::Ok,
263            "плохо" => Token::Bad,
264            "ничего" => Token::None,
265            "истина" => Token::Bool(true),
266            "ложь" => Token::Bool(false),
267            // Thai
268            "ผูก" => Token::Bind,
269            "ทำ" => Token::Do,
270            "ฟังก์ชัน" => Token::Fn,
271            "โมดูล" => Token::Mod,
272            "ถ้า" => Token::If,
273            "มิฉะนั้น" => Token::Else,
274            "ขณะที่" => Token::While,
275            "สำหรับ" => Token::For,
276            "ใน" => Token::In,
277            "จับคู่" => Token::Match,
278            "คืน" => Token::Return,
279            "รอ" => Token::Wait,
280            "ไม่พร้อมกัน" => Token::Async,
281            "จริง" => Token::Bool(true),
282            "เท็จ" => Token::Bool(false),
283            // Hindi
284            "बाँधो" => Token::Bind,
285            "करो" => Token::Do,
286            "अगर" => Token::If,
287            "नहींतो" => Token::Else,
288            "जबकि" => Token::While,
289            "केलिए" => Token::For,
290            "वापस" => Token::Return,
291            "सत्य" => Token::Bool(true),
292            "असत्य" => Token::Bool(false),
293            // Arabic (العربية) — full keyword set
294            "ربط" => Token::Bind,
295            "افعل" => Token::Do,
296            "دالة" => Token::Fn,
297            "وحدة" => Token::Mod,
298            "نوع" => Token::Type,
299            "استخدم" => Token::Use,
300            "إذا" => Token::If,
301            "وإلا" => Token::Else,
302            "بينما" => Token::While,
303            "لأجل" => Token::For,
304            "في" => Token::In,
305            "طابق" => Token::Match,
306            "أعد" => Token::Return,
307            "امتلك" => Token::Own,
308            "أقرض" => Token::Lend,
309            "شارك" => Token::Share,
310            "انقل" => Token::Move,
311            "انسخ" => Token::Copy,
312            "غير_متزامن" => Token::Async,
313            "انتظر" => Token::Wait,
314            "بصفة" => Token::As,
315            "حيث" => Token::Where,
316            "انشر" => Token::Post,
317            "أعط" => Token::Give,
318            "يلائم" => Token::Fit,
319            "شكل" => Token::Form,
320            "اختر" => Token::Choose,
321            "يمكن" => Token::Can,
322            "غيّر" => Token::Change,
323            "توقف" => Token::Stop,
324            "مرة_أخرى" => Token::Again,
325            "حاول" => Token::Try,
326            "متأكد" => Token::Sure,
327            "ربما" => Token::Maybe,
328            "نقي" => Token::Pure,
329            "أنشئ" => Token::Spawn,
330            "تمام" => Token::Ok,
331            "سيء" => Token::Bad,
332            "لا_شيء" => Token::None,
333            "صحيح" => Token::Bool(true),
334            "خطأ" => Token::Bool(false),
335            // Persian / Farsi (فارسی) — full keyword set
336            "پیوند" => Token::Bind,
337            "انجام" => Token::Do,
338            "تابع" => Token::Fn,
339            "ماژول" => Token::Mod,
340            // type: shares Arabic's "نوع" above (common loanword in both)
341            "استفاده" => Token::Use,
342            "اگر" => Token::If,
343            "وگرنه" => Token::Else,
344            "هنگامی" => Token::While,
345            "برای" => Token::For,
346            "در" => Token::In,
347            "تطبیق" => Token::Match,
348            "بازگشت" => Token::Return,
349            "مالک" => Token::Own,
350            "قرض" => Token::Lend,
351            "اشتراک" => Token::Share,
352            "انتقال" => Token::Move,
353            "کپی" => Token::Copy,
354            "ناهمگام" => Token::Async,
355            "انتظار" => Token::Wait,
356            "به_عنوان" => Token::As,
357            "جایی_که" => Token::Where,
358            "ارسال" => Token::Post,
359            "بده" => Token::Give,
360            "تناسب" => Token::Fit,
361            "ساختار" => Token::Form,
362            "انتخاب" => Token::Choose,
363            "امکان" => Token::Can,
364            "تغییر" => Token::Change,
365            "دوباره" => Token::Again,
366            "تلاش" => Token::Try,
367            "مطمئن" => Token::Sure,
368            "شاید" => Token::Maybe,
369            "خالص" => Token::Pure,
370            "ایجاد" => Token::Spawn,
371            "تایید" => Token::Ok,
372            "هیچ" => Token::None,
373            "درست" => Token::Bool(true),
374            "نادرست" => Token::Bool(false),
375            // Hebrew (עברית) — full keyword set
376            "קשר" => Token::Bind,
377            "בצע" => Token::Do,
378            "פונקציה" => Token::Fn,
379            "מודול" => Token::Mod,
380            "סוג" => Token::Type,
381            "השתמש" => Token::Use,
382            "אם" => Token::If,
383            "אחרת" => Token::Else,
384            "כל_עוד" => Token::While,
385            "עבור" => Token::For,
386            "בתוך" => Token::In,
387            "התאמה" => Token::Match,
388            "החזר" => Token::Return,
389            "בעל" => Token::Own,
390            "השאלה" => Token::Lend,
391            "שתף" => Token::Share,
392            "הזז" => Token::Move,
393            "העתק" => Token::Copy,
394            "אסינכרוני" => Token::Async,
395            "המתן" => Token::Wait,
396            "בתור" => Token::As,
397            "היכן" => Token::Where,
398            "פרסם" => Token::Post,
399            "תן" => Token::Give,
400            "מתאים" => Token::Fit,
401            "צורה" => Token::Form,
402            "בחר" => Token::Choose,
403            "יכול" => Token::Can,
404            "שנה" => Token::Change,
405            "עצור" => Token::Stop,
406            "שוב" => Token::Again,
407            "נסה" => Token::Try,
408            "בטוח" => Token::Sure,
409            "אולי" => Token::Maybe,
410            "טהור" => Token::Pure,
411            "צור" => Token::Spawn,
412            "בסדר" => Token::Ok,
413            "רע" => Token::Bad,
414            "כלום" => Token::None,
415            "אמת" => Token::Bool(true),
416            "שקר" => Token::Bool(false),
417            // Urdu (اردو) — full keyword set
418            "باندھو" => Token::Bind,
419            "کرو" => Token::Do,
420            "تفاعل" => Token::Fn,
421            "ماڈیول" => Token::Mod,
422            "قسم" => Token::Type,
423            "استعمال" => Token::Use,
424            // if: shares Persian's "اگر" above (common word in both)
425            "ورنہ" => Token::Else,
426            "جب_تک" => Token::While,
427            "کے_لیے" => Token::For,
428            "میں" => Token::In,
429            "مطابقت" => Token::Match,
430            "واپس" => Token::Return,
431            // own: shares Persian's "مالک" above (common word in both)
432            "ادھار" => Token::Lend,
433            "شراکت" => Token::Share,
434            "منتقل" => Token::Move,
435            "نقل" => Token::Copy,
436            "غیر_ہمزمان" => Token::Async,
437            // wait: shares Persian's "انتظار" above (common word in both)
438            "بطور" => Token::As,
439            "جہاں" => Token::Where,
440            "شائع" => Token::Post,
441            "دو" => Token::Give,
442            "موزوں" => Token::Fit,
443            "شکل" => Token::Form,
444            "چنو" => Token::Choose,
445            "قابل" => Token::Can,
446            "تبدیل" => Token::Change,
447            "رکو" => Token::Stop,
448            "دوبارہ" => Token::Again,
449            "کوشش" => Token::Try,
450            "یقینی" => Token::Sure,
451            // maybe: shares Persian's "شاید" above (common word in both)
452            // pure: shares Persian's "خالص" above (common word in both)
453            "پیدا" => Token::Spawn,
454            "ٹھیک" => Token::Ok,
455            "برا" => Token::Bad,
456            "کچھ_نہیں" => Token::None,
457            "سچ" => Token::Bool(true),
458            "جھوٹ" => Token::Bool(false),
459            // Spanish
460            "enlazar" => Token::Bind,
461            "hacer" => Token::Do,
462            "si" => Token::If,
463            "sino" => Token::Else,
464            "mientras" => Token::While,
465            "para" => Token::For,
466            "retornar" => Token::Return,
467            "verdadero" => Token::Bool(true),
468            "falso" => Token::Bool(false),
469            // French (français) — full keyword set
470            "lier" => Token::Bind,
471            "faire" => Token::Do,
472            "func" | "fonction" => Token::Fn,
473            "module" => Token::Mod,
474            "sinon" => Token::Else,
475            "tantque" | "tant_que" => Token::While,
476            "pour" => Token::For,
477            "dans" => Token::In,
478            "correspondre" => Token::Match,
479            "retourner" => Token::Return,
480            "posséder" => Token::Own,
481            "prêter" => Token::Lend,
482            "partager" => Token::Share,
483            "déplacer" => Token::Move,
484            "copier" => Token::Copy,
485            "asynchrone" => Token::Async,
486            "attendre" => Token::Wait,
487            "comme" => Token::As,
488            "où" => Token::Where,
489            "publier" => Token::Post,
490            "donner" => Token::Give,
491            "convenir" => Token::Fit,
492            "forme" => Token::Form,
493            "choisir" => Token::Choose,
494            "peut" => Token::Can,
495            "changer" => Token::Change,
496            "arrêter" => Token::Stop,
497            "encore" => Token::Again,
498            "essayer" => Token::Try,
499            "sûr" => Token::Sure,
500            "peut_être" => Token::Maybe,
501            "pur" => Token::Pure,
502            "engendrer" => Token::Spawn,
503            "bon" => Token::Ok,
504            "mauvais" => Token::Bad,
505            "rien" => Token::None,
506            "vrai" => Token::Bool(true),
507            "faux" => Token::Bool(false),
508            // German (Deutsch) — full keyword set
509            "binden" => Token::Bind,
510            "machen" => Token::Do,
511            "funktion" => Token::Fn,
512            "modul" => Token::Mod,
513            "typ" => Token::Type,
514            "verwenden" => Token::Use,
515            "wenn" => Token::If,
516            "sonst" => Token::Else,
517            "solange" => Token::While,
518            "für" => Token::For,
519            "abgleichen" => Token::Match,
520            "zurück" => Token::Return,
521            "besitzen" => Token::Own,
522            "leihen" => Token::Lend,
523            "teilen" => Token::Share,
524            "bewegen" => Token::Move,
525            "kopieren" => Token::Copy,
526            "asynchron" => Token::Async,
527            "warten" => Token::Wait,
528            "als" => Token::As,
529            "wobei" => Token::Where,
530            "veröffentlichen" => Token::Post,
531            "geben" => Token::Give,
532            "passen" => Token::Fit,
533            "wählen" => Token::Choose,
534            "kann" => Token::Can,
535            "ändern" => Token::Change,
536            "stoppen" => Token::Stop,
537            "wieder" => Token::Again,
538            "versuchen" => Token::Try,
539            "sicher" => Token::Sure,
540            "vielleicht" => Token::Maybe,
541            "rein" => Token::Pure,
542            "erzeugen" => Token::Spawn,
543            "gut" => Token::Ok,
544            "schlecht" => Token::Bad,
545            "nichts" => Token::None,
546            "wahr" => Token::Bool(true),
547            "falsch" => Token::Bool(false),
548            // Portuguese (deduplicated from Spanish)
549            "ligar" => Token::Bind,
550            "fazer" => Token::Do,
551            "se" => Token::If,
552            "senão" => Token::Else,
553            "enquanto" => Token::While,
554            "verdadeiro" => Token::Bool(true),
555            // Everything else is an identifier
556            other => Token::Ident(other.to_string()),
557        }
558    }
559
560    pub fn next_token(&mut self) -> Option<Token> {
561        self.skip_whitespace_and_comments();
562        let ch = self.peek()?;
563
564        // String literal
565        if ch == '"' {
566            return Some(self.lex_string());
567        }
568
569        // Number literal
570        if ch.is_ascii_digit() {
571            return Some(self.lex_number());
572        }
573
574        // Word (keyword or identifier) — handles ASCII and all Unicode letters/ideographs
575        if ch.is_alphabetic() || ch == '_' || is_unicode_combining(ch) {
576            return Some(self.lex_word());
577        }
578
579        // Multi-character punctuation / operators
580        let rest = self.rest();
581
582        // `..`
583        if rest.starts_with("..") {
584            self.advance();
585            self.advance();
586            return Some(Token::DotDot);
587        }
588        // `::`
589        if rest.starts_with("::") {
590            self.advance();
591            self.advance();
592            return Some(Token::ColonColon);
593        }
594        // `==`
595        if rest.starts_with("==") {
596            self.advance();
597            self.advance();
598            return Some(Token::EqEq);
599        }
600        // `!=`
601        if rest.starts_with("!=") {
602            self.advance();
603            self.advance();
604            return Some(Token::Ne);
605        }
606        // `<=`
607        if rest.starts_with("<=") {
608            self.advance();
609            self.advance();
610            return Some(Token::Le);
611        }
612        // `>=`
613        if rest.starts_with(">=") {
614            self.advance();
615            self.advance();
616            return Some(Token::Ge);
617        }
618        // `->`
619        if rest.starts_with("->") {
620            self.advance();
621            self.advance();
622            return Some(Token::Arrow);
623        }
624        // `=>`
625        if rest.starts_with("=>") {
626            self.advance();
627            self.advance();
628            return Some(Token::FatArrow);
629        }
630        // `&&`
631        if rest.starts_with("&&") {
632            self.advance();
633            self.advance();
634            return Some(Token::And);
635        }
636        // `||`
637        if rest.starts_with("||") {
638            self.advance();
639            self.advance();
640            return Some(Token::Or);
641        }
642
643        // Single-character tokens
644        self.advance();
645        Some(match ch {
646            '=' => Token::Eq,
647            '<' => Token::Lt,
648            '>' => Token::Gt,
649            '!' => Token::Not,
650            '+' => Token::Plus,
651            '-' => Token::Minus,
652            '*' => Token::Star,
653            '/' => Token::Slash,
654            '%' => Token::Percent,
655            '.' => Token::Dot,
656            '&' => Token::Ampersand,
657            '(' => Token::LParen,
658            ')' => Token::RParen,
659            '{' => Token::LBrace,
660            '}' => Token::RBrace,
661            '[' => Token::LBracket,
662            ']' => Token::RBracket,
663            ',' => Token::Comma,
664            ':' => Token::Colon,
665            ';' => Token::Semicolon,
666            '|' => Token::Or, // single | treated as Or (for closure context)
667            c => Token::Error(c.to_string()),
668        })
669    }
670}
671
672/// Returns true for Unicode combining marks (Mn/Mc category) that Rust's
673/// `is_alphabetic()` may miss — Thai tone marks, Devanagari vowel signs,
674/// Arabic diacritics, Hebrew cantillation, etc.
675fn is_unicode_combining(c: char) -> bool {
676    let cp = c as u32;
677    matches!(cp,
678        0x0300..=0x036F |   // Combining Diacritical Marks (Latin)
679        0x0483..=0x0489 |   // Combining Cyrillic
680        0x0591..=0x05C7 |   // Hebrew cantillation / points
681        0x0610..=0x061A |   // Arabic extended
682        0x064B..=0x065F |   // Arabic diacritics (harakat)
683        0x0670         |    // Arabic superscript alef
684        0x06D6..=0x06DC |   // Arabic small high letters
685        0x0730..=0x074A |   // Syriac diacritics
686        0x0816..=0x082D |   // Samaritan diacritics
687        0x0900..=0x0903 |   // Devanagari (anusvara, visarga)
688        0x093A..=0x094F |   // Devanagari vowel signs / halant
689        0x0951..=0x0957 |   // Devanagari stress marks
690        0x0962..=0x0963 |   // Devanagari vowel signs
691        0x0981..=0x0983 |   // Bengali
692        0x09BC         |    // Bengali nukta
693        0x09BE..=0x09C4 |   // Bengali vowel signs
694        0x09C7..=0x09C8 |   // Bengali vowel signs
695        0x09CB..=0x09CD |   // Bengali
696        0x0A01..=0x0A03 |   // Gurmukhi
697        0x0A3C         |    // Gurmukhi nukta
698        0x0A3E..=0x0A42 |   // Gurmukhi vowels
699        0x0B01..=0x0B03 |   // Oriya
700        0x0B3C..=0x0B4D |   // Oriya
701        0x0C00..=0x0C03 |   // Telugu
702        0x0C3E..=0x0C56 |   // Telugu vowels
703        0x0D00..=0x0D03 |   // Malayalam
704        0x0D3B..=0x0D4D |   // Malayalam vowels / chandrakkala
705        0x0E31         |    // Thai SARA AM
706        0x0E34..=0x0E3A |   // Thai vowel signs
707        0x0E47..=0x0E4E |   // Thai tone marks & other signs
708        0x0EB1         |    // Lao vowel sign
709        0x0EB4..=0x0EBC |   // Lao vowel signs
710        0x0EC8..=0x0ECD |   // Lao tone marks
711        0x3099..=0x309A |   // Japanese combining dakuten
712        0xFE20..=0xFE2F     // Combining Half Marks
713    )
714}
ling/lexer/mod.rs

ling/lexer/
mod.rs
