qql_core/token.rs
1use core::fmt;
2
3use crate::error::Span;
4
5// ── Token kind definitions ─────────────────────────────────────
6// All variants are listed ONCE in the `token_table!`.
7// Two helper macros consume that table to produce:
8// • as_str() – for all variants
9// • KEYWORDS map – for keyword-only variants
10
11macro_rules! gen_as_str {
12 ($($var:ident => $str:expr),* $(,)?) => {
13 impl TokenKind {
14 /// Returns the canonical display string for this token kind.
15 pub fn as_str(&self) -> &'static str {
16 match self {
17 $( Self::$var => $str, )*
18 }
19 }
20 }
21 };
22}
23
24/// The lexical token kinds produced by the QQL lexer.
25#[derive(Debug, Clone, Copy, PartialEq, Eq)]
26#[cfg_attr(feature = "serde", derive(serde::Serialize))]
27pub enum TokenKind {
28 /// The `ABS` keyword.
29 Abs,
30 /// The `ACORN` keyword.
31 Acorn,
32 /// The `ACOSH` keyword.
33 Acosh,
34 /// The `AFTER` keyword.
35 After,
36 /// The `ALL` keyword.
37 All,
38 /// The `ALTER` keyword.
39 Alter,
40 /// The `AND` keyword.
41 And,
42 /// The `ANY` keyword.
43 Any,
44 /// The `AS` keyword.
45 As,
46 /// The `ASC` keyword.
47 Asc,
48 /// The `AVERAGE_VECTOR` keyword.
49 AverageVector,
50 /// The `BATCH` keyword.
51 Batch,
52 /// The `BEST_SCORE` keyword.
53 BestScore,
54 /// The `BETWEEN` keyword.
55 Between,
56 /// The `BOOL` keyword.
57 Bool,
58 /// The `BOTTOM_RIGHT` keyword.
59 BottomRight,
60 /// The `BY` keyword.
61 By,
62 /// The `CANDIDATES` keyword.
63 Candidates,
64 /// The `CASE` keyword.
65 Case,
66 /// The `CENTER` keyword.
67 Center,
68 /// The `CLEAR` keyword.
69 Clear,
70 /// The `COLLECTION` keyword.
71 Collection,
72 /// The `COLLECTIONS` keyword.
73 Collections,
74 /// The `CONSISTENCY` keyword.
75 Consistency,
76 /// The `CONTEXT` keyword.
77 Context,
78 /// The `COSINE` keyword.
79 Cosine,
80 /// The `COUNT` keyword.
81 Count,
82 /// The `CREATE` keyword.
83 Create,
84 /// The `CROSS` keyword.
85 Cross,
86 /// The `DATETIME` keyword.
87 Datetime,
88 /// The `DATETIME_KEY` keyword.
89 DatetimeKey,
90 /// The `DBSF` keyword.
91 Dbsf,
92 /// The `DECAY` keyword.
93 Decay,
94 /// The `DEFAULT` keyword.
95 Default,
96 /// The `DEFAULTS` keyword.
97 Defaults,
98 /// The `DELETE` keyword.
99 Delete,
100 /// The `DENSE` keyword.
101 Dense,
102 /// The `DESC` keyword.
103 Desc,
104 /// The `DISCOVER` keyword.
105 Discover,
106 /// The `DIVERSITY` keyword.
107 Diversity,
108 /// The `DOT` keyword.
109 Dot,
110 /// The `DROP` keyword.
111 Drop,
112 /// The `ELSE` keyword.
113 Else,
114 /// The `EMBED` keyword.
115 Embed,
116 /// The `EMPTY` keyword.
117 Empty,
118 /// The `END` keyword.
119 End,
120 /// The `EUCLID` keyword.
121 Euclid,
122 /// The `EXCEPT` keyword (used by `MATCH EXCEPT`).
123 Except,
124 /// The `EXACT` keyword.
125 Exact,
126 /// The `FACET` keyword.
127 Facet,
128 /// The `EXCLUDE` keyword.
129 Exclude,
130 /// The `EXP` keyword.
131 Exp,
132 /// The `EXP_DECAY` keyword.
133 ExpDecay,
134 /// The `EXTERIOR` keyword.
135 Exterior,
136 /// The `FALSE` keyword.
137 False,
138 /// The `FEEDBACK` keyword.
139 Feedback,
140 /// The `FIELD` keyword.
141 Field,
142 /// The `FILTER` keyword (upsert `UPDATE FILTER` guard).
143 Filter,
144 /// A float literal token; also the `FLOAT` field-type keyword.
145 Float,
146 /// The `FOR` keyword.
147 For,
148 /// The `FORMULA` keyword.
149 Formula,
150 /// The `FROM` keyword.
151 From,
152 /// The `FUSION` keyword.
153 Fusion,
154 /// The `GAUSS_DECAY` keyword.
155 GaussDecay,
156 /// The `GEO` keyword.
157 Geo,
158 /// The `GEO_BBOX` keyword.
159 GeoBbox,
160 /// The `GEO_DISTANCE` keyword.
161 GeoDistance,
162 /// The `GEO_POLYGON` keyword.
163 GeoPolygon,
164 /// The `GEO_RADIUS` keyword.
165 GeoRadius,
166 /// The `GLOBAL` keyword.
167 Global,
168 /// The `GROUP` keyword.
169 Group,
170 /// The `HAS_VECTOR` keyword.
171 HasVector,
172 /// The `HNSW` keyword.
173 Hnsw,
174 /// The `HNSW_EF` keyword.
175 HnswEf,
176 /// The `HYBRID` keyword.
177 Hybrid,
178 /// The `ID` keyword.
179 Id,
180 /// The `IDF` keyword.
181 Idf,
182 /// The `IGNORE` keyword.
183 Ignore,
184 /// The `IMAGE` keyword.
185 Image,
186 /// The `IN` keyword.
187 In,
188 /// The `INCLUDE` keyword.
189 Include,
190 /// The `INDEX` keyword.
191 Index,
192 /// The `INSERT_ONLY` keyword (upsert `UPDATE MODE` value).
193 InsertOnly,
194 /// The `INDEXED_ONLY` keyword.
195 IndexedOnly,
196 /// The `INDICES` keyword.
197 Indices,
198 /// An integer literal token; also the `INTEGER` field-type keyword.
199 Integer,
200 /// The `INTERIORS` keyword.
201 Interiors,
202 /// The `INTO` keyword.
203 Into,
204 /// The `IS` keyword.
205 Is,
206 /// The `KEY` keyword.
207 Key,
208 /// The `KEYS` keyword.
209 Keys,
210 /// The `KEYWORD` keyword.
211 Keyword,
212 /// The `LAT` keyword.
213 Lat,
214 /// The `LIMIT` keyword.
215 Limit,
216 /// The `LIN_DECAY` keyword.
217 LinDecay,
218 /// The `LN` keyword.
219 Ln,
220 /// The `LOG` keyword.
221 Log,
222 /// The `LON` keyword.
223 Lon,
224 /// The `LOOKUP` keyword.
225 Lookup,
226 /// The `MAJORITY` keyword.
227 Majority,
228 /// The `MANHATTAN` keyword.
229 Manhattan,
230 /// The `MATCH` keyword.
231 Match,
232 /// The `MATCH_ANY` keyword.
233 MatchAny,
234 /// The `MAX` keyword.
235 Max,
236 /// The `MAX_SELECTIVITY` keyword.
237 MaxSelectivity,
238 /// The `METADATA` keyword.
239 Metadata,
240 /// The `MIDPOINT` keyword.
241 Midpoint,
242 /// The `MIN` keyword.
243 Min,
244 /// The `MMR` keyword.
245 Mmr,
246 /// The `MODEL` keyword.
247 Model,
248 /// The `MODE` keyword (upsert `UPDATE MODE` guard).
249 Mode,
250 /// The `MULTI` keyword.
251 Multi,
252 /// The `MULTIVECTOR` keyword.
253 Multivector,
254 /// The `NAIVE` keyword.
255 Naive,
256 /// The `NEAREST` keyword.
257 Nearest,
258 /// The `NEGATIVE` keyword.
259 Negative,
260 /// The `NESTED` keyword.
261 Nested,
262 /// The `NOT` keyword.
263 Not,
264 /// The `NULL` keyword.
265 Null,
266 /// The `OBJECT` keyword.
267 Object,
268 /// The `OFFSET` keyword.
269 Offset,
270 /// The `ON` keyword.
271 On,
272 /// The `OPTIMIZERS` keyword.
273 Optimizers,
274 /// The `OPTIONS` keyword.
275 Options,
276 /// The `OR` keyword.
277 Or,
278 /// The `ORDER` keyword.
279 Order,
280 /// The `OVERWRITE` keyword (payload `OVERWRITE` flag).
281 Overwrite,
282 /// The `OVERSAMPLING` keyword.
283 Oversampling,
284 /// The `PARAMS` keyword.
285 Params,
286 /// The `PAYLOAD` keyword.
287 Payload,
288 /// The `PHRASE` keyword.
289 Phrase,
290 /// The `POINT` keyword.
291 Point,
292 /// The `POINTS` keyword.
293 Points,
294 /// The `POSITIVE` keyword.
295 Positive,
296 /// The `POW` keyword.
297 Pow,
298 /// The `PREFETCH` keyword.
299 Prefetch,
300 /// The `PREFIX` keyword.
301 Prefix,
302 /// The `QUANTIZATION` keyword.
303 Quantization,
304 /// The `QUERY` keyword.
305 Query,
306 /// The `QUOTA` keyword.
307 Quota,
308 /// The `QUOTAS` keyword.
309 Quotas,
310 /// The `QUORUM` keyword.
311 Quorum,
312 /// The `RADIUS` keyword.
313 Radius,
314 /// The `RANDOM` keyword.
315 Random,
316 /// The `RECOMMEND` keyword.
317 Recommend,
318 /// The `RELEVANCE` keyword.
319 Relevance,
320 /// The `RERANK` keyword.
321 Rerank,
322 /// The `RESCORE` keyword.
323 Rescore,
324 /// The `RRF` keyword.
325 Rrf,
326 /// The `RRF_K` keyword.
327 RrfK,
328 /// The `RRF_WEIGHTS` keyword.
329 RrfWeights,
330 /// The `SAMPLE` keyword.
331 Sample,
332 /// The `SCALE` keyword.
333 Scale,
334 /// The `SCORE` keyword.
335 Score,
336 /// The `SCROLL` keyword.
337 Scroll,
338 /// The `SET` keyword.
339 Set,
340 /// The `SHARD` keyword.
341 Shard,
342 /// The `SHOW` keyword.
343 Show,
344 /// The `SHOULD` keyword (used by `MIN SHOULD`).
345 Should,
346 /// The `SIZE` keyword.
347 Size,
348 /// The `SLICE` keyword.
349 Slice,
350 /// The `SPARSE` keyword.
351 Sparse,
352 /// The `SQRT` keyword.
353 Sqrt,
354 /// The `START` keyword (`ORDER BY … START FROM <value>` paging origin).
355 Start,
356 /// The `STRATEGY` keyword.
357 Strategy,
358 /// The `STRICT_MODE` keyword.
359 StrictMode,
360 /// The `SUM_SCORES` keyword.
361 SumScores,
362 /// The `TARGET` keyword.
363 Target,
364 /// The `TEXT` keyword.
365 Text,
366 /// The `THEN` keyword.
367 Then,
368 /// The `THRESHOLD` keyword.
369 Threshold,
370 /// The `TIMEOUT` keyword.
371 Timeout,
372 /// The `TOKENS` keyword (used by `MATCH TOKENS`).
373 Tokens,
374 /// The `TOP_LEFT` keyword.
375 TopLeft,
376 /// The `TRUE` keyword.
377 True,
378 /// The `TYPE` keyword.
379 Type,
380 /// The `UPDATE` keyword.
381 Update,
382 /// The `UPDATE_ONLY` keyword (upsert `UPDATE MODE` value).
383 UpdateOnly,
384 /// The `UPSERT` keyword.
385 Upsert,
386 /// The `USING` keyword.
387 Using,
388 /// The `UUID` keyword.
389 Uuid,
390 /// The `VALUES` keyword.
391 Values,
392 /// The `VALUES_COUNT` keyword.
393 ValuesCount,
394 /// The `VECTOR` keyword.
395 Vector,
396 /// The `WAIT` keyword.
397 Wait,
398 /// The `WAL` keyword.
399 Wal,
400 /// The `WHEN` keyword.
401 When,
402 /// The `WHERE` keyword.
403 Where,
404 /// The `WITH` keyword.
405 With,
406 /// A `*` token (wildcard or multiplication).
407 Star,
408 /// An identifier token (field, collection, or parameter name).
409 Identifier,
410 /// A string literal token (quoted, raw, backtick, or triple-quoted).
411 String,
412 /// A `{` token.
413 Lbrace,
414 /// A `}` token.
415 Rbrace,
416 /// A `[` token.
417 Lbracket,
418 /// A `]` token.
419 Rbracket,
420 /// A `(` token.
421 Lparen,
422 /// A `)` token.
423 Rparen,
424 /// A `:` token.
425 Colon,
426 /// A `,` token.
427 Comma,
428 /// An `=` token.
429 Equals,
430 /// A `!=` token.
431 NotEquals,
432 /// A `>` token.
433 Gt,
434 /// A `>=` token.
435 Gte,
436 /// A `<` token.
437 Lt,
438 /// A `<=` token.
439 Lte,
440 /// A `+` token.
441 Plus,
442 /// A `-` token.
443 Minus,
444 /// A `/` token.
445 Slash,
446 /// A `;` token.
447 Semicolon,
448 /// A `?` token.
449 Question,
450 /// The end-of-input token.
451 Eof,
452}
453
454gen_as_str! {
455 Abs => "ABS",
456 Acorn => "ACORN",
457 Acosh => "ACOSH",
458 After => "AFTER",
459 All => "ALL",
460 Alter => "ALTER",
461 And => "AND",
462 Any => "ANY",
463 As => "AS",
464 Asc => "ASC",
465 AverageVector => "AVERAGE_VECTOR",
466 Batch => "BATCH",
467 BestScore => "BEST_SCORE",
468 Between => "BETWEEN",
469 Bool => "BOOL",
470 BottomRight => "BOTTOM_RIGHT",
471 By => "BY",
472 Candidates => "CANDIDATES",
473 Case => "CASE",
474 Center => "CENTER",
475 Clear => "CLEAR",
476 Collection => "COLLECTION",
477 Collections => "COLLECTIONS",
478 Consistency => "CONSISTENCY",
479 Context => "CONTEXT",
480 Cosine => "COSINE",
481 Count => "COUNT",
482 Create => "CREATE",
483 Cross => "CROSS",
484 Datetime => "DATETIME",
485 DatetimeKey => "DATETIME_KEY",
486 Dbsf => "DBSF",
487 Decay => "DECAY",
488 Default => "DEFAULT",
489 Defaults => "DEFAULTS",
490 Delete => "DELETE",
491 Dense => "DENSE",
492 Desc => "DESC",
493 Discover => "DISCOVER",
494 Diversity => "DIVERSITY",
495 Dot => "DOT",
496 Drop => "DROP",
497 Else => "ELSE",
498 Embed => "EMBED",
499 Empty => "EMPTY",
500 End => "END",
501 Euclid => "EUCLID",
502 Except => "EXCEPT",
503 Exact => "EXACT",
504 Facet => "FACET",
505 Exclude => "EXCLUDE",
506 Exp => "EXP",
507 ExpDecay => "EXP_DECAY",
508 Exterior => "EXTERIOR",
509 False => "FALSE",
510 Feedback => "FEEDBACK",
511 Field => "FIELD",
512 Filter => "FILTER",
513 Float => "FLOAT",
514 For => "FOR",
515 Formula => "FORMULA",
516 From => "FROM",
517 Fusion => "FUSION",
518 GaussDecay => "GAUSS_DECAY",
519 Geo => "GEO",
520 GeoBbox => "GEO_BBOX",
521 GeoDistance => "GEO_DISTANCE",
522 GeoPolygon => "GEO_POLYGON",
523 GeoRadius => "GEO_RADIUS",
524 Global => "GLOBAL",
525 Group => "GROUP",
526 HasVector => "HAS_VECTOR",
527 Hnsw => "HNSW",
528 HnswEf => "HNSW_EF",
529 Hybrid => "HYBRID",
530 Id => "ID",
531 Idf => "IDF",
532 Ignore => "IGNORE",
533 Image => "IMAGE",
534 In => "IN",
535 Include => "INCLUDE",
536 Index => "INDEX",
537 InsertOnly => "INSERT_ONLY",
538 IndexedOnly => "INDEXED_ONLY",
539 Indices => "INDICES",
540 Integer => "INTEGER",
541 Interiors => "INTERIORS",
542 Into => "INTO",
543 Is => "IS",
544 Key => "KEY",
545 Keys => "KEYS",
546 Keyword => "KEYWORD",
547 Lat => "LAT",
548 Limit => "LIMIT",
549 LinDecay => "LIN_DECAY",
550 Ln => "LN",
551 Log => "LOG",
552 Lon => "LON",
553 Lookup => "LOOKUP",
554 Majority => "MAJORITY",
555 Manhattan => "MANHATTAN",
556 Match => "MATCH",
557 MatchAny => "MATCH_ANY",
558 Max => "MAX",
559 MaxSelectivity => "MAX_SELECTIVITY",
560 Metadata => "METADATA",
561 Midpoint => "MIDPOINT",
562 Min => "MIN",
563 Mmr => "MMR",
564 Model => "MODEL",
565 Mode => "MODE",
566 Multi => "MULTI",
567 Multivector => "MULTIVECTOR",
568 Naive => "NAIVE",
569 Nearest => "NEAREST",
570 Negative => "NEGATIVE",
571 Nested => "NESTED",
572 Not => "NOT",
573 Null => "NULL",
574 Object => "OBJECT",
575 Offset => "OFFSET",
576 On => "ON",
577 Optimizers => "OPTIMIZERS",
578 Options => "OPTIONS",
579 Or => "OR",
580 Order => "ORDER",
581 Overwrite => "OVERWRITE",
582 Oversampling => "OVERSAMPLING",
583 Params => "PARAMS",
584 Payload => "PAYLOAD",
585 Phrase => "PHRASE",
586 Point => "POINT",
587 Points => "POINTS",
588 Positive => "POSITIVE",
589 Pow => "POW",
590 Prefetch => "PREFETCH",
591 Prefix => "PREFIX",
592 Quantization => "QUANTIZATION",
593 Query => "QUERY",
594 Quota => "QUOTA",
595 Quotas => "QUOTAS",
596 Quorum => "QUORUM",
597 Radius => "RADIUS",
598 Random => "RANDOM",
599 Recommend => "RECOMMEND",
600 Relevance => "RELEVANCE",
601 Rerank => "RERANK",
602 Rescore => "RESCORE",
603 Rrf => "RRF",
604 RrfK => "RRF_K",
605 RrfWeights => "RRF_WEIGHTS",
606 Sample => "SAMPLE",
607 Scale => "SCALE",
608 Score => "SCORE",
609 Scroll => "SCROLL",
610 Set => "SET",
611 Shard => "SHARD",
612 Show => "SHOW",
613 Should => "SHOULD",
614 Size => "SIZE",
615 Slice => "SLICE",
616 Sparse => "SPARSE",
617 Sqrt => "SQRT",
618 Start => "START",
619 Strategy => "STRATEGY",
620 StrictMode => "STRICT_MODE",
621 SumScores => "SUM_SCORES",
622 Target => "TARGET",
623 Text => "TEXT",
624 Then => "THEN",
625 Threshold => "THRESHOLD",
626 Timeout => "TIMEOUT",
627 Tokens => "TOKENS",
628 TopLeft => "TOP_LEFT",
629 True => "TRUE",
630 Type => "TYPE",
631 Update => "UPDATE",
632 UpdateOnly => "UPDATE_ONLY",
633 Upsert => "UPSERT",
634 Using => "USING",
635 Uuid => "UUID",
636 Values => "VALUES",
637 ValuesCount => "VALUES_COUNT",
638 Vector => "VECTOR",
639 Wait => "WAIT",
640 Wal => "WAL",
641 When => "WHEN",
642 Where => "WHERE",
643 With => "WITH",
644 Star => "*",
645 Identifier => "IDENTIFIER",
646 String => "STRING",
647 Lbrace => "LBRACE",
648 Rbrace => "RBRACE",
649 Lbracket => "LBRACKET",
650 Rbracket => "RBRACKET",
651 Lparen => "LPAREN",
652 Rparen => "RPAREN",
653 Colon => "COLON",
654 Comma => "COMMA",
655 Equals => "EQUALS",
656 NotEquals => "NOT_EQUALS",
657 Gt => "GT",
658 Gte => "GTE",
659 Lt => "LT",
660 Lte => "LTE",
661 Plus => "PLUS",
662 Minus => "MINUS",
663 Slash => "SLASH",
664 Semicolon => "SEMICOLON",
665 Question => "QUESTION",
666 Eof => "EOF",
667}
668
669include!("keywords.generated.rs");
670
671impl TokenKind {
672 /// Returns true for keyword and identifier kinds.
673 ///
674 /// Numeric kinds (`Integer` / `Float`) are excluded here: the word
675 /// spellings `FLOAT` / `INTEGER` (field-type keywords mapped onto those
676 /// kinds) are recovered by [`Token::is_keyword_or_identifier`].
677 pub fn is_keyword_or_identifier(&self) -> bool {
678 !matches!(
679 self,
680 Self::String
681 | Self::Integer
682 | Self::Float
683 | Self::Lbrace
684 | Self::Rbrace
685 | Self::Lbracket
686 | Self::Rbracket
687 | Self::Lparen
688 | Self::Rparen
689 | Self::Colon
690 | Self::Comma
691 | Self::Equals
692 | Self::NotEquals
693 | Self::Gt
694 | Self::Gte
695 | Self::Lt
696 | Self::Lte
697 | Self::Plus
698 | Self::Minus
699 | Self::Star
700 | Self::Slash
701 | Self::Semicolon
702 | Self::Question
703 | Self::Eof
704 )
705 }
706
707 /// Keywords that open a top-level statement (`parse_stmt` match arms).
708 pub fn is_statement_start(&self) -> bool {
709 matches!(
710 self,
711 Self::Create
712 | Self::Alter
713 | Self::Drop
714 | Self::Show
715 | Self::Upsert
716 | Self::Scroll
717 | Self::Query
718 | Self::With
719 | Self::Delete
720 | Self::Clear
721 | Self::Update
722 | Self::Count
723 | Self::Facet
724 | Self::Set
725 )
726 }
727
728 /// Panic-mode sync tokens after a parse error.
729 ///
730 /// `WITH` is omitted: it is both a CTE opener and a clause (`WITH PAYLOAD`,
731 /// `WITH PARAMS`), so treating it as a boundary would split a query at the
732 /// clause. Missing-semicolon recovery still uses [`Self::is_statement_start`].
733 pub fn is_recovery_sync(&self) -> bool {
734 self.is_statement_start() && *self != Self::With
735 }
736}
737
738impl fmt::Display for TokenKind {
739 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
740 f.write_str(self.as_str())
741 }
742}
743
744/// A lexed token: its kind, source text, and byte-offset span.
745#[derive(Debug, Clone, Copy, PartialEq, Eq)]
746#[cfg_attr(feature = "serde", derive(serde::Serialize))]
747pub struct Token<'a> {
748 /// The token kind.
749 pub kind: TokenKind,
750 /// The raw source text of the token.
751 pub text: &'a str,
752 /// The byte-offset span of the token in the source input.
753 pub span: Span,
754 #[cfg_attr(feature = "serde", serde(skip))]
755 pub(crate) pos: usize,
756}
757
758impl<'a> Token<'a> {
759 /// Creates a token from a kind, source text, and span.
760 pub fn new(kind: TokenKind, text: &'a str, span: Span) -> Self {
761 Token {
762 kind,
763 text,
764 pos: span.start,
765 span,
766 }
767 }
768
769 /// Creates the end-of-input token at the given byte offset.
770 pub fn eof(position: usize) -> Self {
771 Token {
772 kind: TokenKind::Eof,
773 text: "",
774 span: Span::point(position),
775 pos: position,
776 }
777 }
778
779 /// Returns true when this token can stand in for a keyword or identifier.
780 ///
781 /// `FLOAT` / `INTEGER` field-type keywords share kinds with numeric
782 /// literals; they count as identifiers only when the source text starts
783 /// with a letter.
784 pub fn is_keyword_or_identifier(&self) -> bool {
785 match self.kind {
786 TokenKind::Integer | TokenKind::Float => self
787 .text
788 .bytes()
789 .next()
790 .is_some_and(|b| b.is_ascii_alphabetic()),
791 other => other.is_keyword_or_identifier(),
792 }
793 }
794}
795
796impl<'a> fmt::Display for Token<'a> {
797 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
798 write!(f, "{}({})", self.kind, self.text)
799 }
800}
801
802/// Resolves a word to its keyword token kind, matching ASCII case-insensitively.
803pub fn lookup_keyword(s: &str) -> Option<TokenKind> {
804 let bytes = s.as_bytes();
805 let len = bytes.len();
806 if len == 0 || len > 32 {
807 return None;
808 }
809 let mut buf = [0u8; 32];
810 for (i, b) in bytes.iter().enumerate() {
811 buf[i] = b.to_ascii_uppercase();
812 }
813 let upper = core::str::from_utf8(&buf[..len]).ok()?;
814 KEYWORDS.get(upper).copied()
815}