qql_core/
token.rs

1use core::fmt;
2
3use crate::error::Span;
4
5// ── Token kind definitions ─────────────────────────────────────
6// All variants are listed ONCE in the `token_table!`.
7// Two helper macros consume that table to produce:
8//   • as_str()     – for all variants
9//   • KEYWORDS map – for keyword-only variants
10
11macro_rules! gen_as_str {
12    ($($var:ident => $str:expr),* $(,)?) => {
13        impl TokenKind {
14            pub fn as_str(&self) -> &'static str {
15                match self {
16                    $( Self::$var => $str, )*
17                }
18            }
19        }
20    };
21}
22
23#[derive(Debug, Clone, Copy, PartialEq, Eq)]
24#[cfg_attr(feature = "serde", derive(serde::Serialize))]
25pub enum TokenKind {
26    Abs,
27    Acorn,
28    After,
29    All,
30    Alter,
31    And,
32    Any,
33    As,
34    Asc,
35    AverageVector,
36    BestScore,
37    Between,
38    Bool,
39    BottomRight,
40    By,
41    Candidates,
42    Case,
43    Center,
44    Clear,
45    Collection,
46    Collections,
47    Consistency,
48    Context,
49    Cosine,
50    Count,
51    Create,
52    Cross,
53    Datetime,
54    DatetimeKey,
55    Dbsf,
56    Decay,
57    Default,
58    Defaults,
59    Delete,
60    Dense,
61    Desc,
62    Discover,
63    Diversity,
64    Dot,
65    Drop,
66    Else,
67    Embed,
68    Empty,
69    End,
70    Euclid,
71    Exact,
72    Exclude,
73    Exp,
74    ExpDecay,
75    Exterior,
76    False,
77    Feedback,
78    Field,
79    Float,
80    For,
81    Formula,
82    From,
83    Fusion,
84    GaussDecay,
85    Geo,
86    GeoBbox,
87    GeoDistance,
88    GeoPolygon,
89    GeoRadius,
90    Group,
91    HasVector,
92    Hnsw,
93    HnswEf,
94    Hybrid,
95    Id,
96    Ignore,
97    Image,
98    In,
99    Include,
100    Index,
101    IndexedOnly,
102    Indices,
103    Integer,
104    Interiors,
105    Into,
106    Is,
107    Key,
108    Keys,
109    Keyword,
110    Lat,
111    Limit,
112    LinDecay,
113    Ln,
114    Log,
115    Lon,
116    Lookup,
117    Majority,
118    Manhattan,
119    Match,
120    MatchAny,
121    MaxSelectivity,
122    Midpoint,
123    Mmr,
124    Model,
125    Multi,
126    Multivector,
127    Naive,
128    Nearest,
129    Negative,
130    Nested,
131    Not,
132    Null,
133    Offset,
134    On,
135    Optimizers,
136    Or,
137    Order,
138    Oversampling,
139    Params,
140    Payload,
141    Phrase,
142    Point,
143    Points,
144    Positive,
145    Pow,
146    Prefetch,
147    Quantization,
148    Query,
149    Quorum,
150    Radius,
151    Random,
152    Recommend,
153    Relevance,
154    Rerank,
155    Rescore,
156    Rrf,
157    RrfK,
158    RrfWeights,
159    Sample,
160    Scale,
161    Score,
162    Scroll,
163    Set,
164    Shard,
165    Show,
166    Size,
167    Sparse,
168    Sqrt,
169    Strategy,
170    SumScores,
171    Target,
172    Text,
173    Then,
174    Threshold,
175    Timeout,
176    TopLeft,
177    True,
178    Type,
179    Update,
180    Upsert,
181    Using,
182    Uuid,
183    Values,
184    ValuesCount,
185    Vector,
186    When,
187    Where,
188    With,
189    Star,
190    Identifier,
191    String,
192    Lbrace,
193    Rbrace,
194    Lbracket,
195    Rbracket,
196    Lparen,
197    Rparen,
198    Colon,
199    Comma,
200    Equals,
201    NotEquals,
202    Gt,
203    Gte,
204    Lt,
205    Lte,
206    Plus,
207    Minus,
208    Slash,
209    Semicolon,
210    Eof,
211}
212
213gen_as_str! {
214    Abs => "ABS",
215    Acorn => "ACORN",
216    After => "AFTER",
217    All => "ALL",
218    Alter => "ALTER",
219    And => "AND",
220    Any => "ANY",
221    As => "AS",
222    Asc => "ASC",
223    AverageVector => "AVERAGE_VECTOR",
224    BestScore => "BEST_SCORE",
225    Between => "BETWEEN",
226    Bool => "BOOL",
227    BottomRight => "BOTTOM_RIGHT",
228    By => "BY",
229    Candidates => "CANDIDATES",
230    Case => "CASE",
231    Center => "CENTER",
232    Clear => "CLEAR",
233    Collection => "COLLECTION",
234    Collections => "COLLECTIONS",
235    Consistency => "CONSISTENCY",
236    Context => "CONTEXT",
237    Cosine => "COSINE",
238    Count => "COUNT",
239    Create => "CREATE",
240    Cross => "CROSS",
241    Datetime => "DATETIME",
242    DatetimeKey => "DATETIME_KEY",
243    Dbsf => "DBSF",
244    Decay => "DECAY",
245    Default => "DEFAULT",
246    Defaults => "DEFAULTS",
247    Delete => "DELETE",
248    Dense => "DENSE",
249    Desc => "DESC",
250    Discover => "DISCOVER",
251    Diversity => "DIVERSITY",
252    Dot => "DOT",
253    Drop => "DROP",
254    Else => "ELSE",
255    Embed => "EMBED",
256    Empty => "EMPTY",
257    End => "END",
258    Euclid => "EUCLID",
259    Exact => "EXACT",
260    Exclude => "EXCLUDE",
261    Exp => "EXP",
262    ExpDecay => "EXP_DECAY",
263    Exterior => "EXTERIOR",
264    False => "FALSE",
265    Feedback => "FEEDBACK",
266    Field => "FIELD",
267    Float => "FLOAT",
268    For => "FOR",
269    Formula => "FORMULA",
270    From => "FROM",
271    Fusion => "FUSION",
272    GaussDecay => "GAUSS_DECAY",
273    Geo => "GEO",
274    GeoBbox => "GEO_BBOX",
275    GeoDistance => "GEO_DISTANCE",
276    GeoPolygon => "GEO_POLYGON",
277    GeoRadius => "GEO_RADIUS",
278    Group => "GROUP",
279    HasVector => "HAS_VECTOR",
280    Hnsw => "HNSW",
281    HnswEf => "HNSW_EF",
282    Hybrid => "HYBRID",
283    Id => "ID",
284    Ignore => "IGNORE",
285    Image => "IMAGE",
286    In => "IN",
287    Include => "INCLUDE",
288    Index => "INDEX",
289    IndexedOnly => "INDEXED_ONLY",
290    Indices => "INDICES",
291    Integer => "INTEGER",
292    Interiors => "INTERIORS",
293    Into => "INTO",
294    Is => "IS",
295    Key => "KEY",
296    Keys => "KEYS",
297    Keyword => "KEYWORD",
298    Lat => "LAT",
299    Limit => "LIMIT",
300    LinDecay => "LIN_DECAY",
301    Ln => "LN",
302    Log => "LOG",
303    Lon => "LON",
304    Lookup => "LOOKUP",
305    Majority => "MAJORITY",
306    Manhattan => "MANHATTAN",
307    Match => "MATCH",
308    MatchAny => "MATCH_ANY",
309    MaxSelectivity => "MAX_SELECTIVITY",
310    Midpoint => "MIDPOINT",
311    Mmr => "MMR",
312    Model => "MODEL",
313    Multi => "MULTI",
314    Multivector => "MULTIVECTOR",
315    Naive => "NAIVE",
316    Nearest => "NEAREST",
317    Negative => "NEGATIVE",
318    Nested => "NESTED",
319    Not => "NOT",
320    Null => "NULL",
321    Offset => "OFFSET",
322    On => "ON",
323    Optimizers => "OPTIMIZERS",
324    Or => "OR",
325    Order => "ORDER",
326    Oversampling => "OVERSAMPLING",
327    Params => "PARAMS",
328    Payload => "PAYLOAD",
329    Phrase => "PHRASE",
330    Point => "POINT",
331    Points => "POINTS",
332    Positive => "POSITIVE",
333    Pow => "POW",
334    Prefetch => "PREFETCH",
335    Quantization => "QUANTIZATION",
336    Query => "QUERY",
337    Quorum => "QUORUM",
338    Radius => "RADIUS",
339    Random => "RANDOM",
340    Recommend => "RECOMMEND",
341    Relevance => "RELEVANCE",
342    Rerank => "RERANK",
343    Rescore => "RESCORE",
344    Rrf => "RRF",
345    RrfK => "RRF_K",
346    RrfWeights => "RRF_WEIGHTS",
347    Sample => "SAMPLE",
348    Scale => "SCALE",
349    Score => "SCORE",
350    Scroll => "SCROLL",
351    Set => "SET",
352    Shard => "SHARD",
353    Show => "SHOW",
354    Size => "SIZE",
355    Sparse => "SPARSE",
356    Sqrt => "SQRT",
357    Strategy => "STRATEGY",
358    SumScores => "SUM_SCORES",
359    Target => "TARGET",
360    Text => "TEXT",
361    Then => "THEN",
362    Threshold => "THRESHOLD",
363    Timeout => "TIMEOUT",
364    TopLeft => "TOP_LEFT",
365    True => "TRUE",
366    Type => "TYPE",
367    Update => "UPDATE",
368    Upsert => "UPSERT",
369    Using => "USING",
370    Uuid => "UUID",
371    Values => "VALUES",
372    ValuesCount => "VALUES_COUNT",
373    Vector => "VECTOR",
374    When => "WHEN",
375    Where => "WHERE",
376    With => "WITH",
377    Star => "*",
378    Identifier => "IDENTIFIER",
379    String => "STRING",
380    Lbrace => "LBRACE",
381    Rbrace => "RBRACE",
382    Lbracket => "LBRACKET",
383    Rbracket => "RBRACKET",
384    Lparen => "LPAREN",
385    Rparen => "RPAREN",
386    Colon => "COLON",
387    Comma => "COMMA",
388    Equals => "EQUALS",
389    NotEquals => "NOT_EQUALS",
390    Gt => "GT",
391    Gte => "GTE",
392    Lt => "LT",
393    Lte => "LTE",
394    Plus => "PLUS",
395    Minus => "MINUS",
396    Slash => "SLASH",
397    Semicolon => "SEMICOLON",
398    Eof => "EOF",
399}
400
401include!("keywords.generated.rs");
402
403impl TokenKind {
404    pub fn is_keyword_or_identifier(&self) -> bool {
405        !matches!(
406            self,
407            Self::String
408                | Self::Lbrace
409                | Self::Rbrace
410                | Self::Lbracket
411                | Self::Rbracket
412                | Self::Lparen
413                | Self::Rparen
414                | Self::Colon
415                | Self::Comma
416                | Self::Equals
417                | Self::NotEquals
418                | Self::Gt
419                | Self::Gte
420                | Self::Lt
421                | Self::Lte
422                | Self::Plus
423                | Self::Minus
424                | Self::Slash
425                | Self::Semicolon
426                | Self::Eof
427        )
428    }
429}
430
431impl fmt::Display for TokenKind {
432    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
433        f.write_str(self.as_str())
434    }
435}
436
437#[derive(Debug, Clone, Copy, PartialEq, Eq)]
438#[cfg_attr(feature = "serde", derive(serde::Serialize))]
439pub struct Token<'a> {
440    pub kind: TokenKind,
441    pub text: &'a str,
442    pub span: Span,
443    #[cfg_attr(feature = "serde", serde(skip))]
444    pub(crate) pos: usize,
445}
446
447impl<'a> Token<'a> {
448    pub fn new(kind: TokenKind, text: &'a str, span: Span) -> Self {
449        Token {
450            kind,
451            text,
452            pos: span.start,
453            span,
454        }
455    }
456
457    pub fn eof(position: usize) -> Self {
458        Token {
459            kind: TokenKind::Eof,
460            text: "",
461            span: Span::point(position),
462            pos: position,
463        }
464    }
465
466    pub fn is_keyword_or_identifier(&self) -> bool {
467        match self.kind {
468            TokenKind::String
469            | TokenKind::Lbrace
470            | TokenKind::Rbrace
471            | TokenKind::Lbracket
472            | TokenKind::Rbracket
473            | TokenKind::Lparen
474            | TokenKind::Rparen
475            | TokenKind::Colon
476            | TokenKind::Comma
477            | TokenKind::Equals
478            | TokenKind::NotEquals
479            | TokenKind::Gt
480            | TokenKind::Gte
481            | TokenKind::Lt
482            | TokenKind::Lte
483            | TokenKind::Plus
484            | TokenKind::Minus
485            | TokenKind::Slash
486            | TokenKind::Semicolon
487            | TokenKind::Eof => false,
488            TokenKind::Integer | TokenKind::Float => self
489                .text
490                .bytes()
491                .next()
492                .is_some_and(|b| b.is_ascii_alphabetic()),
493            _ => true,
494        }
495    }
496}
497
498impl<'a> fmt::Display for Token<'a> {
499    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
500        write!(f, "{}({})", self.kind, self.text)
501    }
502}
503
504pub fn lookup_keyword(s: &str) -> Option<TokenKind> {
505    let bytes = s.as_bytes();
506    let len = bytes.len();
507    if len == 0 || len > 32 {
508        return None;
509    }
510    let mut buf = [0u8; 32];
511    for (i, b) in bytes.iter().enumerate() {
512        buf[i] = b.to_ascii_uppercase();
513    }
514    let upper = core::str::from_utf8(&buf[..len]).ok()?;
515    KEYWORDS.get(upper).copied()
516}
qql_core/token.rs

qql_core/
token.rs
