use core::fmt;
use crate::error::Span;
macro_rules! gen_as_str {
($($var:ident => $str:expr),* $(,)?) => {
impl TokenKind {
pub fn as_str(&self) -> &'static str {
match self {
$( Self::$var => $str, )*
}
}
}
};
}
macro_rules! gen_keywords {
($($str:expr => TokenKind::$var:ident),* $(,)?) => {
pub static KEYWORDS: phf::Map<&'static str, TokenKind> = phf::phf_map! {
$( $str => TokenKind::$var, )*
};
};
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[cfg_attr(feature = "serde", derive(serde::Serialize))]
pub enum TokenKind {
Upsert,
Into,
Collection,
Values,
Using,
Model,
Hybrid,
Dense,
Sparse,
Rerank,
Exact,
With,
As,
Acorn,
Scalar,
Binary,
Product,
Turbo,
Bits,
Quantile,
Always,
Ram,
Hnsw,
Optimizers,
Params,
Disabled,
Create,
Alter,
Drop,
Show,
Collections,
Scroll,
Recommend,
Limit,
Group,
By,
Strategy,
Delete,
Update,
Set,
Offset,
Score,
Threshold,
Lookup,
Vector,
Payload,
From,
Where,
Id,
Index,
On,
For,
Type,
And,
Or,
Not,
In,
Between,
Is,
Null,
Empty,
Match,
Any,
Phrase,
Cosine,
Dot,
Euclid,
Manhattan,
Order,
Asc,
Desc,
Query,
Nearest,
Context,
Discover,
Pairs,
Target,
Prefetch,
Fusion,
Sample,
Defaults,
Case,
When,
Then,
Else,
End,
GeoBbox,
GeoRadius,
GeoPolygon,
ValuesCount,
HasVector,
Relevance,
Feedback,
Star,
After,
Shard,
Key,
Keys,
Identifier,
String,
Integer,
Float,
Lbrace,
Rbrace,
Lbracket,
Rbracket,
Lparen,
Rparen,
Colon,
Comma,
Equals,
NotEquals,
Gt,
Gte,
Lt,
Lte,
Plus,
Minus,
Slash,
Count,
Clear,
Semicolon,
Eof,
}
gen_as_str! {
Upsert => "UPSERT",
Into => "INTO",
Collection => "COLLECTION",
Values => "VALUES",
Using => "USING",
Model => "MODEL",
Hybrid => "HYBRID",
Dense => "DENSE",
Sparse => "SPARSE",
Rerank => "RERANK",
Exact => "EXACT",
With => "WITH",
As => "AS",
Acorn => "ACORN",
Scalar => "SCALAR",
Binary => "BINARY",
Product => "PRODUCT",
Turbo => "TURBO",
Bits => "BITS",
Quantile => "QUANTILE",
Always => "ALWAYS",
Ram => "RAM",
Hnsw => "HNSW",
Optimizers => "OPTIMIZERS",
Params => "PARAMS",
Disabled => "DISABLED",
Create => "CREATE",
Alter => "ALTER",
Drop => "DROP",
Show => "SHOW",
Collections => "COLLECTIONS",
Scroll => "SCROLL",
Recommend => "RECOMMEND",
Limit => "LIMIT",
Group => "GROUP",
By => "BY",
Strategy => "STRATEGY",
Delete => "DELETE",
Update => "UPDATE",
Set => "SET",
Offset => "OFFSET",
Score => "SCORE",
Threshold => "THRESHOLD",
Lookup => "LOOKUP",
Vector => "VECTOR",
Payload => "PAYLOAD",
From => "FROM",
Where => "WHERE",
Id => "ID",
Index => "INDEX",
On => "ON",
For => "FOR",
Type => "TYPE",
And => "AND",
Or => "OR",
Not => "NOT",
In => "IN",
Between => "BETWEEN",
Is => "IS",
Null => "NULL",
Empty => "EMPTY",
Match => "MATCH",
Any => "ANY",
Phrase => "PHRASE",
Cosine => "COSINE",
Dot => "DOT",
Euclid => "EUCLID",
Manhattan => "MANHATTAN",
Order => "ORDER",
Asc => "ASC",
Desc => "DESC",
Query => "QUERY",
Nearest => "NEAREST",
Context => "CONTEXT",
Discover => "DISCOVER",
Pairs => "PAIRS",
Target => "TARGET",
Prefetch => "PREFETCH",
Fusion => "FUSION",
Sample => "SAMPLE",
Defaults => "DEFAULTS",
Case => "CASE",
When => "WHEN",
Then => "THEN",
Else => "ELSE",
End => "END",
GeoBbox => "GEO_BBOX",
GeoRadius => "GEO_RADIUS",
GeoPolygon => "GEO_POLYGON",
ValuesCount => "VALUES_COUNT",
HasVector => "HAS_VECTOR",
Relevance => "RELEVANCE",
Feedback => "FEEDBACK",
Star => "STAR",
After => "AFTER",
Count => "COUNT",
Clear => "CLEAR",
Shard => "SHARD",
Key => "KEY",
Keys => "KEYS",
Identifier => "IDENTIFIER",
String => "STRING",
Integer => "INTEGER",
Float => "FLOAT",
Lbrace => "LBRACE",
Rbrace => "RBRACE",
Lbracket => "LBRACKET",
Rbracket => "RBRACKET",
Lparen => "LPAREN",
Rparen => "RPAREN",
Colon => "COLON",
Comma => "COMMA",
Equals => "EQUALS",
NotEquals => "NOT_EQUALS",
Gt => "GT",
Gte => "GTE",
Lt => "LT",
Lte => "LTE",
Plus => "PLUS",
Minus => "MINUS",
Slash => "SLASH",
Semicolon => "SEMICOLON",
Eof => "EOF",
}
gen_keywords! {
"UPSERT" => TokenKind::Upsert,
"INTO" => TokenKind::Into,
"COLLECTION" => TokenKind::Collection,
"VALUES" => TokenKind::Values,
"USING" => TokenKind::Using,
"MODEL" => TokenKind::Model,
"HYBRID" => TokenKind::Hybrid,
"DENSE" => TokenKind::Dense,
"SPARSE" => TokenKind::Sparse,
"RERANK" => TokenKind::Rerank,
"EXACT" => TokenKind::Exact,
"WITH" => TokenKind::With,
"AS" => TokenKind::As,
"ACORN" => TokenKind::Acorn,
"SCALAR" => TokenKind::Scalar,
"BINARY" => TokenKind::Binary,
"PRODUCT" => TokenKind::Product,
"TURBO" => TokenKind::Turbo,
"BITS" => TokenKind::Bits,
"QUANTILE" => TokenKind::Quantile,
"ALWAYS" => TokenKind::Always,
"RAM" => TokenKind::Ram,
"HNSW" => TokenKind::Hnsw,
"OPTIMIZERS" => TokenKind::Optimizers,
"PARAMS" => TokenKind::Params,
"DISABLED" => TokenKind::Disabled,
"CREATE" => TokenKind::Create,
"ALTER" => TokenKind::Alter,
"DROP" => TokenKind::Drop,
"SHOW" => TokenKind::Show,
"COLLECTIONS" => TokenKind::Collections,
"SCROLL" => TokenKind::Scroll,
"RECOMMEND" => TokenKind::Recommend,
"LIMIT" => TokenKind::Limit,
"GROUP" => TokenKind::Group,
"BY" => TokenKind::By,
"STRATEGY" => TokenKind::Strategy,
"DELETE" => TokenKind::Delete,
"UPDATE" => TokenKind::Update,
"SET" => TokenKind::Set,
"OFFSET" => TokenKind::Offset,
"SCORE" => TokenKind::Score,
"THRESHOLD" => TokenKind::Threshold,
"LOOKUP" => TokenKind::Lookup,
"VECTOR" => TokenKind::Vector,
"PAYLOAD" => TokenKind::Payload,
"FROM" => TokenKind::From,
"WHERE" => TokenKind::Where,
"ID" => TokenKind::Id,
"INDEX" => TokenKind::Index,
"ON" => TokenKind::On,
"FOR" => TokenKind::For,
"TYPE" => TokenKind::Type,
"AND" => TokenKind::And,
"OR" => TokenKind::Or,
"NOT" => TokenKind::Not,
"IN" => TokenKind::In,
"BETWEEN" => TokenKind::Between,
"IS" => TokenKind::Is,
"NULL" => TokenKind::Null,
"EMPTY" => TokenKind::Empty,
"MATCH" => TokenKind::Match,
"ANY" => TokenKind::Any,
"PHRASE" => TokenKind::Phrase,
"COSINE" => TokenKind::Cosine,
"DOT" => TokenKind::Dot,
"EUCLID" => TokenKind::Euclid,
"MANHATTAN" => TokenKind::Manhattan,
"ORDER" => TokenKind::Order,
"ASC" => TokenKind::Asc,
"DESC" => TokenKind::Desc,
"QUERY" => TokenKind::Query,
"NEAREST" => TokenKind::Nearest,
"CONTEXT" => TokenKind::Context,
"DISCOVER" => TokenKind::Discover,
"PAIRS" => TokenKind::Pairs,
"TARGET" => TokenKind::Target,
"PREFETCH" => TokenKind::Prefetch,
"FUSION" => TokenKind::Fusion,
"SAMPLE" => TokenKind::Sample,
"DEFAULTS" => TokenKind::Defaults,
"CASE" => TokenKind::Case,
"WHEN" => TokenKind::When,
"THEN" => TokenKind::Then,
"ELSE" => TokenKind::Else,
"END" => TokenKind::End,
"GEO_BBOX" => TokenKind::GeoBbox,
"GEO_RADIUS" => TokenKind::GeoRadius,
"GEO_POLYGON" => TokenKind::GeoPolygon,
"VALUES_COUNT" => TokenKind::ValuesCount,
"HAS_VECTOR" => TokenKind::HasVector,
"RELEVANCE" => TokenKind::Relevance,
"FEEDBACK" => TokenKind::Feedback,
"STAR" => TokenKind::Star,
"AFTER" => TokenKind::After,
"SHARD" => TokenKind::Shard,
"KEY" => TokenKind::Key,
"KEYS" => TokenKind::Keys,
"COUNT" => TokenKind::Count,
"CLEAR" => TokenKind::Clear,
}
impl fmt::Display for TokenKind {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(self.as_str())
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[cfg_attr(feature = "serde", derive(serde::Serialize))]
pub struct Token<'a> {
pub kind: TokenKind,
pub text: &'a str,
pub span: Span,
#[cfg_attr(feature = "serde", serde(skip))]
pub(crate) pos: usize,
}
impl<'a> Token<'a> {
pub fn new(kind: TokenKind, text: &'a str, span: Span) -> Self {
Token {
kind,
text,
pos: span.start,
span,
}
}
pub fn eof(position: usize) -> Self {
Token {
kind: TokenKind::Eof,
text: "",
span: Span::point(position),
pos: position,
}
}
}
impl<'a> fmt::Display for Token<'a> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}({})", self.kind, self.text)
}
}
pub fn lookup_keyword(s: &str) -> Option<TokenKind> {
let bytes = s.as_bytes();
let len = bytes.len();
if len == 0 || len > 32 {
return None;
}
let mut buf = [0u8; 32];
for (i, b) in bytes.iter().enumerate() {
buf[i] = b.to_ascii_uppercase();
}
let upper = core::str::from_utf8(&buf[..len]).ok()?;
KEYWORDS.get(upper).copied()
}