use std::collections::HashMap;
use std::sync::LazyLock;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum TokenKind {
String,
Number,
Boolean,
Null,
Identifier,
Variable, Keyword,
Assign, Plus, Minus, Multiply, Divide, Modulo,
Eq, Ne, Gt, Lt, Gte, Lte,
And, Or, Not,
LParen, RParen, LBracket, RBracket, LBrace, RBrace, Comma, Colon, Dot,
Decorator, Comment, Newline,
Eof,
SubexpressionOpen, TemplateLiteral, ChunkMarker, CellOpen, CellClose, PromptOpen, PromptClose, RawText, }
#[derive(Debug, Clone)]
pub struct Token {
pub kind: TokenKind,
pub text: String,
pub line: usize, pub column: usize, pub value: Option<TokenValue>,
pub is_continuation: bool,
}
#[derive(Debug, Clone)]
pub enum TokenValue {
String(String),
Number(f64),
Bool(bool),
}
impl Token {
pub fn new(kind: TokenKind, text: impl Into<String>, line: usize, column: usize) -> Self {
Self {
kind,
text: text.into(),
line,
column,
value: None,
is_continuation: false,
}
}
pub fn with_value(mut self, value: TokenValue) -> Self {
self.value = Some(value);
self
}
pub fn string_value(&self) -> Option<&str> {
match &self.value {
Some(TokenValue::String(s)) => Some(s),
_ => None,
}
}
pub fn number_value(&self) -> Option<f64> {
match &self.value {
Some(TokenValue::Number(n)) => Some(*n),
_ => None,
}
}
pub fn bool_value(&self) -> Option<bool> {
match &self.value {
Some(TokenValue::Bool(b)) => Some(*b),
_ => None,
}
}
pub fn is_keyword(&self, kw: &str) -> bool {
self.kind == TokenKind::Keyword && self.text == kw
}
pub fn is_identifier(&self, name: &str) -> bool {
self.kind == TokenKind::Identifier && self.text == name
}
}
pub static KEYWORDS: LazyLock<HashMap<&'static str, &'static str>> = LazyLock::new(|| {
let mut m = HashMap::new();
let kws = [
"if", "else", "elseif", "endif", "then", "do", "enddo", "with", "endwith", "def",
"define", "enddef", "for", "endfor", "in", "on", "endon", "return", "break", "continue",
"together", "endtogether", "into", "from", "to", "by", "step", "key", "var", "const",
"log", "true", "false", "null", "repeat", "iftrue", "iffalse", "not", "and", "or", "set",
"as", "use", "say",
];
for kw in &kws {
m.insert(*kw, *kw);
}
m
});
pub fn is_keyword(word: &str) -> bool {
KEYWORDS.contains_key(word)
}