#![allow(clippy::missing_docs_in_private_items)]
use rust_decimal::Decimal;
use rust_decimal::prelude::*;
use crate::error::ParseError;
#[derive(Debug, Clone, PartialEq)]
pub enum Token {
Number(Decimal),
StringLit(String),
True,
False,
Null,
DateLiteral(String),
DateTimeLiteral(String),
Identifier(String),
Let,
In,
If,
Then,
Else,
And,
Or,
Not,
Bang,
Plus,
Minus,
Star,
Slash,
Percent,
Ampersand,
Eq,
NotEq,
Lt,
Gt,
LtEq,
GtEq,
DoubleQuestion,
Question,
LParen,
RParen,
LBracket,
RBracket,
LBrace,
RBrace,
Comma,
Dot,
Colon,
Dollar,
At,
Eof,
}
#[derive(Debug, Clone)]
pub struct Span {
pub start: usize,
pub end: usize,
}
#[derive(Debug, Clone)]
pub struct SpannedToken {
pub token: Token,
pub span: Span,
}
pub struct Lexer<'a> {
_input: &'a str,
chars: Vec<char>,
pos: usize,
}
impl<'a> Lexer<'a> {
pub fn new(input: &'a str) -> Self {
Lexer {
_input: input,
chars: input.chars().collect(),
pos: 0,
}
}
pub fn tokenize(&mut self) -> Result<Vec<SpannedToken>, ParseError> {
let mut tokens = Vec::new();
loop {
self.skip_whitespace_and_comments()?;
if self.pos >= self.chars.len() {
tokens.push(SpannedToken {
token: Token::Eof,
span: Span {
start: self.pos,
end: self.pos,
},
});
break;
}
let start = self.pos;
let token = self.next_token()?;
tokens.push(SpannedToken {
token,
span: Span {
start,
end: self.pos,
},
});
}
Ok(tokens)
}
fn peek(&self) -> Option<char> {
self.chars.get(self.pos).copied()
}
fn peek_at(&self, offset: usize) -> Option<char> {
self.chars.get(self.pos + offset).copied()
}
fn advance(&mut self) -> Option<char> {
let c = self.chars.get(self.pos).copied();
if c.is_some() {
self.pos += 1;
}
c
}
fn skip_whitespace_and_comments(&mut self) -> Result<(), ParseError> {
let len = self.chars.len();
loop {
while self.pos < len && self.chars[self.pos].is_whitespace() {
self.pos += 1;
}
if self.pos + 1 < len && self.chars[self.pos] == '/' && self.chars[self.pos + 1] == '/'
{
while self.pos < len && self.chars[self.pos] != '\n' {
self.pos += 1;
}
continue;
}
if self.pos + 1 < len && self.chars[self.pos] == '/' && self.chars[self.pos + 1] == '*'
{
let start = self.pos;
self.pos += 2;
let mut closed = false;
while self.pos + 1 < len {
if self.chars[self.pos] == '*' && self.chars[self.pos + 1] == '/' {
self.pos += 2;
closed = true;
break;
}
self.pos += 1;
}
if !closed {
let end = self.pos.min(len);
return Err(ParseError::with_span(
start..end,
format!("unterminated block comment at position {start}"),
));
}
continue;
}
break;
}
Ok(())
}
fn next_token(&mut self) -> Result<Token, ParseError> {
let c = self.peek().unwrap();
if c == '@' && self.is_date_literal_ahead() {
return self.read_date_literal();
}
if c.is_ascii_digit() {
return self.read_number();
}
if c == '"' || c == '\'' {
return self.read_string(c);
}
if c.is_ascii_alphabetic() || c == '_' {
return self.read_identifier();
}
self.advance();
match c {
'+' => Ok(Token::Plus),
'-' => Ok(Token::Minus),
'*' => Ok(Token::Star),
'/' => Ok(Token::Slash),
'%' => Ok(Token::Percent),
'&' => Ok(Token::Ampersand),
'(' => Ok(Token::LParen),
')' => Ok(Token::RParen),
'[' => Ok(Token::LBracket),
']' => Ok(Token::RBracket),
'{' => Ok(Token::LBrace),
'}' => Ok(Token::RBrace),
',' => Ok(Token::Comma),
'.' => Ok(Token::Dot),
':' => Ok(Token::Colon),
'$' => Ok(Token::Dollar),
'@' => Ok(Token::At),
'?' => {
if self.peek() == Some('?') {
self.advance();
Ok(Token::DoubleQuestion)
} else {
Ok(Token::Question)
}
}
'=' => {
if self.peek() == Some('=') {
self.advance();
}
Ok(Token::Eq)
}
'!' => {
if self.peek() == Some('=') {
self.advance();
Ok(Token::NotEq)
} else {
Ok(Token::Bang)
}
}
'<' => {
if self.peek() == Some('=') {
self.advance();
Ok(Token::LtEq)
} else {
Ok(Token::Lt)
}
}
'>' => {
if self.peek() == Some('=') {
self.advance();
Ok(Token::GtEq)
} else {
Ok(Token::Gt)
}
}
'|' => {
if self.peek() == Some('>') {
self.advance();
let pos = self.pos - 2;
Err(ParseError::with_span(
pos..self.pos,
format!("pipe operator `|>` is reserved for future use at position {pos}"),
))
} else {
let pos = self.pos - 1;
Err(ParseError::with_span(
pos..self.pos,
format!("unexpected character '|' at position {pos}"),
))
}
}
_ => {
let pos = self.pos - 1;
Err(ParseError::with_span(
pos..self.pos,
format!("unexpected character '{c}' at position {pos}"),
))
}
}
}
fn is_date_literal_ahead(&self) -> bool {
if self.pos + 11 > self.chars.len() {
return false;
}
let s = &self.chars[self.pos..self.pos + 11];
s[0] == '@'
&& s[1..5].iter().all(|c| c.is_ascii_digit())
&& s[5] == '-'
&& s[6..8].iter().all(|c| c.is_ascii_digit())
&& s[8] == '-'
&& s[9..11].iter().all(|c| c.is_ascii_digit())
}
fn read_date_literal(&mut self) -> Result<Token, ParseError> {
let start = self.pos;
self.advance(); for _ in 0..10 {
self.advance();
}
if self.peek() == Some('T') {
self.advance(); for _ in 0..8 {
self.advance();
}
match self.peek() {
Some('Z') => {
self.advance();
}
Some('+') | Some('-') if self.peek_at(1).is_some_and(|c| c.is_ascii_digit()) => {
self.advance(); for _ in 0..5 {
self.advance();
}
}
_ => {}
}
let s: String = self.chars[start..self.pos].iter().collect();
Ok(Token::DateTimeLiteral(s))
} else {
let s: String = self.chars[start..self.pos].iter().collect();
Ok(Token::DateLiteral(s))
}
}
fn read_number(&mut self) -> Result<Token, ParseError> {
let start = self.pos;
if self.peek() == Some('-') {
self.advance();
}
while self.peek().is_some_and(|c| c.is_ascii_digit()) {
self.advance();
}
if self.peek() == Some('.') && self.peek_at(1).is_some_and(|c| c.is_ascii_digit()) {
self.advance(); while self.peek().is_some_and(|c| c.is_ascii_digit()) {
self.advance();
}
}
if self.peek().is_some_and(|c| c == 'e' || c == 'E') {
self.advance();
if self.peek().is_some_and(|c| c == '+' || c == '-') {
self.advance();
}
while self.peek().is_some_and(|c| c.is_ascii_digit()) {
self.advance();
}
}
let s: String = self.chars[start..self.pos].iter().collect();
let n: Decimal = s
.parse()
.or_else(|_| s.parse::<f64>().ok().and_then(Decimal::from_f64).ok_or(()))
.map_err(|_| {
ParseError::with_span(
start..self.pos,
format!("invalid number '{s}' at position {start}"),
)
})?;
Ok(Token::Number(n))
}
fn read_string(&mut self, quote: char) -> Result<Token, ParseError> {
let open_start = self.pos;
self.advance(); let mut result = String::new();
loop {
match self.advance() {
None => {
return Err(ParseError::with_span(
open_start..self.pos,
"unterminated string".to_string(),
));
}
Some(c) if c == quote => return Ok(Token::StringLit(result)),
Some('\\') => {
let esc_pos = self.pos - 1; match self.advance() {
Some('n') => result.push('\n'),
Some('t') => result.push('\t'),
Some('r') => result.push('\r'),
Some('\\') => result.push('\\'),
Some('"') => result.push('"'),
Some('\'') => result.push('\''),
Some('u') => {
let mut hex = String::with_capacity(4);
for _ in 0..4 {
match self.peek() {
Some(h) if h.is_ascii_hexdigit() => {
hex.push(h);
self.advance();
}
_ => {
return Err(ParseError::with_span(
esc_pos..self.pos,
format!(
"invalid unicode escape '\\u{hex}' at position {esc_pos}: expected 4 hex digits"
),
));
}
}
}
let cp = u32::from_str_radix(&hex, 16).unwrap();
match char::from_u32(cp) {
Some(ch) => result.push(ch),
None => {
return Err(ParseError::with_span(
esc_pos..self.pos,
format!(
"invalid unicode codepoint '\\u{hex}' at position {esc_pos}"
),
));
}
}
}
Some(c) => {
return Err(ParseError::with_span(
esc_pos..self.pos,
format!(
"unrecognized escape sequence '\\{c}' at position {esc_pos}"
),
));
}
None => {
return Err(ParseError::with_span(
esc_pos..self.pos,
"unterminated string escape".to_string(),
));
}
}
}
Some(c) => result.push(c),
}
}
}
fn read_identifier(&mut self) -> Result<Token, ParseError> {
let start = self.pos;
while self
.peek()
.is_some_and(|c| c.is_ascii_alphanumeric() || c == '_')
{
self.advance();
}
let word: String = self.chars[start..self.pos].iter().collect();
Ok(match word.as_str() {
"true" => Token::True,
"false" => Token::False,
"null" => Token::Null,
"let" => Token::Let,
"in" => Token::In,
"if" => Token::If,
"then" => Token::Then,
"else" => Token::Else,
"and" => Token::And,
"or" => Token::Or,
"not" => Token::Not,
_ => Token::Identifier(word),
})
}
}
const FEL_KEYWORDS: &[&str] = &[
"true", "false", "null", "let", "in", "if", "then", "else", "and", "or", "not",
];
pub fn is_valid_fel_identifier(s: &str) -> bool {
if s.is_empty() {
return false;
}
let mut chars = s.chars();
let first = chars.next().unwrap();
if !first.is_ascii_alphabetic() && first != '_' {
return false;
}
if !chars.all(|c| c.is_ascii_alphanumeric() || c == '_') {
return false;
}
!FEL_KEYWORDS.contains(&s)
}
pub fn sanitize_fel_identifier(s: &str) -> String {
let mut result: String = s
.chars()
.filter(|c| c.is_ascii_alphanumeric() || *c == '_')
.collect();
if result.is_empty() {
return "_".to_string();
}
if result.chars().next().unwrap().is_ascii_digit() {
result.insert(0, '_');
}
if FEL_KEYWORDS.contains(&result.as_str()) {
result.push('_');
}
result
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PositionedToken {
pub token_type: String,
pub text: String,
pub start: usize,
pub end: usize,
}
fn token_type_name(token: &Token) -> &'static str {
match token {
Token::Number(_) => "NumberLiteral",
Token::StringLit(_) => "StringLiteral",
Token::True => "True",
Token::False => "False",
Token::Null => "Null",
Token::DateLiteral(_) => "DateLiteral",
Token::DateTimeLiteral(_) => "DateTimeLiteral",
Token::Identifier(_) => "Identifier",
Token::Let => "Let",
Token::In => "In",
Token::If => "If",
Token::Then => "Then",
Token::Else => "Else",
Token::And => "And",
Token::Or => "Or",
Token::Not => "Not",
Token::Bang => "Bang",
Token::Plus => "Plus",
Token::Minus => "Minus",
Token::Star => "Asterisk",
Token::Slash => "Slash",
Token::Percent => "Percent",
Token::Ampersand => "Ampersand",
Token::Eq => "Equals",
Token::NotEq => "NotEquals",
Token::Lt => "Less",
Token::Gt => "Greater",
Token::LtEq => "LessEqual",
Token::GtEq => "GreaterEqual",
Token::DoubleQuestion => "DoubleQuestion",
Token::Question => "Question",
Token::LParen => "LRound",
Token::RParen => "RRound",
Token::LBracket => "LSquare",
Token::RBracket => "RSquare",
Token::LBrace => "LCurly",
Token::RBrace => "RCurly",
Token::Comma => "Comma",
Token::Dot => "Dot",
Token::Colon => "Colon",
Token::Dollar => "Dollar",
Token::At => "At",
Token::Eof => "EOF",
}
}
fn slice_by_char_offsets(input: &str, start: usize, end: usize) -> String {
input
.chars()
.skip(start)
.take(end.saturating_sub(start))
.collect()
}
pub fn tokenize(input: &str) -> Result<Vec<PositionedToken>, String> {
let mut lexer = Lexer::new(input);
let tokens = lexer.tokenize().map_err(|e| e.to_string())?;
Ok(tokens
.into_iter()
.map(|token| PositionedToken {
token_type: token_type_name(&token.token).to_string(),
text: slice_by_char_offsets(input, token.span.start, token.span.end),
start: token.span.start,
end: token.span.end,
})
.collect())
}
pub fn tokenize_to_json_value(input: &str) -> Result<serde_json::Value, String> {
tokenize_to_json_value_styled(input, crate::wire_style::JsonWireStyle::JsCamel)
}
pub fn tokenize_to_json_value_styled(
input: &str,
style: crate::wire_style::JsonWireStyle,
) -> Result<serde_json::Value, String> {
let tokens = tokenize(input)?;
Ok(serde_json::Value::Array(
tokens
.into_iter()
.map(|token| match style {
crate::wire_style::JsonWireStyle::JsCamel => serde_json::json!({
"tokenType": token.token_type,
"text": token.text,
"start": token.start,
"end": token.end,
}),
crate::wire_style::JsonWireStyle::PythonSnake => serde_json::json!({
"token_type": token.token_type,
"text": token.text,
"start": token.start,
"end": token.end,
}),
})
.collect(),
))
}
#[cfg(test)]
mod tests {
#![allow(clippy::missing_docs_in_private_items)]
use super::*;
fn lex_string(input: &str) -> Result<Vec<Token>, String> {
let mut lexer = Lexer::new(input);
lexer
.tokenize()
.map(|tokens| tokens.into_iter().map(|st| st.token).collect())
.map_err(|e| e.to_string())
}
#[test]
fn test_unicode_escape_basic() {
let tokens = lex_string(r#""\u0041""#).unwrap();
assert_eq!(tokens[0], Token::StringLit("A".into()));
}
#[test]
fn test_unicode_escape_multibyte() {
let tokens = lex_string(r#""\u00E9""#).unwrap();
assert_eq!(tokens[0], Token::StringLit("\u{00E9}".into()));
}
#[test]
fn test_unicode_escape_too_few_hex_digits() {
let err = lex_string(r#""\u00G1""#).unwrap_err();
assert!(err.contains("invalid unicode escape"), "got: {err}");
}
#[test]
fn test_unicode_escape_no_digits() {
let err = lex_string(r#""\u""#).unwrap_err();
assert!(err.contains("invalid unicode escape"), "got: {err}");
}
#[test]
fn test_unrecognized_escape_error() {
let err = lex_string(r#""\q""#).unwrap_err();
assert!(
err.contains("unrecognized escape sequence '\\q'"),
"got: {err}"
);
}
#[test]
fn test_slash_escape_removed() {
let err = lex_string(r#""\/""#).unwrap_err();
assert!(err.contains("unrecognized escape sequence"), "got: {err}");
}
#[test]
fn test_valid_escapes_still_work() {
let tokens = lex_string(r#""a\nb\tc\\d\"e\'f""#).unwrap();
assert_eq!(tokens[0], Token::StringLit("a\nb\tc\\d\"e'f".into()));
}
#[test]
fn test_pipe_operator_rejected_as_reserved() {
let err = lex_string("$a |> $b").unwrap_err();
assert!(
err.contains("pipe operator `|>` is reserved for future use"),
"got: {err}"
);
}
#[test]
fn test_bare_pipe_rejected() {
let err = lex_string("$a | $b").unwrap_err();
assert!(err.contains("unexpected character '|'"), "got: {err}");
}
#[test]
fn test_pipe_in_expression_rejected() {
let err = lex_string("concat('a', 'b') |> upper()").unwrap_err();
assert!(
err.contains("pipe operator `|>` is reserved for future use"),
"got: {err}"
);
}
#[test]
fn test_valid_identifiers() {
assert!(is_valid_fel_identifier("foo"));
assert!(is_valid_fel_identifier("_bar"));
assert!(is_valid_fel_identifier("camelCase123"));
assert!(is_valid_fel_identifier("_"));
assert!(is_valid_fel_identifier("x"));
assert!(is_valid_fel_identifier("snake_case_name"));
}
#[test]
fn test_invalid_identifiers() {
assert!(!is_valid_fel_identifier(""));
assert!(!is_valid_fel_identifier("123abc"));
assert!(!is_valid_fel_identifier("$ref"));
assert!(!is_valid_fel_identifier("foo-bar"));
assert!(!is_valid_fel_identifier("foo bar"));
assert!(!is_valid_fel_identifier("a.b"));
}
#[test]
fn test_keywords_are_not_valid_identifiers() {
for kw in FEL_KEYWORDS {
assert!(
!is_valid_fel_identifier(kw),
"keyword '{kw}' should be invalid"
);
}
}
#[test]
fn test_sanitize_valid_stays_unchanged() {
assert_eq!(sanitize_fel_identifier("foo"), "foo");
assert_eq!(sanitize_fel_identifier("_bar"), "_bar");
}
#[test]
fn test_sanitize_strips_invalid_chars() {
assert_eq!(sanitize_fel_identifier("foo-bar"), "foobar");
assert_eq!(sanitize_fel_identifier("hello world"), "helloworld");
assert_eq!(sanitize_fel_identifier("$ref"), "ref");
assert_eq!(sanitize_fel_identifier("a.b.c"), "abc");
}
#[test]
fn test_sanitize_prepends_underscore_for_digit_start() {
assert_eq!(sanitize_fel_identifier("123abc"), "_123abc");
}
#[test]
fn test_sanitize_appends_underscore_for_keyword() {
assert_eq!(sanitize_fel_identifier("true"), "true_");
assert_eq!(sanitize_fel_identifier("null"), "null_");
assert_eq!(sanitize_fel_identifier("and"), "and_");
}
#[test]
fn test_sanitize_empty_or_all_invalid() {
assert_eq!(sanitize_fel_identifier(""), "_");
assert_eq!(sanitize_fel_identifier("!@#$"), "_");
}
#[test]
fn tokenize_json_respects_wire_style() {
use serde_json::json;
use crate::wire_style::JsonWireStyle;
let js = tokenize_to_json_value_styled("1", JsonWireStyle::JsCamel).expect("tokenize");
let py = tokenize_to_json_value_styled("1", JsonWireStyle::PythonSnake).expect("tokenize");
assert_eq!(
js,
json!([
{ "tokenType": "NumberLiteral", "text": "1", "start": 0, "end": 1 },
{ "tokenType": "EOF", "text": "", "start": 1, "end": 1 }
])
);
assert_eq!(
py,
json!([
{ "token_type": "NumberLiteral", "text": "1", "start": 0, "end": 1 },
{ "token_type": "EOF", "text": "", "start": 1, "end": 1 }
])
);
}
}