use std::fmt;
#[derive(Debug, Clone, PartialEq)]
pub enum TokenKind {
Text(String),
Variable(String), RawVariable(String),
If(String), Else, ElseIf(String), Fi,
Foreach(String, String), End, Break, Continue,
SectionDef(String), SectionCall(String),
HelperDef(String, Vec<String>), HelperCall(String, Vec<String>),
View(String, Option<String>), Import(Vec<String>),
Meta(Option<String>, Option<String>, Option<String>), Title(String), Description(String), FormField(String, String), Body, Head, Content, Csrf,
Translate(String), TranslateKey(String),
Config(String),
Repository(String), App(String), Main(String), R(String), Model(String), M(String), Session(String), Query(String), User(Option<String>),
Index, Newline,
Eof,
}
#[derive(Debug, Clone)]
pub struct Token {
pub kind: TokenKind,
pub line: usize,
pub column: usize,
}
impl Token {
pub fn new(kind: TokenKind, line: usize, column: usize) -> Self {
Self { kind, line, column }
}
}
impl fmt::Display for Token {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{:?} at {}:{}", self.kind, self.line, self.column)
}
}
pub struct Lexer {
input: Vec<char>,
position: usize,
current_char: Option<char>,
line: usize,
column: usize,
}
impl Lexer {
pub fn new(input: &str) -> Self {
let chars: Vec<char> = input.chars().collect();
let current_char = if chars.is_empty() {
None
} else {
Some(chars[0])
};
Self {
input: chars,
position: 0,
current_char,
line: 1,
column: 1,
}
}
fn advance(&mut self) {
if self.current_char == Some('\n') {
self.line += 1;
self.column = 1;
} else {
self.column += 1;
}
self.position += 1;
self.current_char = if self.position < self.input.len() {
Some(self.input[self.position])
} else {
None
};
}
fn peek(&self) -> Option<char> {
if self.position + 1 < self.input.len() {
Some(self.input[self.position + 1])
} else {
None
}
}
#[allow(dead_code)]
fn peek_n(&self, n: usize) -> Option<char> {
if self.position + n < self.input.len() {
Some(self.input[self.position + n])
} else {
None
}
}
fn is_directive_start(&self) -> bool {
self.current_char == Some('@') && (self.peek() == Some('{') || self.peek() == Some('('))
}
fn skip_whitespace(&mut self) {
while let Some(ch) = self.current_char {
if ch.is_whitespace() && ch != '\n' {
self.advance();
} else {
break;
}
}
}
fn read_until(&mut self, delimiter: char) -> String {
let mut result = String::new();
let mut brace_depth = 0;
while let Some(ch) = self.current_char {
if ch == '{' {
brace_depth += 1;
} else if ch == '}' {
if brace_depth == 0 && delimiter == '}' {
break;
}
brace_depth -= 1;
} else if ch == delimiter && brace_depth == 0 {
break;
}
result.push(ch);
self.advance();
}
result
}
fn read_text(&mut self) -> String {
let mut result = String::new();
while let Some(ch) = self.current_char {
if self.is_directive_start() {
break;
}
result.push(ch);
self.advance();
}
result
}
fn parse_directive(&mut self) -> Token {
let line = self.line;
let column = self.column;
self.advance(); self.advance();
self.skip_whitespace();
if self.current_char == Some('!') {
self.advance();
let var_name = self.read_until('}').trim().to_string();
self.advance(); return Token::new(TokenKind::RawVariable(var_name), line, column);
}
let content = self.read_until('}').trim().to_string();
self.advance();
let token_kind = self.parse_directive_content(&content);
Token::new(token_kind, line, column)
}
fn has_top_level_operator(expr: &str) -> bool {
let chars: Vec<char> = expr.chars().collect();
let mut paren_depth: i32 = 0;
let mut in_quote = false;
let mut quote_char = ' ';
let mut i = 0;
while i < chars.len() {
let c = chars[i];
if (c == '"' || c == '\'') && (i == 0 || chars[i - 1] != '\\') {
if !in_quote {
in_quote = true;
quote_char = c;
} else if c == quote_char {
in_quote = false;
}
i += 1;
continue;
}
if in_quote {
i += 1;
continue;
}
if c == '(' {
paren_depth += 1;
i += 1;
continue;
}
if c == ')' {
paren_depth -= 1;
i += 1;
continue;
}
if paren_depth == 0 {
if c == '?' {
return true;
}
if matches!(c, '+' | '*' | '/' | '%' | '<' | '>') {
return true;
}
if i + 1 < chars.len() {
let next = chars[i + 1];
if (c == '=' && next == '=')
|| (c == '!' && next == '=')
|| (c == '<' && next == '=')
|| (c == '>' && next == '=')
|| (c == '&' && next == '&')
|| (c == '|' && next == '|')
{
return true;
}
}
}
i += 1;
}
false
}
fn parse_directive_content(&self, content: &str) -> TokenKind {
let trimmed = content.trim();
if trimmed.starts_with("if ") {
let condition = trimmed[3..].trim().to_string();
return TokenKind::If(condition);
}
if trimmed == "else" {
return TokenKind::Else;
}
if trimmed.starts_with("elif ") {
let condition = trimmed[5..].trim().to_string();
return TokenKind::ElseIf(condition);
}
if trimmed == "fi" {
return TokenKind::Fi;
}
if trimmed.starts_with("foreach ") {
let parts = trimmed[8..].trim();
if let Some(in_pos) = parts.find(" in ") {
let item = parts[..in_pos].trim().to_string();
let collection = parts[in_pos + 4..].trim().to_string();
return TokenKind::Foreach(item, collection);
}
}
if trimmed == "end" {
return TokenKind::End;
}
if trimmed == "break" {
return TokenKind::Break;
}
if trimmed == "continue" {
return TokenKind::Continue;
}
if trimmed == "index" {
return TokenKind::Index;
}
if trimmed.starts_with("section ") {
let name = trimmed[8..].trim().to_string();
return TokenKind::SectionDef(name);
}
if trimmed.starts_with("section(") && trimmed.ends_with(')') {
let name = trimmed[8..trimmed.len() - 1]
.trim_matches('\'')
.trim_matches('"')
.to_string();
return TokenKind::SectionCall(name);
}
if trimmed.starts_with("helper ") {
if let Some(paren_pos) = trimmed.find('(') {
let name = trimmed[7..paren_pos].trim().to_string();
let args_str = &trimmed[paren_pos + 1..trimmed.len() - 1];
let args = self.parse_args(args_str);
return TokenKind::HelperDef(name, args);
}
}
if trimmed.starts_with("view(") && trimmed.ends_with(')') {
let args_str = &trimmed[5..trimmed.len() - 1];
let parts: Vec<&str> = args_str.split(',').collect();
let view_name = parts[0].trim_matches('\'').trim_matches('"').to_string();
let model = if parts.len() > 1 {
Some(parts[1].trim().to_string())
} else {
None
};
return TokenKind::View(view_name, model);
}
if trimmed.starts_with("import(") && trimmed.ends_with(')') {
let args_str = &trimmed[7..trimmed.len() - 1];
let files = self.parse_string_args(args_str);
return TokenKind::Import(files);
}
if trimmed.starts_with("meta") {
if trimmed == "meta" {
return TokenKind::Meta(None, None, None);
}
if trimmed.starts_with("meta(") && trimmed.ends_with(')') {
let args_str = &trimmed[5..trimmed.len() - 1];
let parts = self.parse_string_args(args_str);
let title = parts.first().cloned();
let desc = parts.get(1).cloned();
let keywords = parts.get(2).cloned();
return TokenKind::Meta(title, desc, keywords);
}
}
if trimmed.starts_with("title(") && trimmed.ends_with(')') {
let arg = trimmed[6..trimmed.len() - 1].trim().to_string();
return TokenKind::Title(arg);
}
if trimmed.starts_with("description(") && trimmed.ends_with(')') {
let arg = trimmed[12..trimmed.len() - 1].trim().to_string();
return TokenKind::Description(arg);
}
for kind in [
"text", "password", "hidden", "textarea", "checkbox", "radio",
] {
let prefix = format!("{}(", kind);
if trimmed.starts_with(&prefix) && trimmed.ends_with(')') {
let args = trimmed[prefix.len()..trimmed.len() - 1].trim().to_string();
return TokenKind::FormField(kind.to_string(), args);
}
}
if trimmed == "body" {
return TokenKind::Body;
}
if trimmed == "head" {
return TokenKind::Head;
}
if trimmed == "content" {
return TokenKind::Content;
}
if trimmed == "csrf" {
return TokenKind::Csrf;
}
if Self::has_top_level_operator(trimmed) {
return TokenKind::Variable(trimmed.to_string());
}
if trimmed.starts_with("APP.") {
let key = trimmed[4..].to_string();
return TokenKind::App(key);
}
if trimmed.starts_with("MAIN.") {
let key = trimmed[5..].to_string();
return TokenKind::Main(key);
}
if trimmed.starts_with("repository.") {
let key = trimmed[11..].to_string();
return TokenKind::Repository(key);
}
if trimmed.starts_with("R.") {
let key = trimmed[2..].to_string();
return TokenKind::R(key);
}
if trimmed.starts_with("model.") {
let key = trimmed[6..].to_string();
return TokenKind::Model(key);
}
if trimmed.starts_with("M.") {
let key = trimmed[2..].to_string();
return TokenKind::M(key);
}
if trimmed.starts_with("session.") {
let key = trimmed[8..].to_string();
return TokenKind::Session(key);
}
if trimmed.starts_with("query.") {
let key = trimmed[6..].to_string();
return TokenKind::Query(key);
}
if trimmed == "user" {
return TokenKind::User(None);
}
if trimmed.starts_with("user.") {
let property = trimmed[5..].to_string();
return TokenKind::User(Some(property));
}
if trimmed.starts_with('\'') && trimmed.ends_with('\'') {
let inner = &trimmed[1..trimmed.len() - 1];
if inner.starts_with('%') {
let config_key = inner[1..].to_string();
return TokenKind::Config(config_key);
}
}
if let Some(paren_pos) = trimmed.find('(') {
if trimmed.ends_with(')') {
let name = trimmed[..paren_pos].trim().to_string();
let args_str = &trimmed[paren_pos + 1..trimmed.len() - 1];
let args = self.parse_args(args_str);
return TokenKind::HelperCall(name, args);
}
}
TokenKind::Variable(trimmed.to_string())
}
fn parse_localization(&mut self) -> Token {
let line = self.line;
let column = self.column;
self.advance(); self.advance();
self.skip_whitespace();
if self.current_char == Some('#') {
self.advance();
let key = self.read_until(')').trim().to_string();
self.advance(); return Token::new(TokenKind::TranslateKey(key), line, column);
}
let text = self.read_until(')').trim().to_string();
self.advance();
Token::new(TokenKind::Translate(text), line, column)
}
fn parse_args(&self, args_str: &str) -> Vec<String> {
args_str
.split(',')
.map(|s| s.trim().to_string())
.filter(|s| !s.is_empty())
.collect()
}
fn parse_string_args(&self, args_str: &str) -> Vec<String> {
args_str
.split(',')
.map(|s| s.trim().trim_matches('\'').trim_matches('"').to_string())
.filter(|s| !s.is_empty())
.collect()
}
pub fn next_token(&mut self) -> Token {
if self.current_char.is_none() {
return Token::new(TokenKind::Eof, self.line, self.column);
}
if self.current_char == Some('\n') {
let token = Token::new(TokenKind::Newline, self.line, self.column);
self.advance();
return token;
}
if self.current_char == Some('@') {
if self.peek() == Some('{') {
return self.parse_directive();
} else if self.peek() == Some('(') {
return self.parse_localization();
}
}
let text = self.read_text();
if !text.is_empty() {
Token::new(TokenKind::Text(text), self.line, self.column)
} else {
self.advance();
self.next_token()
}
}
pub fn tokenize(&mut self) -> Vec<Token> {
let mut tokens = Vec::new();
loop {
let token = self.next_token();
if token.kind == TokenKind::Eof {
tokens.push(token);
break;
}
tokens.push(token);
}
tokens
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_basic_text() {
let mut lexer = Lexer::new("Hello World");
let tokens = lexer.tokenize();
assert_eq!(tokens.len(), 2);
match &tokens[0].kind {
TokenKind::Text(t) => assert_eq!(t, "Hello World"),
_ => panic!("Expected text token"),
}
}
#[test]
fn test_variable() {
let mut lexer = Lexer::new("Hello @{name}!");
let tokens = lexer.tokenize();
assert_eq!(tokens.len(), 4);
match &tokens[1].kind {
TokenKind::Variable(v) => assert_eq!(v, "name"),
_ => panic!("Expected variable token"),
}
}
#[test]
fn test_raw_variable() {
let mut lexer = Lexer::new("@{!htmlContent}");
let tokens = lexer.tokenize();
match &tokens[0].kind {
TokenKind::RawVariable(v) => assert_eq!(v, "htmlContent"),
_ => panic!("Expected raw variable token"),
}
}
#[test]
fn test_if_else() {
let mut lexer = Lexer::new("@{if user.isActive}\nActive\n@{else}\nInactive\n@{fi}");
let tokens = lexer.tokenize();
match &tokens[0].kind {
TokenKind::If(cond) => assert_eq!(cond, "user.isActive"),
_ => panic!("Expected if token"),
}
let else_pos = tokens
.iter()
.position(|t| matches!(t.kind, TokenKind::Else))
.expect("Should have an Else token");
assert!(matches!(tokens[else_pos].kind, TokenKind::Else));
let fi_pos = tokens
.iter()
.position(|t| matches!(t.kind, TokenKind::Fi))
.expect("Should have a Fi token");
assert!(matches!(tokens[fi_pos].kind, TokenKind::Fi));
}
#[test]
fn test_foreach() {
let mut lexer = Lexer::new("@{foreach item in items}\n@{item.name}\n@{end}");
let tokens = lexer.tokenize();
match &tokens[0].kind {
TokenKind::Foreach(item, collection) => {
assert_eq!(item, "item");
assert_eq!(collection, "items");
}
_ => panic!("Expected foreach token"),
}
}
#[test]
fn test_section() {
let mut lexer = Lexer::new("@{section header}\nHeader content\n@{end}");
let tokens = lexer.tokenize();
match &tokens[0].kind {
TokenKind::SectionDef(name) => assert_eq!(name, "header"),
_ => panic!("Expected section definition token"),
}
}
#[test]
fn test_view() {
let mut lexer = Lexer::new("@{view('partial', model)}");
let tokens = lexer.tokenize();
match &tokens[0].kind {
TokenKind::View(name, model) => {
assert_eq!(name, "partial");
assert_eq!(model.as_ref().unwrap(), "model");
}
_ => panic!("Expected view token"),
}
}
#[test]
fn test_import() {
let mut lexer = Lexer::new("@{import('style.css', 'script.js')}");
let tokens = lexer.tokenize();
match &tokens[0].kind {
TokenKind::Import(files) => {
assert_eq!(files.len(), 2);
assert_eq!(files[0], "style.css");
assert_eq!(files[1], "script.js");
}
_ => panic!("Expected import token"),
}
}
#[test]
fn test_localization() {
let mut lexer = Lexer::new("@(Hello World)");
let tokens = lexer.tokenize();
match &tokens[0].kind {
TokenKind::Translate(text) => assert_eq!(text, "Hello World"),
_ => panic!("Expected translate token"),
}
}
#[test]
fn test_localization_key() {
let mut lexer = Lexer::new("@(#welcome.message)");
let tokens = lexer.tokenize();
match &tokens[0].kind {
TokenKind::TranslateKey(key) => assert_eq!(key, "welcome.message"),
_ => panic!("Expected translate key token"),
}
}
#[test]
fn test_special_variables() {
let mut lexer = Lexer::new("@{repository.data} @{session.user} @{query.page}");
let tokens = lexer.tokenize();
let special_tokens: Vec<_> = tokens
.iter()
.filter(|t| !matches!(t.kind, TokenKind::Text(_) | TokenKind::Eof))
.collect();
match &special_tokens[0].kind {
TokenKind::Repository(key) => assert_eq!(key, "data"),
_ => panic!(
"Expected repository token, got {:?}",
special_tokens[0].kind
),
}
match &special_tokens[1].kind {
TokenKind::Session(key) => assert_eq!(key, "user"),
_ => panic!("Expected session token, got {:?}", special_tokens[1].kind),
}
match &special_tokens[2].kind {
TokenKind::Query(key) => assert_eq!(key, "page"),
_ => panic!("Expected query token, got {:?}", special_tokens[2].kind),
}
}
}