rusty_handlebars_parser/
expression_tokenizer.rs

1//! Handlebars expression tokenization
2//!
3//! This module provides functionality for tokenizing Handlebars expressions into their component parts.
4//! It handles various token types including:
5//! - Literals: Plain text values
6//! - Private variables: Variables prefixed with @ (e.g. @index)
7//! - Sub-expressions: Parenthesized expressions
8//!
9//! # Token Types
10//!
11//! ## Literals
12//! Plain text values that are not special tokens:
13//! ```
14//! name
15//! user.age
16//! ```
17//!
18//! ## Private Variables
19//! Variables prefixed with @ that have special meaning:
20//! ```
21//! @index
22//! @first
23//! @last
24//! ```
25//!
26//! ## Sub-expressions
27//! Parenthesized expressions that are evaluated first:
28//! ```
29//! (helper arg1 arg2)
30//! (math.add 1 2)
31//! ```
32//!
33//! # Examples
34//!
35//! ```rust
36//! use rusty_handlebars_parser::expression_tokenizer::{Token, TokenType};
37//!
38//! let src = "user.name (helper arg) @index";
39//! let token = Token::first(src).unwrap().unwrap();
40//! assert_eq!(token.value, "user.name");
41//! assert_eq!(token.token_type, TokenType::Literal);
42//! ```
43
44use crate::error::{rcap, ParseError, Result};
45
46/// Types of tokens that can be parsed from an expression
47#[derive(Clone)]
48pub enum TokenType<'a> {
49    /// A parenthesized sub-expression
50    SubExpression(&'a str),
51    /// A private variable prefixed with @
52    PrivateVariable,
53    Variable,
54    /// A plain text literal
55    Literal
56}
57
58/// A token parsed from an expression
59#[derive(Clone)]
60pub struct Token<'a> {
61    /// The type of token
62    pub token_type: TokenType<'a>,
63    /// The token's value
64    pub value: &'a str,
65    /// The remaining text after this token
66    pub tail: &'a str
67}
68
69/// Finds the closing parenthesis for a sub-expression
70fn find_closing(src: &str) -> Result<usize> {
71    let mut count = 1;
72    let rest = &src[1..];
73    for (i, c) in rest.char_indices() {
74        match c {
75            '(' => count += 1,
76            ')' => count -= 1,
77            _ => ()
78        }
79        if count == 0 {
80            return Ok(i + 1);
81        }
82    }
83    Err(ParseError{ message: format!("unmatched brackets near {}", rcap(src))})
84}
85
86fn find_end_of_string(src: &str) -> Result<usize> {
87    let cliped = &src[1..];
88    let mut escaped = false;
89    for (i, c) in cliped.char_indices() {
90        match c {
91            '\\' => escaped = !escaped,
92            '"' => {
93                if !escaped {
94                    return Ok(i + 2);
95                }
96            }
97            _ => ()
98        }
99    }
100    Err(ParseError{ message: format!("unterminated string near {}", rcap(src))})
101}
102
103/// Finds the end of a token by looking for whitespace or special characters
104fn find_end(src: &str) -> usize {
105    for (i, c) in src.char_indices() {
106        if " (\n\r\t".contains(c) {
107            return i
108        }
109    }
110    src.len()
111}
112
113fn is_number(src: &str) -> bool {
114    src.chars().all(|c| c.is_digit(10) || c == '.')
115}
116
117/// Parses a single token from the input string
118fn parse<'a>(src: &'a str) -> Result<Option<Token<'a>>> {
119    Ok(match src.chars().next() {
120        Some('@') => {
121            let end = find_end(src);
122            Some(Token {
123                token_type: TokenType::PrivateVariable,
124                value: &src[1..end],
125                tail: &src[end..].trim_start()
126            })
127        },
128        Some('(') => {
129            let end = find_closing(&src)?;
130            Some(Token {
131                token_type: TokenType::SubExpression(&src[..end]),
132                value: &src[1..end],
133                tail: &src[end + 1..].trim_start()
134            })
135        },
136        None => None,
137        _ => {
138            let (end, token_type) = if src.starts_with('"') {
139                (find_end_of_string(src)?, TokenType::Literal)
140            } else {
141                (find_end(src), if is_number(src) { TokenType::Literal } else { TokenType::Variable })
142            };
143            Some(Token {
144                token_type,
145                value: &src[..end],
146                tail: &src[end..].trim_start()
147            })
148        }
149    })
150}
151
152impl<'a> Token<'a> {
153    /// Parses the first token from a string
154    pub fn first(src: &'a str) -> Result<Option<Self>> {
155        parse(src.trim())
156    }
157
158    /// Parses the next token after this one
159    pub fn next(&self) -> Result<Option<Self>> {
160        parse(self.tail)
161    }
162}
163
rusty_handlebars_parser/expression_tokenizer.rs

rusty_handlebars_parser/
expression_tokenizer.rs
