rusty_handlebars_parser/
expression_tokenizer.rs

1//! Handlebars expression tokenization
2//!
3//! This module provides functionality for tokenizing Handlebars expressions into their component parts.
4//! It handles various token types including:
5//! - Literals: Plain text values
6//! - Private variables: Variables prefixed with @ (e.g. @index)
7//! - Sub-expressions: Parenthesized expressions
8//!
9//! # Token Types
10//!
11//! ## Literals
12//! Plain text values that are not special tokens:
13//! ```
14//! name
15//! user.age
16//! ```
17//!
18//! ## Private Variables
19//! Variables prefixed with @ that have special meaning:
20//! ```
21//! @index
22//! @first
23//! @last
24//! ```
25//!
26//! ## Sub-expressions
27//! Parenthesized expressions that are evaluated first:
28//! ```
29//! (helper arg1 arg2)
30//! (math.add 1 2)
31//! ```
32//!
33//! # Examples
34//!
35//! ```rust
36//! use rusty_handlebars_parser::expression_tokenizer::{Token, TokenType};
37//!
38//! let src = "user.name (helper arg) @index";
39//! let token = Token::first(src).unwrap().unwrap();
40//! assert_eq!(token.value, "user.name");
41//! assert_eq!(token.token_type, TokenType::Literal);
42//! ```
43
44use crate::error::{rcap, ParseError, Result};
45
46/// Types of tokens that can be parsed from an expression
47#[derive(Clone)]
48pub enum TokenType<'a> {
49    /// A parenthesized sub-expression
50    SubExpression(&'a str),
51    /// A private variable prefixed with @
52    PrivateVariable,
53    /// A plain text literal
54    Literal
55}
56
57/// A token parsed from an expression
58#[derive(Clone)]
59pub struct Token<'a> {
60    /// The type of token
61    pub token_type: TokenType<'a>,
62    /// The token's value
63    pub value: &'a str,
64    /// The remaining text after this token
65    pub tail: &'a str
66}
67
68/// Finds the closing parenthesis for a sub-expression
69fn find_closing(src: &str) -> Result<usize> {
70    let mut count = 1;
71    let rest = &src[1..];
72    for (i, c) in rest.char_indices() {
73        match c {
74            '(' => count += 1,
75            ')' => count -= 1,
76            _ => ()
77        }
78        if count == 0 {
79            return Ok(i + 1);
80        }
81    }
82    Err(ParseError{ message: format!("unmatched brackets near {}", rcap(src))})
83}
84
85/// Finds the end of a token by looking for whitespace or special characters
86fn find_end(src: &str) -> usize {
87    for (i, c) in src.char_indices() {
88        if " (\n\r\t".contains(c) {
89            return i
90        }
91    }
92    src.len()
93}
94
95/// Parses a single token from the input string
96fn parse<'a>(src: &'a str) -> Result<Option<Token<'a>>> {
97    Ok(match src.chars().next() {
98        Some('@') => {
99            let end = find_end(src);
100            Some(Token {
101                token_type: TokenType::PrivateVariable,
102                value: &src[1..end],
103                tail: &src[end..].trim_start()
104            })
105        },
106        Some('(') => {
107            let end = find_closing(&src)?;
108            Some(Token {
109                token_type: TokenType::SubExpression(&src[..end]),
110                value: &src[1..end],
111                tail: &src[end + 1..].trim_start()
112            })
113        },
114        None => None,
115        _ => {
116            let end = find_end(src);
117            Some(Token {
118                token_type: TokenType::Literal,
119                value: &src[..end],
120                tail: &src[end..].trim_start()
121            })
122        }
123    })
124}
125
126impl<'a> Token<'a> {
127    /// Parses the first token from a string
128    pub fn first(src: &'a str) -> Result<Option<Self>> {
129        parse(src.trim())
130    }
131
132    /// Parses the next token after this one
133    pub fn next(&self) -> Result<Option<Self>> {
134        parse(self.tail)
135    }
136}
137
rusty_handlebars_parser/expression_tokenizer.rs

rusty_handlebars_parser/
expression_tokenizer.rs
