helm_template_parser/
lexer.rs

1use crate::{error::Error, token::Token};
2
3pub fn tokenize(input: &str) -> Result<Vec<Token>, Error> {
4    let mut tokens = Vec::new();
5
6    let mut iter = input.chars();
7    loop {
8        match iter.next() {
9            Some(c) => {
10                let token = to_token(c)?;
11                tokens.push(token);
12            }
13            None => break,
14        }
15    }
16
17    Ok(tokens)
18}
19
20fn to_token(c: char) -> Result<Token, Error> {
21    match c {
22        letter if letter.is_alphabetic() => Ok(Token::Letter(c)),
23        number if number.is_digit(10) => match number.to_digit(10) {
24            Some(digit) => Ok(Token::Number(digit)),
25            None => Err(Error::LexerError(format!(
26                "failed to convert {} to digit",
27                c
28            ))),
29        },
30        ' ' => Ok(Token::Space),
31        '-' => Ok(Token::Dash),
32        ':' => Ok(Token::Colon),
33        '/' => Ok(Token::Slash),
34        '|' => Ok(Token::Pipe),
35        '.' => Ok(Token::Dot),
36        ',' => Ok(Token::Comma),
37        '\'' => Ok(Token::SingleQuote),
38        '"' => Ok(Token::DoubleQuote),
39        '*' => Ok(Token::Asterisk),
40        '=' => Ok(Token::Equals),
41        '%' => Ok(Token::Percent),
42        '\n' => Ok(Token::Newline),
43        '(' => Ok(Token::LeftParen),
44        ')' => Ok(Token::RightParen),
45        '[' => Ok(Token::LeftBracket),
46        ']' => Ok(Token::RightBracket),
47        '{' => Ok(Token::LeftBrace),
48        '}' => Ok(Token::RightBrace),
49        '$' => Ok(Token::Dollar),
50        _ => Err(Error::LexerError(format!("unrecognized input '{}'", c))),
51    }
52}
53
54#[cfg(test)]
55mod tests {
56    use super::*;
57
58    #[test]
59    fn to_token() {
60        let tokens = tokenize("a1 -:/|.,'\"*=%\n()[]{}$").expect("failed to tokenize");
61        assert_eq!(
62            tokens,
63            vec![
64                Token::Letter('a'),
65                Token::Number(1),
66                Token::Space,
67                Token::Dash,
68                Token::Colon,
69                Token::Slash,
70                Token::Pipe,
71                Token::Dot,
72                Token::Comma,
73                Token::SingleQuote,
74                Token::DoubleQuote,
75                Token::Asterisk,
76                Token::Equals,
77                Token::Percent,
78                Token::Newline,
79                Token::LeftParen,
80                Token::RightParen,
81                Token::LeftBracket,
82                Token::RightBracket,
83                Token::LeftBrace,
84                Token::RightBrace,
85                Token::Dollar,
86            ]
87        );
88    }
89}
helm_template_parser/lexer.rs

helm_template_parser/
lexer.rs
