sqparse/lexer/
mod.rs

1use crate::lexer::token_iter::TokenIter;
2use crate::token::{TerminalToken, Token, TokenType};
3use crate::Flavor;
4use std::collections::VecDeque;
5
6mod comment;
7mod error;
8mod identifier;
9mod literal;
10mod parse_str;
11mod symbol;
12mod token_iter;
13
14pub use self::error::{LexerError, LexerErrorType};
15
16/// A token with attached metadata.
17#[derive(Debug, Clone, PartialEq)]
18pub struct TokenItem<'s> {
19    /// The actual token.
20    pub token: Token<'s>,
21
22    /// The index of the corresponding closing delimiter token, if this token is an opening
23    /// delimiter.
24    ///
25    /// # Example
26    /// ```text
27    /// { some other tokens }
28    /// ^ open              ^ close
29    /// ```
30    /// In this example, the opening `{` token would have a `close_index` of 5, the index of the
31    /// closing delimiter.
32    pub close_index: Option<usize>,
33}
34
35// Returns the token that closes a tree, if the provided token is a valid opening token.
36fn closing_token(opening: TokenType) -> Option<TokenType> {
37    match opening {
38        TokenType::Terminal(TerminalToken::OpenBrace) => {
39            Some(TokenType::Terminal(TerminalToken::CloseBrace))
40        }
41        TokenType::Terminal(TerminalToken::OpenSquare) => {
42            Some(TokenType::Terminal(TerminalToken::CloseSquare))
43        }
44        TokenType::Terminal(TerminalToken::OpenBracket) => {
45            Some(TokenType::Terminal(TerminalToken::CloseBracket))
46        }
47        TokenType::Terminal(TerminalToken::OpenAttributes) => {
48            Some(TokenType::Terminal(TerminalToken::CloseAttributes))
49        }
50        _ => None,
51    }
52}
53
54struct Layer<'s> {
55    open_index: usize,
56    close_ty: TokenType<'s>,
57}
58
59/// Parses an input string into a list of tokens.
60///
61/// # Example
62/// ```
63/// use sqparse::{Flavor, tokenize};
64///
65/// let source = r#"
66/// global function MyFunction
67///
68/// struct {
69///     int a
70/// } file
71///
72/// string function MyFunction( List<number> values ) {
73///     values.push(1 + 2)
74/// }
75/// "#;
76///
77/// let tokens = tokenize(source, Flavor::SquirrelRespawn).unwrap();
78/// assert_eq!(tokens.len(), 29);
79/// ```
80pub fn tokenize(val: &str, flavor: Flavor) -> Result<Vec<TokenItem>, LexerError> {
81    let mut items = Vec::<TokenItem>::new();
82    let mut layers = VecDeque::<Layer>::new();
83
84    for maybe_token in TokenIter::new(val, flavor) {
85        let token = maybe_token?;
86        let token_index = items.len();
87
88        // If this token matches the top layer's close token, pop the layer.
89        if let Some(top_layer) = layers.back() {
90            if top_layer.close_ty == token.ty {
91                items[top_layer.open_index].close_index = Some(token_index);
92                layers.pop_back();
93            }
94        }
95
96        // If this token is a valid opening token, push a new layer.
97        if let Some(close_ty) = closing_token(token.ty) {
98            layers.push_back(Layer {
99                open_index: token_index,
100                close_ty,
101            });
102        }
103
104        items.push(TokenItem {
105            token,
106            close_index: None,
107        });
108    }
109
110    // If there are remaining layers, there are one or more unmatched opening tokens. Otherwise
111    // at this point tokenization is successful.
112    match layers.back() {
113        None => Ok(items),
114        Some(layer) => {
115            let open_token = &items[layer.open_index].token;
116            Err(LexerError::new(
117                LexerErrorType::UnmatchedOpener {
118                    open: open_token.ty,
119                    close: layer.close_ty,
120                },
121                open_token.range.clone(),
122            ))
123        }
124    }
125}
sqparse/lexer/mod.rs

sqparse/lexer/
mod.rs
