pipeline_script/lexer/
iter.rs

1use crate::lexer::position::Position;
2use crate::lexer::token::Token;
3use crate::lexer::Lexer;
4
5pub struct TokenStream {
6    tokenizer: Lexer,
7    peek: Vec<(Token, Position)>,
8}
9
10impl Iterator for TokenStream {
11    type Item = (Token, Position);
12
13    fn next(&mut self) -> Option<Self::Item> {
14        match self.next_token() {
15            (Token::Eof, _) => None,
16            t => Some(t),
17        }
18    }
19}
20
21impl TokenStream {
22    pub fn new(lexer: Lexer) -> Self {
23        Self {
24            tokenizer: lexer,
25            peek: Vec::new(),
26        }
27    }
28    pub fn is_eof(&self) -> bool {
29        self.peek.is_empty() && self.tokenizer.is_eof()
30    }
31    pub fn set_lexer(&mut self, lexer: Lexer) {
32        self.tokenizer = lexer;
33    }
34    pub fn next_token(&mut self) -> (Token, Position) {
35        if !self.peek.is_empty() {
36            return self.peek.remove(0);
37        }
38        self.tokenizer
39            .next_token()
40            .unwrap_or((Token::Eof, Position::none()))
41    }
42    pub fn peek(&mut self) -> (Token, Position) {
43        self.peek_nth(0)
44    }
45    pub fn peek_nth(&mut self, n: usize) -> (Token, Position) {
46        // 确保缓存中有足够的元素
47        while self.peek.len() <= n {
48            match self.tokenizer.next_token() {
49                Some(token) => self.peek.push(token),
50                None => self.peek.push((Token::Eof, Position::none())),
51            }
52        }
53
54        // 返回第n个元素的克隆
55        self.peek[n].clone()
56    }
57
58    // 获取从当前位置开始的n个token
59    pub fn peek_n(&mut self, n: usize) -> Vec<(Token, Position)> {
60        // 确保缓存中有足够的元素
61        while self.peek.len() < n {
62            match self.tokenizer.next_token() {
63                Some(token) => self.peek.push(token),
64                None => self.peek.push((Token::Eof, Position::none())),
65            }
66        }
67
68        // 返回前n个元素的克隆
69        self.peek.iter().take(n).cloned().collect()
70    }
71}
pipeline_script/lexer/iter.rs

pipeline_script/lexer/
iter.rs
