litex/parse/
token_block.rs

1use super::tokenizer::tokenize_line;
2use crate::prelude::*;
3use std::rc::Rc;
4
5#[derive(Debug, Clone, PartialEq, Eq)]
6pub struct TokenBlock {
7    pub header: Vec<String>,
8    pub body: Vec<TokenBlock>,
9    pub line_file: LineFile,
10    pub parse_index: usize,
11}
12
13fn indent_level(line: &str) -> usize {
14    let mut n = 0;
15    for c in line.chars() {
16        match c {
17            ' ' => n += 1,
18            '\t' => n += 4,
19            _ => break,
20        }
21    }
22    n
23}
24
25fn ends_with_colon(s: &str) -> bool {
26    let trimmed = s.trim_end();
27    trimmed.ends_with(COLON)
28}
29
30impl TokenBlock {
31    pub fn parse_blocks(
32        s: &str,
33        current_file_path: Rc<str>,
34    ) -> Result<Vec<TokenBlock>, RuntimeError> {
35        let stripped_source_code = strip_triple_quote_comment_blocks(s);
36        let lines: Vec<_> = stripped_source_code.lines().collect();
37        let mut i = 0;
38        parse_level(&lines, &mut i, 0, current_file_path)
39    }
40}
41
42fn strip_triple_quote_comment_blocks(source_code: &str) -> String {
43    // Treat a line that consists only of `"` characters (after trimming) as a delimiter.
44    // Between two delimiter lines, everything is replaced with empty lines so
45    // the parser will ignore those lines.
46    let mut in_comment = false;
47    let line_count_upper_bound = source_code.lines().count();
48    let mut out_lines: Vec<String> = Vec::with_capacity(line_count_upper_bound);
49
50    for line in source_code.lines() {
51        let trimmed = line.trim();
52        let only_quote_chars = !trimmed.is_empty() && trimmed.chars().all(|c| c == '"');
53        if only_quote_chars {
54            in_comment = !in_comment;
55            out_lines.push(String::new());
56            continue;
57        }
58
59        if in_comment {
60            out_lines.push(String::new());
61        } else {
62            out_lines.push(line.to_string());
63        }
64    }
65
66    out_lines.join("\n")
67}
68
69fn parse_level(
70    lines: &[&str],
71    i: &mut usize,
72    base_indent: usize,
73    current_file_path: Rc<str>,
74) -> Result<Vec<TokenBlock>, RuntimeError> {
75    let remaining_line_count_upper_bound = lines.len().saturating_sub(*i);
76    let mut items = Vec::with_capacity(remaining_line_count_upper_bound);
77    let mut body_indent = None;
78
79    while *i < lines.len() {
80        let raw = lines[*i];
81        let line_no = *i + 1;
82        let line_file = (line_no, current_file_path.clone());
83        let indent = indent_level(raw);
84        let content = raw.trim();
85
86        if content.is_empty() {
87            *i += 1;
88            continue;
89        }
90
91        if indent < base_indent {
92            break;
93        }
94
95        if indent > base_indent {
96            return Err({
97                RuntimeError::from(ParseRuntimeError(RuntimeErrorStruct::new_with_msg_and_line_file(format!(
98                        "unexpected indent at line {} in {}",
99                        line_file.0,
100                        line_file.1.as_ref()
101                    ), line_file)))
102            });
103        }
104
105        *i += 1;
106
107        // Tokenize header; if it's empty (e.g. whole line comment),
108        // treat it like a blank line for block parsing.
109        let header_tokens = tokenize_line(content);
110        if header_tokens.is_empty() {
111            continue;
112        }
113
114        if ends_with_colon(content) {
115            // 必须有 body
116            if *i >= lines.len() {
117                return Err({
118                    let line_file = (line_no, current_file_path.clone());
119                    RuntimeError::from(ParseRuntimeError(RuntimeErrorStruct::new_with_msg_and_line_file(format!(
120                            "block header missing body at line {} in {}",
121                            line_file.0,
122                            line_file.1.as_ref()
123                        ), line_file)))
124                });
125            }
126
127            let next_indent = indent_level(lines[*i]);
128            if next_indent <= indent {
129                return Err({
130                    let line_file = (*i + 1, current_file_path.clone());
131                    RuntimeError::from(ParseRuntimeError(RuntimeErrorStruct::new_with_msg_and_line_file(format!(
132                            "expected indent at line {} in {}",
133                            line_file.0,
134                            line_file.1.as_ref()
135                        ), line_file)))
136                });
137            }
138
139            let body = parse_level(lines, i, next_indent, current_file_path.clone())?;
140            items.push(TokenBlock::new(
141                header_tokens,
142                body,
143                (line_no, current_file_path.clone()),
144            ));
145        } else {
146            items.push(TokenBlock::new(
147                header_tokens,
148                vec![],
149                (line_no, current_file_path.clone()),
150            ));
151        }
152
153        if let Some(expected) = body_indent {
154            if indent != expected {
155                return Err({
156                    let line_file = (line_no, current_file_path.clone());
157                    RuntimeError::from(ParseRuntimeError(RuntimeErrorStruct::new_with_msg_and_line_file(format!(
158                            "inconsistent indent at line {} in {}",
159                            line_file.0,
160                            line_file.1.as_ref()
161                        ), line_file)))
162                });
163            }
164        } else {
165            body_indent = Some(indent);
166        }
167    }
168
169    Ok(items)
170}
171
172impl TokenBlock {
173    /// 返回当前 token；若已读完则返回 Error。
174    pub fn current(&self) -> Result<&str, RuntimeError> {
175        self.header
176            .get(self.parse_index)
177            .map(|s| s.as_str())
178            .ok_or_else(|| {
179                RuntimeError::from(ParseRuntimeError(RuntimeErrorStruct::new_with_msg_and_line_file("Unexpected end of tokens".to_string(), self.line_file.clone())))
180            })
181    }
182
183    pub fn skip_token(self: &mut Self, token: &str) -> Result<(), RuntimeError> {
184        if self.current()? == token {
185            self.parse_index += 1;
186            Ok(())
187        } else {
188            Err(RuntimeError::from(ParseRuntimeError(
189                RuntimeErrorStruct::new_with_msg_and_line_file(format!("Expected token: {}", token), self.line_file.clone()),
190            )))
191        }
192    }
193
194    pub fn advance(&mut self) -> Result<String, RuntimeError> {
195        let t = self.current()?.to_string();
196        self.parse_index += 1;
197        Ok(t)
198    }
199
200    pub fn skip(&mut self) -> Result<(), RuntimeError> {
201        self.current()?;
202        self.parse_index += 1;
203        Ok(())
204    }
205
206    pub fn exceed_end_of_head(&self) -> bool {
207        return self.parse_index >= self.header.len();
208    }
209
210    pub fn skip_token_and_colon_and_exceed_end_of_head(
211        &mut self,
212        token: &str,
213    ) -> Result<(), RuntimeError> {
214        self.skip_token(token)?;
215        self.skip_token(COLON)?;
216        if !self.exceed_end_of_head() {
217            return Err(RuntimeError::from(ParseRuntimeError(
218                RuntimeErrorStruct::new_with_msg_and_line_file("Expected token: at head".to_string(), self.line_file.clone()),
219            )));
220        }
221        Ok(())
222    }
223
224    pub fn token_at_index(&self, index: usize) -> Result<&str, RuntimeError> {
225        self.header.get(index).map(|s| s.as_str()).ok_or_else(|| {
226            RuntimeError::from(ParseRuntimeError(RuntimeErrorStruct::new_with_msg_and_line_file(format!("Expected token: at index {}", index), self.line_file.clone())))
227        })
228    }
229
230    pub fn current_token_empty_if_exceed_end_of_head(&self) -> &str {
231        if self.exceed_end_of_head() {
232            return "";
233        }
234        self.header
235            .get(self.parse_index)
236            .map(|s| s.as_str())
237            .unwrap_or("")
238    }
239
240    pub fn current_token_is_equal_to(&self, token: &str) -> bool {
241        self.current_token_empty_if_exceed_end_of_head() == token
242    }
243
244    pub fn token_at_end_of_head(&self) -> &str {
245        self.header
246            .get(self.header.len() - 1)
247            .map(|s| s.as_str())
248            .unwrap_or("")
249    }
250
251    pub fn token_at_add_index(&self, index: usize) -> &str {
252        self.header
253            .get(self.parse_index + index)
254            .map(|s| s.as_str())
255            .unwrap_or("")
256    }
257}
258
259impl TokenBlock {
260    pub fn new(tokens: Vec<String>, body: Vec<TokenBlock>, line_file: LineFile) -> TokenBlock {
261        TokenBlock {
262            header: tokens,
263            body,
264            line_file,
265            parse_index: 0,
266        }
267    }
268}
litex/parse/token_block.rs

litex/parse/
token_block.rs
