litex/parse/
token_block.rs

1use super::tokenizer::tokenize_line;
2use crate::prelude::*;
3use std::rc::Rc;
4
5#[derive(Debug, Clone, PartialEq, Eq)]
6pub struct TokenBlock {
7    pub header: Vec<String>,
8    pub body: Vec<TokenBlock>,
9    pub line_file: LineFile,
10    pub parse_index: usize,
11}
12
13fn indent_level(line: &str) -> usize {
14    let mut n = 0;
15    for c in line.chars() {
16        match c {
17            ' ' => n += 1,
18            '\t' => n += 4,
19            _ => break,
20        }
21    }
22    n
23}
24
25fn ends_with_colon(s: &str) -> bool {
26    let trimmed = s.trim_end();
27    trimmed.ends_with(':') || trimmed.ends_with('：')
28}
29
30impl TokenBlock {
31    pub fn parse_blocks(
32        s: &str,
33        current_file_path: Rc<str>,
34    ) -> Result<Vec<TokenBlock>, RuntimeError> {
35        let stripped_source_code = strip_triple_quote_comment_blocks(s);
36        let lines: Vec<_> = stripped_source_code.lines().collect();
37        let mut i = 0;
38        parse_level(&lines, &mut i, 0, current_file_path)
39    }
40}
41
42fn strip_triple_quote_comment_blocks(source_code: &str) -> String {
43    // Treat a line that consists only of `"` characters (after trimming) as a delimiter.
44    // Between two delimiter lines, everything is replaced with empty lines so
45    // the parser will ignore those lines.
46    let mut in_comment = false;
47    let line_count_upper_bound = source_code.lines().count();
48    let mut out_lines: Vec<String> = Vec::with_capacity(line_count_upper_bound);
49
50    for line in source_code.lines() {
51        let trimmed = line.trim();
52        let only_quote_chars = !trimmed.is_empty() && trimmed.chars().all(|c| c == '"');
53        if only_quote_chars {
54            in_comment = !in_comment;
55            out_lines.push(String::new());
56            continue;
57        }
58
59        if in_comment {
60            out_lines.push(String::new());
61        } else {
62            out_lines.push(line.to_string());
63        }
64    }
65
66    out_lines.join("\n")
67}
68
69fn parse_level(
70    lines: &[&str],
71    i: &mut usize,
72    base_indent: usize,
73    current_file_path: Rc<str>,
74) -> Result<Vec<TokenBlock>, RuntimeError> {
75    let remaining_line_count_upper_bound = lines.len().saturating_sub(*i);
76    let mut items = Vec::with_capacity(remaining_line_count_upper_bound);
77    let mut body_indent = None;
78
79    while *i < lines.len() {
80        let raw = lines[*i];
81        let line_no = *i + 1;
82        let indent = indent_level(raw);
83        let content = raw.trim();
84
85        if content.is_empty() {
86            *i += 1;
87            continue;
88        }
89
90        if indent < base_indent {
91            break;
92        }
93
94        if indent > base_indent {
95            return Err({
96                let line_file = (line_no, current_file_path.clone());
97                RuntimeError::from(ParseRuntimeError(RuntimeErrorStruct::new(
98                    None,
99                    format!(
100                        "unexpected indent at line {} in {}",
101                        line_file.0,
102                        line_file.1.as_ref()
103                    ),
104                    line_file,
105                    None,
106                    vec![],
107                )))
108            });
109        }
110
111        *i += 1;
112
113        // Tokenize header; if it's empty (e.g. whole line comment),
114        // treat it like a blank line for block parsing.
115        let header_tokens = tokenize_line(content);
116        if header_tokens.is_empty() {
117            continue;
118        }
119
120        if ends_with_colon(content) {
121            // 必须有 body
122            if *i >= lines.len() {
123                return Err({
124                    let line_file = (line_no, current_file_path.clone());
125                    RuntimeError::from(ParseRuntimeError(RuntimeErrorStruct::new(
126                        None,
127                        format!(
128                            "block header missing body at line {} in {}",
129                            line_file.0,
130                            line_file.1.as_ref()
131                        ),
132                        line_file,
133                        None,
134                        vec![],
135                    )))
136                });
137            }
138
139            let next_indent = indent_level(lines[*i]);
140            if next_indent <= indent {
141                return Err({
142                    let line_file = (*i + 1, current_file_path.clone());
143                    RuntimeError::from(ParseRuntimeError(RuntimeErrorStruct::new(
144                        None,
145                        format!(
146                            "expected indent at line {} in {}",
147                            line_file.0,
148                            line_file.1.as_ref()
149                        ),
150                        line_file,
151                        None,
152                        vec![],
153                    )))
154                });
155            }
156
157            let body = parse_level(lines, i, next_indent, current_file_path.clone())?;
158            items.push(TokenBlock::new(
159                header_tokens,
160                body,
161                (line_no, current_file_path.clone()),
162            ));
163        } else {
164            items.push(TokenBlock::new(
165                header_tokens,
166                vec![],
167                (line_no, current_file_path.clone()),
168            ));
169        }
170
171        if let Some(expected) = body_indent {
172            if indent != expected {
173                return Err({
174                    let line_file = (line_no, current_file_path.clone());
175                    RuntimeError::from(ParseRuntimeError(RuntimeErrorStruct::new(
176                        None,
177                        format!(
178                            "inconsistent indent at line {} in {}",
179                            line_file.0,
180                            line_file.1.as_ref()
181                        ),
182                        line_file,
183                        None,
184                        vec![],
185                    )))
186                });
187            }
188        } else {
189            body_indent = Some(indent);
190        }
191    }
192
193    Ok(items)
194}
195
196impl TokenBlock {
197    /// 返回当前 token；若已读完则返回 Error。
198    pub fn current(&self) -> Result<&str, RuntimeError> {
199        self.header
200            .get(self.parse_index)
201            .map(|s| s.as_str())
202            .ok_or_else(|| {
203                RuntimeError::from(ParseRuntimeError(RuntimeErrorStruct::new(
204                    None,
205                    "Unexpected end of tokens".to_string(),
206                    self.line_file.clone(),
207                    None,
208                    vec![],
209                )))
210            })
211    }
212
213    pub fn skip_token(self: &mut Self, token: &str) -> Result<(), RuntimeError> {
214        if self.current()? == token {
215            self.parse_index += 1;
216            Ok(())
217        } else {
218            Err(RuntimeError::from(ParseRuntimeError(
219                RuntimeErrorStruct::new(
220                    None,
221                    format!("Expected token: {}", token),
222                    self.line_file.clone(),
223                    None,
224                    vec![],
225                ),
226            )))
227        }
228    }
229
230    pub fn advance(&mut self) -> Result<String, RuntimeError> {
231        let t = self.current()?.to_string();
232        self.parse_index += 1;
233        Ok(t)
234    }
235
236    pub fn skip(&mut self) -> Result<(), RuntimeError> {
237        self.current()?;
238        self.parse_index += 1;
239        Ok(())
240    }
241
242    pub fn exceed_end_of_head(&self) -> bool {
243        return self.parse_index >= self.header.len();
244    }
245
246    pub fn skip_token_and_colon_and_exceed_end_of_head(
247        &mut self,
248        token: &str,
249    ) -> Result<(), RuntimeError> {
250        self.skip_token(token)?;
251        self.skip_token(COLON)?;
252        if !self.exceed_end_of_head() {
253            return Err(RuntimeError::from(ParseRuntimeError(
254                RuntimeErrorStruct::new(
255                    None,
256                    "Expected token: at head".to_string(),
257                    self.line_file.clone(),
258                    None,
259                    vec![],
260                ),
261            )));
262        }
263        Ok(())
264    }
265
266    pub fn token_at_index(&self, index: usize) -> Result<&str, RuntimeError> {
267        self.header.get(index).map(|s| s.as_str()).ok_or_else(|| {
268            RuntimeError::from(ParseRuntimeError(RuntimeErrorStruct::new(
269                None,
270                format!("Expected token: at index {}", index),
271                self.line_file.clone(),
272                None,
273                vec![],
274            )))
275        })
276    }
277
278    pub fn current_token_empty_if_exceed_end_of_head(&self) -> &str {
279        if self.exceed_end_of_head() {
280            return "";
281        }
282        self.header
283            .get(self.parse_index)
284            .map(|s| s.as_str())
285            .unwrap_or("")
286    }
287
288    pub fn current_token_is_equal_to(&self, token: &str) -> bool {
289        self.current_token_empty_if_exceed_end_of_head() == token
290    }
291
292    pub fn token_at_end_of_head(&self) -> &str {
293        self.header
294            .get(self.header.len() - 1)
295            .map(|s| s.as_str())
296            .unwrap_or("")
297    }
298
299    pub fn token_at_add_index(&self, index: usize) -> &str {
300        self.header
301            .get(self.parse_index + index)
302            .map(|s| s.as_str())
303            .unwrap_or("")
304    }
305}
306
307impl TokenBlock {
308    pub fn new(tokens: Vec<String>, body: Vec<TokenBlock>, line_file: LineFile) -> TokenBlock {
309        TokenBlock {
310            header: tokens,
311            body,
312            line_file,
313            parse_index: 0,
314        }
315    }
316}
litex/parse/token_block.rs

litex/parse/
token_block.rs
