litex/parse/
token_block.rs

1use super::tokenizer::tokenize_line;
2use crate::prelude::*;
3use std::rc::Rc;
4
5#[derive(Debug, Clone, PartialEq, Eq)]
6pub struct TokenBlock {
7    pub header: Vec<String>,
8    pub body: Vec<TokenBlock>,
9    pub line_file: LineFile,
10    pub parse_index: usize,
11}
12
13fn indent_level(line: &str) -> usize {
14    let mut n = 0;
15    for c in line.chars() {
16        match c {
17            ' ' => n += 1,
18            '\t' => n += 4,
19            _ => break,
20        }
21    }
22    n
23}
24
25fn ends_with_colon(s: &str) -> bool {
26    let trimmed = s.trim_end();
27    trimmed.ends_with(COLON)
28}
29
30impl TokenBlock {
31    pub fn parse_blocks(
32        s: &str,
33        current_file_path: Rc<str>,
34    ) -> Result<Vec<TokenBlock>, RuntimeError> {
35        let stripped_source_code = strip_triple_quote_comment_blocks(s);
36        let lines: Vec<_> = stripped_source_code.lines().collect();
37        let mut i = 0;
38        parse_level(&lines, &mut i, 0, current_file_path)
39    }
40}
41
42fn strip_triple_quote_comment_blocks(source_code: &str) -> String {
43    // Treat a line that consists only of `"` characters (after trimming) as a delimiter.
44    // Between two delimiter lines, everything is replaced with empty lines so
45    // the parser will ignore those lines.
46    let mut in_comment = false;
47    let line_count_upper_bound = source_code.lines().count();
48    let mut out_lines: Vec<String> = Vec::with_capacity(line_count_upper_bound);
49
50    for line in source_code.lines() {
51        let trimmed = line.trim();
52        let only_quote_chars = !trimmed.is_empty() && trimmed.chars().all(|c| c == '"');
53        if only_quote_chars {
54            in_comment = !in_comment;
55            out_lines.push(String::new());
56            continue;
57        }
58
59        if in_comment {
60            out_lines.push(String::new());
61        } else {
62            out_lines.push(line.to_string());
63        }
64    }
65
66    out_lines.join("\n")
67}
68
69fn parse_level(
70    lines: &[&str],
71    i: &mut usize,
72    base_indent: usize,
73    current_file_path: Rc<str>,
74) -> Result<Vec<TokenBlock>, RuntimeError> {
75    let remaining_line_count_upper_bound = lines.len().saturating_sub(*i);
76    let mut items = Vec::with_capacity(remaining_line_count_upper_bound);
77    let mut body_indent = None;
78
79    while *i < lines.len() {
80        let raw = lines[*i];
81        let line_no = *i + 1;
82        let line_file = (line_no, current_file_path.clone());
83        let indent = indent_level(raw);
84        let content = raw.trim();
85
86        if content.is_empty() {
87            *i += 1;
88            continue;
89        }
90
91        if indent < base_indent {
92            break;
93        }
94
95        if indent > base_indent {
96            // Indented farther than the current block: normally a syntax error, but allow lines that
97            // are only a `# ...` comment (any leading spaces/tabs before `#`). Writers often align
98            // such comments with a surrounding block without attaching them to a sibling item.
99            let trimmed_start = raw.trim_start();
100            if trimmed_start.is_empty() || trimmed_start.starts_with('#') {
101                *i += 1;
102                continue;
103            }
104            return Err({
105                RuntimeError::from(ParseRuntimeError(RuntimeErrorStruct::new_with_msg_and_line_file(format!(
106                        "unexpected indent at line {} in {}",
107                        line_file.0,
108                        line_file.1.as_ref()
109                    ), line_file)))
110            });
111        }
112
113        *i += 1;
114
115        // Tokenize header; if it's empty (e.g. whole line comment),
116        // treat it like a blank line for block parsing.
117        let header_tokens = tokenize_line(content);
118        if header_tokens.is_empty() {
119            continue;
120        }
121
122        if ends_with_colon(content) {
123            // 必须有 body
124            if *i >= lines.len() {
125                return Err({
126                    let line_file = (line_no, current_file_path.clone());
127                    RuntimeError::from(ParseRuntimeError(RuntimeErrorStruct::new_with_msg_and_line_file(format!(
128                            "block header missing body at line {} in {}",
129                            line_file.0,
130                            line_file.1.as_ref()
131                        ), line_file)))
132                });
133            }
134
135            let next_indent = indent_level(lines[*i]);
136            if next_indent <= indent {
137                return Err({
138                    let line_file = (*i + 1, current_file_path.clone());
139                    RuntimeError::from(ParseRuntimeError(RuntimeErrorStruct::new_with_msg_and_line_file(format!(
140                            "expected indent at line {} in {}",
141                            line_file.0,
142                            line_file.1.as_ref()
143                        ), line_file)))
144                });
145            }
146
147            let body = parse_level(lines, i, next_indent, current_file_path.clone())?;
148            items.push(TokenBlock::new(
149                header_tokens,
150                body,
151                (line_no, current_file_path.clone()),
152            ));
153        } else {
154            items.push(TokenBlock::new(
155                header_tokens,
156                vec![],
157                (line_no, current_file_path.clone()),
158            ));
159        }
160
161        if let Some(expected) = body_indent {
162            if indent != expected {
163                return Err({
164                    let line_file = (line_no, current_file_path.clone());
165                    RuntimeError::from(ParseRuntimeError(RuntimeErrorStruct::new_with_msg_and_line_file(format!(
166                            "inconsistent indent at line {} in {}",
167                            line_file.0,
168                            line_file.1.as_ref()
169                        ), line_file)))
170                });
171            }
172        } else {
173            body_indent = Some(indent);
174        }
175    }
176
177    Ok(items)
178}
179
180impl TokenBlock {
181    /// 返回当前 token；若已读完则返回 Error。
182    pub fn current(&self) -> Result<&str, RuntimeError> {
183        self.header
184            .get(self.parse_index)
185            .map(|s| s.as_str())
186            .ok_or_else(|| {
187                RuntimeError::from(ParseRuntimeError(RuntimeErrorStruct::new_with_msg_and_line_file("Unexpected end of tokens".to_string(), self.line_file.clone())))
188            })
189    }
190
191    pub fn skip_token(self: &mut Self, token: &str) -> Result<(), RuntimeError> {
192        if self.current()? == token {
193            self.parse_index += 1;
194            Ok(())
195        } else {
196            Err(RuntimeError::from(ParseRuntimeError(
197                RuntimeErrorStruct::new_with_msg_and_line_file(format!("Expected token: {}", token), self.line_file.clone()),
198            )))
199        }
200    }
201
202    pub fn advance(&mut self) -> Result<String, RuntimeError> {
203        let t = self.current()?.to_string();
204        self.parse_index += 1;
205        Ok(t)
206    }
207
208    pub fn skip(&mut self) -> Result<(), RuntimeError> {
209        self.current()?;
210        self.parse_index += 1;
211        Ok(())
212    }
213
214    pub fn exceed_end_of_head(&self) -> bool {
215        return self.parse_index >= self.header.len();
216    }
217
218    pub fn skip_token_and_colon_and_exceed_end_of_head(
219        &mut self,
220        token: &str,
221    ) -> Result<(), RuntimeError> {
222        self.skip_token(token)?;
223        self.skip_token(COLON)?;
224        if !self.exceed_end_of_head() {
225            return Err(RuntimeError::from(ParseRuntimeError(
226                RuntimeErrorStruct::new_with_msg_and_line_file("Expected token: at head".to_string(), self.line_file.clone()),
227            )));
228        }
229        Ok(())
230    }
231
232    pub fn token_at_index(&self, index: usize) -> Result<&str, RuntimeError> {
233        self.header.get(index).map(|s| s.as_str()).ok_or_else(|| {
234            RuntimeError::from(ParseRuntimeError(RuntimeErrorStruct::new_with_msg_and_line_file(format!("Expected token: at index {}", index), self.line_file.clone())))
235        })
236    }
237
238    pub fn current_token_empty_if_exceed_end_of_head(&self) -> &str {
239        if self.exceed_end_of_head() {
240            return "";
241        }
242        self.header
243            .get(self.parse_index)
244            .map(|s| s.as_str())
245            .unwrap_or("")
246    }
247
248    pub fn current_token_is_equal_to(&self, token: &str) -> bool {
249        self.current_token_empty_if_exceed_end_of_head() == token
250    }
251
252    pub fn token_at_end_of_head(&self) -> &str {
253        self.header
254            .get(self.header.len() - 1)
255            .map(|s| s.as_str())
256            .unwrap_or("")
257    }
258
259    pub fn token_at_add_index(&self, index: usize) -> &str {
260        self.header
261            .get(self.parse_index + index)
262            .map(|s| s.as_str())
263            .unwrap_or("")
264    }
265}
266
267impl TokenBlock {
268    pub fn new(tokens: Vec<String>, body: Vec<TokenBlock>, line_file: LineFile) -> TokenBlock {
269        TokenBlock {
270            header: tokens,
271            body,
272            line_file,
273            parse_index: 0,
274        }
275    }
276}
litex/parse/token_block.rs

litex/parse/
token_block.rs
