bytebraise_syntax/parser/
parser.rs

1use std::iter::Peekable;
2
3use crate::syntax::ast::nodes::Root;
4use crate::syntax::ast::AstNode;
5use crate::syntax::syntax_kind::{SyntaxKind, syntax_kind_for_token_kind};
6use crate::syntax::syntax_node::{SyntaxElement, SyntaxNode};
7use bytebraise_lexer::lexer::tokenize;
8use rowan::{Checkpoint, GreenNodeBuilder, NodeOrToken};
9
10struct Parser<'text, I: Iterator<Item = (SyntaxKind, &'text str)>> {
11    builder: GreenNodeBuilder<'static>,
12    iter: Peekable<I>,
13}
14
15struct TaskContext {
16    is_python: bool,
17    is_fakeroot: bool,
18}
19
20impl<'text, I: Iterator<Item = (SyntaxKind, &'text str)>> Parser<'text, I> {
21    fn eat_ws(&mut self) {
22        while self
23            .iter
24            .peek()
25            .map(|&(t, _)| t == SyntaxKind::Whitespace)
26            .unwrap_or(false)
27        {
28            self.bump();
29        }
30    }
31
32    fn peek(&mut self) -> Option<SyntaxKind> {
33        self.eat_ws();
34        self.iter.peek().map(|&(t, _)| t)
35    }
36
37    fn peek_no_ws(&mut self) -> Option<SyntaxKind> {
38        self.iter.peek().map(|&(t, _)| t)
39    }
40
41    fn bump(&mut self) {
42        if let Some((token, string)) = self.iter.next() {
43            self.builder.token(token.into(), string);
44        }
45    }
46
47    fn parse_include_or_require(&mut self) -> bool {
48        let checkpoint = self.builder.checkpoint();
49
50        match self.peek() {
51            Some(SyntaxKind::Require) => self
52                .builder
53                .start_node_at(checkpoint, SyntaxKind::RequireNode.into()),
54            Some(SyntaxKind::Include) => self
55                .builder
56                .start_node_at(checkpoint, SyntaxKind::IncludeNode.into()),
57            _ => unreachable!(),
58        }
59
60        self.bump();
61
62        self.expect(SyntaxKind::UnquotedValue);
63
64        self.builder.finish_node();
65
66        true
67    }
68
69    fn parse_export(&mut self) -> bool {
70        self.builder.start_node(SyntaxKind::ExportNode.into());
71        self.expect(SyntaxKind::Export);
72        self.eat_ws();
73
74        let checkpoint = self.builder.checkpoint();
75
76        self.parse_identifier_expression();
77        match self.peek() {
78            Some(t) if t.is_assignment_operator() => {
79                self.builder
80                    .start_node_at(checkpoint, SyntaxKind::AssignmentNode.into());
81                self.bump();
82                match self.peek() {
83                    Some(SyntaxKind::DoubleQuotedValue | SyntaxKind::SingleQuotedValue) => {
84                        self.bump()
85                    }
86                    _ => panic!(),
87                }
88                self.builder.finish_node();
89            }
90            Some(SyntaxKind::Newline) | None => {} // fine, this is just an export
91            _ => panic!("{:?}", self.peek()),
92        }
93
94        self.builder.finish_node();
95        true
96    }
97
98    fn parse_assignment_or_task(
99        &mut self,
100        task_checkpoint: Option<Checkpoint>,
101        maybe_anonymous_python: bool,
102    ) -> bool {
103        let checkpoint = self.builder.checkpoint();
104        if !self.parse_identifier_expression() && !maybe_anonymous_python {
105            return false; // TODO
106        }
107
108        if self.allow(SyntaxKind::OpenParenthesis) {
109            // TODO check no varflag with task
110
111            self.expect(SyntaxKind::CloseParenthesis);
112            self.expect(SyntaxKind::Task);
113            let checkpoint = task_checkpoint.unwrap_or(checkpoint);
114            self.builder
115                .start_node_at(checkpoint, SyntaxKind::TaskNode.into());
116            self.builder.finish_node();
117            return true;
118        } else {
119            assert!(task_checkpoint.is_none(), "required a task in this context")
120        };
121
122        // Has to be an assignment
123        self.builder
124            .start_node_at(checkpoint, SyntaxKind::AssignmentNode.into());
125        match self.peek() {
126            Some(t) if t.is_assignment_operator() => self.bump(),
127            _ => panic!(),
128        }
129        match self.peek() {
130            Some(SyntaxKind::DoubleQuotedValue | SyntaxKind::SingleQuotedValue) => self.bump(),
131            _ => panic!(),
132        }
133        self.builder.finish_node();
134
135        true
136    }
137
138    fn expect(&mut self, token: SyntaxKind) {
139        // TODO don't panic, instead add an error token and continue?
140        assert!(
141            self.allow(token),
142            "expected {:?}, got {:?}",
143            token,
144            self.peek()
145        );
146    }
147
148    fn allow_no_ws(&mut self, token: SyntaxKind) -> bool {
149        if Some(token) == self.peek_no_ws() {
150            self.bump();
151            return true;
152        }
153
154        false
155    }
156
157    fn allow(&mut self, token: SyntaxKind) -> bool {
158        if Some(token) == self.peek() {
159            self.bump();
160            return true;
161        }
162
163        false
164    }
165
166    fn parse_identifier_expression(&mut self) -> bool {
167        self.eat_ws();
168
169        let checkpoint = self.builder.checkpoint();
170        if self.allow(SyntaxKind::Identifier) {
171            self.builder
172                .start_node_at(checkpoint, SyntaxKind::IdentifierExpressionNode.into());
173            self.allow_no_ws(SyntaxKind::Varflag);
174            self.builder.finish_node();
175            self.eat_ws();
176            return true;
177        }
178
179        false
180    }
181
182    fn python_keyword(&mut self) -> bool {
183        let checkpoint = self.builder.checkpoint();
184        self.expect(SyntaxKind::Python);
185        // TODO: can anonymous python funcs be fakeroot?
186        self.allow(SyntaxKind::Fakeroot);
187        self.parse_assignment_or_task(Some(checkpoint), true);
188        true
189    }
190
191    fn fakeroot_keyword(&mut self) -> bool {
192        let checkpoint = self.builder.checkpoint();
193        self.expect(SyntaxKind::Fakeroot);
194        // TODO: can anonymous python funcs be fakeroot?
195        let maybe_anonymous_python = self.allow(SyntaxKind::Python);
196        self.parse_assignment_or_task(Some(checkpoint), maybe_anonymous_python);
197        true
198    }
199
200    fn parse_unset_node(&mut self) -> bool {
201        self.builder.start_node(SyntaxKind::UnsetNode.into());
202        self.expect(SyntaxKind::Unset);
203        self.parse_identifier_expression();
204        self.builder.finish_node();
205        true
206    }
207
208    fn parse_export_functions_node(&mut self) -> bool {
209        self.builder
210            .start_node(SyntaxKind::ExportFunctionsNode.into());
211        self.expect(SyntaxKind::ExportFunctions);
212        while self.peek() == Some(SyntaxKind::DirectiveArgument) {
213            self.bump();
214        }
215        self.builder.finish_node();
216        true
217    }
218
219    fn parse_inherit_node(&mut self) -> bool {
220        self.builder.start_node(SyntaxKind::InheritNode.into());
221        self.expect(SyntaxKind::Inherit);
222        self.expect(SyntaxKind::UnquotedValue);
223        self.builder.finish_node();
224        true
225    }
226
227    fn parse_python_def_function_node(&mut self) -> bool {
228        self.builder.start_node(SyntaxKind::PythonDefNode.into());
229        self.expect(SyntaxKind::PythonDefKeyword);
230        self.expect(SyntaxKind::PythonDefFunctionName);
231        self.expect(SyntaxKind::PythonDefFunctionArgs);
232        self.expect(SyntaxKind::Colon);
233        self.expect(SyntaxKind::PythonDefFunctionBody);
234        self.builder.finish_node();
235        true
236    }
237
238    fn parse_add_task_node(&mut self) -> bool {
239        self.builder.start_node(SyntaxKind::AddTaskNode.into());
240        self.expect(SyntaxKind::AddTask);
241        loop {
242            match self.peek() {
243                None => break,
244                Some(SyntaxKind::DirectiveArgument | SyntaxKind::After | SyntaxKind::Before) => {
245                    self.bump()
246                }
247                Some(SyntaxKind::EscapedNewline) => self.bump(),
248                Some(SyntaxKind::Newline) => break,
249                Some(other) => panic!("unexpected token: {other:?}"),
250            }
251        }
252        self.builder.finish_node();
253        true
254    }
255
256    fn parse_del_task_node(&mut self) -> bool {
257        self.builder.start_node(SyntaxKind::DelTaskNode.into());
258        self.expect(SyntaxKind::DelTask);
259        while self.peek() == Some(SyntaxKind::DirectiveArgument) {
260            self.bump();
261        }
262        self.builder.finish_node();
263        true
264    }
265
266    fn parse_add_handler_node(&mut self) -> bool {
267        self.builder.start_node(SyntaxKind::AddHandlerNode.into());
268        self.expect(SyntaxKind::AddHandler);
269        while self.peek() == Some(SyntaxKind::DirectiveArgument) {
270            self.bump();
271        }
272        self.builder.finish_node();
273        true
274    }
275
276    fn parse_next(&mut self) -> bool {
277        match self.peek() {
278            Some(token) => match token {
279                SyntaxKind::Newline | SyntaxKind::Comment => {
280                    self.bump();
281                    true
282                }
283                SyntaxKind::Export => self.parse_export(),
284                SyntaxKind::Include | SyntaxKind::Require => self.parse_include_or_require(),
285                SyntaxKind::Identifier => self.parse_assignment_or_task(None, false),
286                SyntaxKind::Python => self.python_keyword(),
287                SyntaxKind::Fakeroot => self.fakeroot_keyword(),
288                SyntaxKind::Unset => self.parse_unset_node(),
289                SyntaxKind::ExportFunctions => self.parse_export_functions_node(),
290                SyntaxKind::Inherit => self.parse_inherit_node(),
291                SyntaxKind::PythonDefKeyword => self.parse_python_def_function_node(),
292                SyntaxKind::AddTask => self.parse_add_task_node(),
293                SyntaxKind::DelTask => self.parse_del_task_node(),
294                SyntaxKind::AddHandler => self.parse_add_handler_node(),
295
296                _ => panic!("{token:?}"),
297            },
298
299            None => return false,
300        };
301
302        true
303    }
304
305    fn parse(mut self) -> SyntaxNode {
306        self.builder.start_node(SyntaxKind::RootNode.into());
307        while self.parse_next() {}
308        self.builder.finish_node();
309
310        SyntaxNode::new_root(self.builder.finish())
311    }
312}
313
314fn print(indent: usize, element: SyntaxElement) {
315    let kind: SyntaxKind = element.kind();
316    print!("{:indent$}", "", indent = indent);
317    match element {
318        NodeOrToken::Node(node) => {
319            println!("- {kind:?}");
320            for child in node.children_with_tokens() {
321                print(indent + 2, child);
322            }
323        }
324
325        NodeOrToken::Token(token) => println!("- {:?} {:?}", token.text(), kind),
326    }
327}
328
329// TODO error messages
330pub fn parse_bitbake_from_str(input: &str) -> Root {
331    let mut start = 0;
332    let tokens = tokenize(input).into_iter().map(|token| {
333        let text = &input[start..start + token.len];
334        start += token.len;
335        (syntax_kind_for_token_kind(token.kind), text)
336    });
337
338    let ast = Parser {
339        builder: GreenNodeBuilder::new(),
340        iter: tokens.peekable(),
341    }
342    .parse();
343
344    Root::cast(ast).unwrap()
345}
346
347#[cfg(test)]
348mod test {
349
350    use crate::parser::parser::parse_bitbake_from_str;
351
352    use crate::syntax::ast::AstNode;
353
354    #[test]
355    fn wat() {
356        let input = r##"
357RECIPE_MAINTAINER_pn-apt = "Aníbal Limón <limon.anibal@gmail.com>"
358RECIPE_MAINTAINER_pn-apt-native = "Aníbal Limón <limon.anibal@gmail.com>"
359
360export WAT_DO_YOU_SAY = "contents ${"buddy"}"
361include OK buddy then
362A = "OK then"
363python fakeroot do_my_task() {
364    friend :)
365}
366fakeroot do_another_thing_pal(){
367    echo "A"
368}
369do_p() {
370    echo true
371}
372unset TEST
373unset TEST[wat]
374
375EXPORT_FUNCTIONS this is buddy
376
377def this_is_(test, d):
378    print("OK")
379
380THIS = "a"
381
382addtask do_compile after do_friend before \
383    do_buddy
384
385python() {
386    d.setVar("OK")
387}
388
389deltask do_configure do_fetch
390
391        "##;
392
393        let r = parse_bitbake_from_str(input);
394        // for t in r.tasks() {
395        //     let p = t.name();
396        //     println!("{:?}", p.map(|c| c.syntax.text().to_string()));
397        //     println!("is_python: {}", t.is_python());
398        //     println!("is_fakeroot: {}", t.is_fakeroot());
399        //     println!("is_anonymous_python: {}", t.is_anonymous_python());
400        // }
401
402        println!();
403        for i in r.items() {
404            println!("{:?}", i.syntax().text());
405        }
406    }
407}
bytebraise_syntax/parser/parser.rs

bytebraise_syntax/parser/
parser.rs
