Skip to main content

tree_sitter_toolang/
lib.rs

1use tree_sitter_language::LanguageFn;
2
3extern "C" {
4    fn tree_sitter_toolang() -> *const ();
5}
6
7pub const LANGUAGE: LanguageFn = unsafe { LanguageFn::from_raw(tree_sitter_toolang) };
8
9#[cfg(test)]
10mod tests {
11    fn native_errors(node: tree_sitter::Node<'_>) -> Vec<tree_sitter::Node<'_>> {
12        let mut pending = vec![node];
13        let mut errors = Vec::new();
14        while let Some(current) = pending.pop() {
15            if current.is_error() || current.is_missing() {
16                errors.push(current);
17            }
18            let mut cursor = current.walk();
19            pending.extend(current.children(&mut cursor));
20        }
21        errors
22    }
23
24    fn is_valid(node: tree_sitter::Node<'_>) -> bool {
25        if node.has_error() || node.is_missing() || node.kind().starts_with("invalid_") {
26            return false;
27        }
28        let mut cursor = node.walk();
29        let valid = node.children(&mut cursor).all(is_valid);
30        valid
31    }
32
33    #[test]
34    fn text_leaves_and_reducer_ranges_share_the_public_contract() {
35        let mut parser = tree_sitter::Parser::new();
36        parser.set_language(&super::LANGUAGE.into()).unwrap();
37        let source = "#@ Program.\n## @param topic Input # details.\nflow work(topic: Text):\n  models = fast\n  reduce:\n\n    # Heading\n    Merge.\n    initial: Seed.\n  run next\n";
38        let tree = parser.parse(source, None).unwrap();
39        let root = tree.root_node();
40        assert!(!root.has_error());
41        for (index, field) in [(0, "text"), (1, "description")] {
42            let text = root
43                .named_child(index)
44                .unwrap()
45                .child_by_field_name(field)
46                .unwrap();
47            assert_eq!(text.kind(), "text");
48            assert_eq!(text.child_count(), 0);
49        }
50        let body = root
51            .named_child(2)
52            .unwrap()
53            .child_by_field_name("body")
54            .unwrap();
55        let value = body
56            .child_by_field_name("directive")
57            .unwrap()
58            .child_by_field_name("value")
59            .unwrap();
60        assert_eq!(value.kind(), "query_value");
61        let reduce = body.child_by_field_name("stmt").unwrap();
62        let content = reduce
63            .child_by_field_name("target")
64            .unwrap()
65            .child_by_field_name("content")
66            .unwrap();
67        assert_eq!(
68            content.utf8_text(source.as_bytes()).unwrap(),
69            "\n    # Heading\n    Merge.\n"
70        );
71        assert_eq!(content.named_child(0).unwrap().kind(), "blank_line");
72        let text = content.named_child(1).unwrap();
73        assert_eq!(text.kind(), "text");
74        assert_eq!(text.child_count(), 0);
75        assert_eq!(
76            reduce.child_by_field_name("initial").unwrap().kind(),
77            "content"
78        );
79    }
80
81    #[test]
82    fn initial_fields_preserve_nested_literal_text() {
83        let mut parser = tree_sitter::Parser::new();
84        parser.set_language(&super::LANGUAGE.into()).unwrap();
85        let source = "flow work:\n  reduce:\n    Merge.\n      initial: Nested literal.\n    initial: \"Seed # literal\"\n  run next\n";
86        let tree = parser.parse(source, None).unwrap();
87        let root = tree.root_node();
88        assert!(is_valid(root));
89        let body = root
90            .named_child(0)
91            .unwrap()
92            .child_by_field_name("body")
93            .unwrap();
94        let reduce = body.child_by_field_name("stmt").unwrap();
95        assert_eq!(reduce.kind(), "reduce_stmt");
96        let target = reduce.child_by_field_name("target").unwrap();
97        assert_eq!(
98            target
99                .child_by_field_name("content")
100                .unwrap()
101                .utf8_text(source.as_bytes())
102                .unwrap(),
103            "    Merge.\n      initial: Nested literal.\n"
104        );
105        assert_eq!(
106            reduce
107                .child_by_field_name("initial")
108                .unwrap()
109                .utf8_text(source.as_bytes())
110                .unwrap(),
111            " \"Seed # literal\""
112        );
113        assert!(target.end_byte() < reduce.child_by_field_name("initial").unwrap().start_byte());
114    }
115
116    #[test]
117    fn malformed_block_colons_cannot_reopen_enclosing_headers() {
118        let mut parser = tree_sitter::Parser::new();
119        parser.set_language(&super::LANGUAGE.into()).unwrap();
120        for header in ["generate :", "keep in 2 lanes:", "sort ascending:"] {
121            for binding in ["", "let value = ", "let "] {
122                for nested in [false, true] {
123                    let indent = if nested { "    " } else { "  " };
124                    let prefix = if nested {
125                        "  repeat:\n    run inner\n"
126                    } else {
127                        ""
128                    };
129                    let source = format!("flow first:\n  run before\n{prefix}{indent}{binding}{header}\n{indent}  Orphaned text.\n{indent}## Following.\n{indent}run after\n## Next.\nflow next:\n  pass\n");
130                    let tree = parser.parse(&source, None).unwrap();
131                    let root = tree.root_node();
132                    assert!(root.has_error());
133                    let flow = root.named_child(0).unwrap();
134                    assert_eq!(
135                        flow.child_by_field_name("colon").unwrap().start_position(),
136                        tree_sitter::Point::new(0, 10)
137                    );
138                    let body = flow.child_by_field_name("body").unwrap();
139                    assert!(!body.named_child(0).unwrap().has_error());
140                    let body = if nested {
141                        body.named_child(1)
142                            .unwrap()
143                            .child_by_field_name("body")
144                            .unwrap()
145                    } else {
146                        body
147                    };
148                    let mut cursor = body.walk();
149                    let after = body
150                        .named_children(&mut cursor)
151                        .find(|node| {
152                            node.kind() == "run_stmt"
153                                && node
154                                    .child_by_field_name("target")
155                                    .unwrap()
156                                    .utf8_text(source.as_bytes())
157                                    .unwrap()
158                                    == "after"
159                        })
160                        .unwrap();
161                    assert!(!after.has_error());
162                    let mut cursor = body.walk();
163                    assert!(body
164                        .named_children(&mut cursor)
165                        .any(|node| node.kind() == "item_doc"));
166                    assert_eq!(root.named_child(1).unwrap().kind(), "item_doc");
167                    assert!(!root.named_child(2).unwrap().has_error());
168                }
169            }
170        }
171    }
172
173    #[test]
174    fn rejected_first_directives_keep_the_authored_body_baseline() {
175        let mut parser = tree_sitter::Parser::new();
176        parser.set_language(&super::LANGUAGE.into()).unwrap();
177        for (kind, following, entry) in [
178            ("flow", "run after", "run_stmt"),
179            ("agic", "user: Following.", "message"),
180        ] {
181            let source = format!("{kind} first:\n  modles = fast\n    Orphaned content.\n  ## Following.\n  {following}\n## Next.\nflow next:\n  pass\n");
182            let tree = parser.parse(&source, None).unwrap();
183            let root = tree.root_node();
184            assert!(root.has_error());
185            let owner = root.named_child(0).unwrap();
186            assert_eq!(owner.kind(), kind);
187            let body = owner.child_by_field_name("body").unwrap();
188            assert_eq!(body.start_position(), tree_sitter::Point::new(1, 0));
189            let mut cursor = body.walk();
190            let after = body
191                .named_children(&mut cursor)
192                .find(|node| {
193                    node.kind() == entry
194                        && node.utf8_text(source.as_bytes()).unwrap().trim() == following
195                })
196                .unwrap();
197            assert!(!after.has_error());
198            let mut cursor = body.walk();
199            let doc = body
200                .named_children(&mut cursor)
201                .find(|node| node.kind() == "item_doc")
202                .unwrap();
203            assert_eq!(doc.utf8_text(source.as_bytes()).unwrap(), "## Following.\n");
204            assert_eq!(root.named_child(1).unwrap().kind(), "item_doc");
205            let next = root.named_child(2).unwrap();
206            assert_eq!(next.kind(), "flow");
207            assert!(!next.has_error());
208        }
209    }
210
211    #[test]
212    fn rejected_outer_prefix_docs_preserve_the_next_member_documentation() {
213        let mut parser = tree_sitter::Parser::new();
214        parser.set_language(&super::LANGUAGE.into()).unwrap();
215        for (header, member, body_kind) in [
216            ("flow first:", "run one", "flow_body"),
217            ("agic first:", "user: One.", "agic_body"),
218            ("struct First:", "one: Text", "struct_body"),
219            ("prompt first:", "one = Value", "cap_body"),
220            ("chore first:", "one = Value", "chore_body"),
221        ] {
222            let source = format!("{header}\n#@ Wrong scope.\n# Outer.\n\n  ## First.\n  {member}\n## Next.\nflow next:\n  pass\n");
223            let tree = parser.parse(&source, None).unwrap();
224            let root = tree.root_node();
225            assert!(root.has_error());
226            let body = root
227                .named_child(0)
228                .unwrap()
229                .child_by_field_name("body")
230                .unwrap();
231            assert_eq!(body.kind(), body_kind);
232            assert_eq!(body.start_byte(), source.find("  ## First.").unwrap());
233            let doc = body.named_child(0).unwrap();
234            assert_eq!(doc.kind(), "item_doc");
235            assert!(!doc.has_error());
236            assert_eq!(doc.utf8_text(source.as_bytes()).unwrap(), "## First.\n");
237            assert_eq!(root.named_child(1).unwrap().kind(), "item_doc");
238            let next = root.named_child(2).unwrap();
239            assert!(!next.has_error());
240            assert_eq!(next.kind(), "flow");
241        }
242    }
243
244    #[test]
245    fn leading_member_docs_use_the_body_baseline_and_preserve_empty_boundaries() {
246        let mut parser = tree_sitter::Parser::new();
247        parser.set_language(&super::LANGUAGE.into()).unwrap();
248        let source = "flow first:\n  ## Before outer.\n# Outer.\n\n  ## First.\n    ## Deeper.\n  run one\n  ## Second.\n  run two\nprompt empty:\n## Next.\nflow next:\n  pass\n";
249        let tree = parser.parse(source, None).unwrap();
250        let root = tree.root_node();
251        assert!(!root.has_error());
252        let flow = root.named_child(0).unwrap();
253        let body = flow.child_by_field_name("body").unwrap();
254        assert_eq!(body.start_byte(), source.find("  ## First.").unwrap());
255        assert_eq!(body.end_byte(), source.find("prompt empty:").unwrap());
256        let mut cursor = flow.walk();
257        let outer: Vec<_> = flow
258            .named_children(&mut cursor)
259            .filter(|node| node.kind() == "item_doc")
260            .map(|node| node.utf8_text(source.as_bytes()).unwrap())
261            .collect();
262        assert_eq!(outer, ["## Before outer.\n"]);
263        let mut cursor = body.walk();
264        let docs: Vec<_> = body
265            .named_children(&mut cursor)
266            .filter(|node| node.kind() == "item_doc")
267            .map(|node| node.utf8_text(source.as_bytes()).unwrap())
268            .collect();
269        assert_eq!(docs, ["## First.\n", "## Deeper.\n", "## Second.\n"]);
270        let empty = root.named_child(1).unwrap();
271        assert!(empty.child_by_field_name("body").is_none());
272        assert_eq!(empty.end_byte(), source.find("## Next.").unwrap());
273        assert_eq!(root.named_child(2).unwrap().kind(), "item_doc");
274        assert_eq!(root.named_child(3).unwrap().kind(), "flow");
275    }
276
277    #[test]
278    fn rejected_comments_pending_statements_and_until_errors_keep_their_owners() {
279        let mut parser = tree_sitter::Parser::new();
280        parser.set_language(&super::LANGUAGE.into()).unwrap();
281        let source = "flow bad_comment:\n  run before\n  #@ Wrong scope.\n  run after\nflow missing:\n  models = fast\n## Next.\nflow bad_until:\n  repeat:\n    run worker\n    until 42\n  run after_repeat\nflow next:\n  pass\n";
282        let tree = parser.parse(source, None).unwrap();
283        let root = tree.root_node();
284        assert!(root.has_error());
285        let mut cursor = root.walk();
286        let items: Vec<_> = root.named_children(&mut cursor).collect();
287        assert_eq!(
288            items.iter().map(|node| node.kind()).collect::<Vec<_>>(),
289            ["flow", "flow", "item_doc", "flow", "flow"]
290        );
291        for (index, name) in [
292            (0, "bad_comment"),
293            (1, "missing"),
294            (3, "bad_until"),
295            (4, "next"),
296        ] {
297            assert_eq!(
298                items[index]
299                    .child_by_field_name("name")
300                    .unwrap()
301                    .utf8_text(source.as_bytes())
302                    .unwrap(),
303                name
304            );
305        }
306        assert!(items[1].end_byte() <= items[2].start_byte());
307        assert_eq!(items[2].utf8_text(source.as_bytes()).unwrap(), "## Next.\n");
308        assert!(!items[4].has_error());
309        for (index, expected) in [(0, vec!["before", "after"]), (3, vec!["after_repeat"])] {
310            let body = items[index].child_by_field_name("body").unwrap();
311            let mut cursor = body.walk();
312            let targets: Vec<_> = body
313                .named_children(&mut cursor)
314                .filter(|node| node.kind() == "run_stmt")
315                .map(|node| {
316                    assert!(!node.has_error());
317                    node.child_by_field_name("target")
318                        .unwrap()
319                        .utf8_text(source.as_bytes())
320                        .unwrap()
321                })
322                .collect();
323            assert_eq!(targets, expected);
324        }
325        let repeat = items[3]
326            .child_by_field_name("body")
327            .unwrap()
328            .child_by_field_name("stmt")
329            .unwrap();
330        let body = repeat.child_by_field_name("body").unwrap();
331        let worker = body.child_by_field_name("stmt").unwrap();
332        assert_eq!(worker.kind(), "run_stmt");
333        assert_eq!(
334            worker.utf8_text(source.as_bytes()).unwrap().trim(),
335            "run worker"
336        );
337        assert_eq!(body.named_child(1).unwrap().kind(), "invalid_flow_stmt");
338    }
339
340    #[test]
341    fn native_recovery_synchronizes_at_statements_and_program_items() {
342        let mut parser = tree_sitter::Parser::new();
343        parser.set_language(&super::LANGUAGE.into()).unwrap();
344        for prefix in [
345            "flow broken:\n  map in 2 lanes in 3 lanes using worker\n  ## Following.\n  run after\n",
346            "flow broken:\n  hands += worker\n  run after\n",
347            "agic broken:\n  recall = wrong\n  user: Following.\n",
348            "struct Broken:\n  value Text\n",
349            "flow broken(\n",
350        ] {
351            let source = format!("{prefix}flow next:\n  pass\n");
352            let tree = parser.parse(&source, None).unwrap();
353            let root = tree.root_node();
354            assert!(!is_valid(root));
355            let next = root.named_child(root.named_child_count() - 1).unwrap();
356            assert_eq!(next.kind(), "flow");
357            assert!(!next.has_error());
358            assert_eq!(next.start_byte(), prefix.len());
359            assert_eq!(
360                next.child_by_field_name("name")
361                    .unwrap()
362                    .utf8_text(source.as_bytes())
363                    .unwrap(),
364                "next"
365            );
366            if prefix.starts_with("flow broken:") {
367                let body = root
368                    .named_child(0)
369                    .unwrap()
370                    .child_by_field_name("body")
371                    .unwrap();
372                assert_eq!(
373                    body.child_by_field_name("stmt")
374                        .unwrap()
375                        .child_by_field_name("target")
376                        .unwrap()
377                        .utf8_text(source.as_bytes())
378                        .unwrap(),
379                    "after"
380                );
381            }
382        }
383    }
384
385    #[test]
386    fn public_cst_fields_and_body_ranges_keep_their_roles() {
387        let mut parser = tree_sitter::Parser::new();
388        parser.set_language(&super::LANGUAGE.into()).unwrap();
389        let source = "prompt empty:\n## Next.\nflow work:\n# Preface.\n  instruct = named\n  run worker\n  repeat:\n    run worker\n    until: Ready.\n";
390        let tree = parser.parse(source, None).unwrap();
391        let root = tree.root_node();
392        assert!(!root.has_error());
393        assert!(root
394            .named_child(0)
395            .unwrap()
396            .child_by_field_name("body")
397            .is_none());
398        assert_eq!(root.named_child(1).unwrap().kind(), "item_doc");
399        let flow = root.named_child(2).unwrap();
400        let body = flow.child_by_field_name("body").unwrap();
401        assert_eq!(body.start_byte(), source.find("  instruct").unwrap());
402        let directive = body.child_by_field_name("directive").unwrap();
403        for (field, kind) in [
404            ("key", "directive_key"),
405            ("operator", "="),
406            ("value", "identifier"),
407        ] {
408            assert_eq!(directive.child_by_field_name(field).unwrap().kind(), kind);
409        }
410        let run = body.child_by_field_name("stmt").unwrap();
411        assert_eq!(
412            run.child_by_field_name("target").unwrap().kind(),
413            "runnable_name"
414        );
415        let condition = body
416            .named_child(body.named_child_count() - 1)
417            .unwrap()
418            .child_by_field_name("body")
419            .unwrap()
420            .child_by_field_name("until")
421            .unwrap();
422        let target = condition.child_by_field_name("target").unwrap();
423        assert_eq!(target.kind(), "inline_agic");
424        assert!(target.child_by_field_name("return").is_none());
425        assert_eq!(
426            target
427                .child_by_field_name("content")
428                .unwrap()
429                .utf8_text(source.as_bytes())
430                .unwrap(),
431            " Ready."
432        );
433        assert_eq!(
434            run.named_child(run.named_child_count() - 1).unwrap().kind(),
435            "newline"
436        );
437    }
438
439    #[test]
440    fn empty_body_and_content_recovery_preserve_peer_documentation() {
441        let mut parser = tree_sitter::Parser::new();
442        parser.set_language(&super::LANGUAGE.into()).unwrap();
443        for header in [
444            "prompt empty:",
445            "chore empty:",
446            "flow empty:",
447            "context empty:",
448            "flow empty:\n  run:",
449        ] {
450            for ending in ["\n", "\r\n"] {
451                let source =
452                    format!("{header}\n## Next.\nflow next:\n  pass\n").replace('\n', ending);
453                let tree = parser.parse(&source, None).unwrap();
454                let root = tree.root_node();
455                assert!(!root.has_error());
456                assert_eq!(root.named_child_count(), 3);
457                let doc = root.named_child(1).unwrap();
458                assert_eq!(doc.kind(), "item_doc");
459                assert_eq!(doc.start_byte(), source.find("## Next.").unwrap());
460                assert!(root.named_child(0).unwrap().end_byte() <= doc.start_byte());
461                assert_eq!(
462                    root.named_child(2)
463                        .unwrap()
464                        .child_by_field_name("name")
465                        .unwrap()
466                        .utf8_text(source.as_bytes())
467                        .unwrap(),
468                    "next"
469                );
470            }
471        }
472    }
473
474    #[test]
475    fn unit_spellings_preserve_integer_fields_and_require_prefix_lanes() {
476        let mut parser = tree_sitter::Parser::new();
477        parser.set_language(&super::LANGUAGE.into()).unwrap();
478        for number in [
479            "0",
480            "1",
481            "01",
482            "2",
483            "0002",
484            "9999999999999999999999999999999999999999",
485        ] {
486            for unit in ["lane", "lanes"] {
487                for (head, connector) in [
488                    ("generate 3", "using"),
489                    ("map", "using"),
490                    ("keep", "if"),
491                    ("drop", "if"),
492                    ("sort ascending", "by"),
493                ] {
494                    let source =
495                        format!("flow work:\n  {head} in {number} {unit} {connector} worker\n");
496                    let tree = parser.parse(&source, None).unwrap();
497                    assert!(!tree.root_node().has_error());
498                    let statement = tree
499                        .root_node()
500                        .named_child(0)
501                        .unwrap()
502                        .child_by_field_name("body")
503                        .unwrap()
504                        .child_by_field_name("stmt")
505                        .unwrap();
506                    let lanes = statement.child_by_field_name("lanes").unwrap();
507                    assert_eq!(lanes.kind(), "integer_literal");
508                    assert_eq!(lanes.utf8_text(source.as_bytes()).unwrap(), number);
509                    assert_eq!(lanes.child_count(), 0);
510                }
511            }
512            for unit in ["time", "times"] {
513                let source = format!("flow work:\n  repeat {number} {unit}:\n    run worker\n");
514                let tree = parser.parse(&source, None).unwrap();
515                assert!(!tree.root_node().has_error());
516                let statement = tree
517                    .root_node()
518                    .named_child(0)
519                    .unwrap()
520                    .child_by_field_name("body")
521                    .unwrap()
522                    .child_by_field_name("stmt")
523                    .unwrap();
524                assert_eq!(
525                    statement
526                        .child_by_field_name("count")
527                        .unwrap()
528                        .utf8_text(source.as_bytes())
529                        .unwrap(),
530                    number
531                );
532            }
533        }
534        for statement in [
535            "keep if worker in 2 lanes",
536            "drop if worker in 1 lane",
537            "sort ascending by worker in 2 lanes",
538        ] {
539            let source = format!("flow work:\n  {statement}\n  run after\nflow next:\n  pass\n");
540            let tree = parser.parse(source, None).unwrap();
541            assert!(tree.root_node().has_error());
542            assert_eq!(tree.root_node().named_child(1).unwrap().kind(), "flow");
543            assert!(!tree.root_node().named_child(1).unwrap().has_error());
544        }
545    }
546
547    #[test]
548    fn unknown_directives_fail_only_in_the_prefix() {
549        let mut parser = tree_sitter::Parser::new();
550        parser.set_language(&super::LANGUAGE.into()).unwrap();
551        for kind in ["agic", "flow"] {
552            let body = if kind == "agic" {
553                "user: Continue."
554            } else {
555                "run after"
556            };
557            let invalid = format!("{kind} work:\n  modles = fast\n  {body}\nflow next:\n  pass\n");
558            let tree = parser.parse(&invalid, None).unwrap();
559            let root = tree.root_node();
560            assert!(root.has_error());
561            let next = root.named_child(root.named_child_count() - 1).unwrap();
562            assert_eq!(next.kind(), "flow");
563            assert!(!next.has_error());
564            assert_eq!(
565                next.child_by_field_name("name")
566                    .unwrap()
567                    .utf8_text(invalid.as_bytes())
568                    .unwrap(),
569                "next"
570            );
571            let valid = format!("{kind} work:\n  {body}\n  modles = literal\n");
572            assert!(!parser.parse(valid, None).unwrap().root_node().has_error());
573        }
574        for kind in ["psyche", "skill", "service", "prompt", "chore"] {
575            let source =
576                format!("{kind} example:\n  unknown = value\n  Body.\n  unknown = literal\n");
577            let tree = parser.parse(source, None).unwrap();
578            assert!(!tree.root_node().has_error());
579            let body = tree
580                .root_node()
581                .named_child(0)
582                .unwrap()
583                .child_by_field_name("body")
584                .unwrap();
585            let property = body.child_by_field_name("property").unwrap();
586            assert_eq!(property.kind(), "property");
587            assert!(body.child_by_field_name("content").is_some());
588        }
589    }
590
591    #[test]
592    fn documentation_fields_and_queries_are_available() {
593        let language = super::LANGUAGE.into();
594        let mut parser = tree_sitter::Parser::new();
595        parser.set_language(&language).unwrap();
596        let source = "#!/usr/bin/env too\n#@ Program.\n##! Important.\n## @param _ Input.\n";
597        let tree = parser.parse(source, None).unwrap();
598        let root = tree.root_node();
599        assert_eq!(root.kind(), "program");
600        assert!(!root.has_error());
601        assert_eq!(root.named_child(0).unwrap().kind(), "shebang_comment");
602        for (index, kind, text) in [
603            (1, "program_doc", "Program."),
604            (2, "item_doc", "! Important."),
605        ] {
606            let comment = root.named_child(index).unwrap();
607            assert_eq!(comment.kind(), kind);
608            assert_eq!(comment.child_by_field_name("text").unwrap().kind(), "text");
609            assert_eq!(
610                comment
611                    .child_by_field_name("text")
612                    .unwrap()
613                    .utf8_text(source.as_bytes())
614                    .unwrap(),
615                text
616            );
617        }
618        let comment = root.named_child(3).unwrap();
619        assert_eq!(comment.kind(), "param_doc");
620        assert_eq!(comment.start_byte(), source.find("## @param").unwrap());
621        assert_eq!(comment.end_byte(), source.len());
622        assert!(comment.child_by_field_name("parameter").is_none());
623        for (field, value) in [("param", "_"), ("description", "Input.")] {
624            assert_eq!(
625                comment
626                    .child_by_field_name(field)
627                    .unwrap()
628                    .utf8_text(source.as_bytes())
629                    .unwrap(),
630                value
631            );
632        }
633        for query in [
634            include_str!("../../queries/highlights.scm"),
635            include_str!("../../queries/injections.scm"),
636            include_str!("../../queries/indents.scm"),
637            include_str!("../../queries/outline.scm"),
638            include_str!("../../queries/tags.scm"),
639        ] {
640            tree_sitter::Query::new(&language, query).unwrap();
641        }
642    }
643
644    #[test]
645    fn spawn_targets_and_bindings_are_available() {
646        let language = super::LANGUAGE.into();
647        let mut parser = tree_sitter::Parser::new();
648        parser.set_language(&language).unwrap();
649        let source = "flow launch:\n  spawn worker\n  let job = spawn -> Text: Research.\n";
650        let tree = parser.parse(source, None).unwrap();
651        assert!(!tree.root_node().has_error());
652        let statements = tree
653            .root_node()
654            .named_child(0)
655            .unwrap()
656            .child_by_field_name("body")
657            .unwrap();
658        let bare = statements.named_child(0).unwrap();
659        assert_eq!(bare.kind(), "spawn_stmt");
660        assert_eq!(
661            bare.child_by_field_name("target").unwrap().kind(),
662            "runnable_name"
663        );
664        let bound = statements.named_child(1).unwrap();
665        assert_eq!(bound.kind(), "let_stmt");
666        assert_eq!(
667            bound
668                .child_by_field_name("local")
669                .unwrap()
670                .utf8_text(source.as_bytes())
671                .unwrap()
672                .trim(),
673            "job"
674        );
675        let spawn = bound.child_by_field_name("stmt").unwrap();
676        assert_eq!(spawn.kind(), "spawn_stmt");
677        assert_eq!(
678            spawn.child_by_field_name("target").unwrap().kind(),
679            "inline_agic"
680        );
681    }
682
683    #[test]
684    fn repeat_body_exposes_condition_in_source_order() {
685        let mut parser = tree_sitter::Parser::new();
686        parser.set_language(&super::LANGUAGE.into()).unwrap();
687        let source = "flow work:\n  repeat:\n    run first\n    until done\n    run last\n";
688        let tree = parser.parse(source, None).unwrap();
689        assert!(!tree.root_node().has_error());
690        let statement = tree
691            .root_node()
692            .named_child(0)
693            .unwrap()
694            .child_by_field_name("body")
695            .unwrap()
696            .named_child(0)
697            .unwrap();
698        assert!(statement.child_by_field_name("until").is_none());
699        let body = statement.child_by_field_name("body").unwrap();
700        assert_eq!(body.kind(), "repeat_body");
701        let condition = body.child_by_field_name("until").unwrap();
702        assert_eq!(condition.kind(), "until_clause");
703        assert_eq!(body.named_child(1).unwrap(), condition);
704        let target = condition.child_by_field_name("target").unwrap();
705        assert_eq!(target.kind(), "runnable_name");
706        assert_eq!(target.utf8_text(source.as_bytes()).unwrap().trim(), "done");
707        let mut cursor = body.walk();
708        let statements: Vec<_> = body.children_by_field_name("stmt", &mut cursor).collect();
709        assert_eq!(statements.len(), 2);
710        assert!(statements[0].end_byte() <= condition.start_byte());
711        assert!(condition.end_byte() <= statements[1].start_byte());
712    }
713
714    #[test]
715    fn launch_handles_share_await_fields() {
716        let language = super::LANGUAGE.into();
717        let mut parser = tree_sitter::Parser::new();
718        parser.set_language(&language).unwrap();
719        for (operation, kind, target) in [
720            ("async run", "run_stmt", "target"),
721            ("spawn", "spawn_stmt", "target"),
722        ] {
723            let source = format!("flow launch:\n  let h = {operation} worker\n  await h\n  let x = await h\n  let await h\n  let h = await h\n");
724            let tree = parser.parse(&source, None).unwrap();
725            assert!(!tree.root_node().has_error());
726            let statements = tree
727                .root_node()
728                .named_child(0)
729                .unwrap()
730                .child_by_field_name("body")
731                .unwrap();
732            let launch = statements.named_child(0).unwrap();
733            let statement = launch.child_by_field_name("stmt").unwrap();
734            assert_eq!(statement.kind(), kind);
735            assert_eq!(
736                statement
737                    .child_by_field_name("async")
738                    .map(|node| node.kind()),
739                if operation == "async run" {
740                    Some("async")
741                } else {
742                    None
743                }
744            );
745            assert_eq!(
746                statement.child_by_field_name(target).unwrap().kind(),
747                "runnable_name"
748            );
749            for (index, name) in [(1, None), (2, Some("x")), (3, None), (4, Some("h"))] {
750                let node = statements.named_child(index).unwrap();
751                let await_node = if index == 1 {
752                    node
753                } else {
754                    assert_eq!(node.kind(), "let_stmt");
755                    assert!(node.child_by_field_name("value").is_none());
756                    assert_eq!(
757                        node.child_by_field_name("local")
758                            .map(|node| node.utf8_text(source.as_bytes()).unwrap().trim()),
759                        name
760                    );
761                    node.child_by_field_name("stmt").unwrap()
762                };
763                assert_eq!(await_node.kind(), "await_stmt");
764                let handle = await_node.child_by_field_name("handle").unwrap();
765                assert_eq!(handle.kind(), "handle_name");
766                assert_eq!(handle.child_count(), 0);
767                assert_eq!(handle.utf8_text(source.as_bytes()).unwrap().trim(), "h");
768            }
769        }
770    }
771
772    #[test]
773    fn anonymous_terminals_keep_fields_and_native_message_errors_stay_local() {
774        let mut parser = tree_sitter::Parser::new();
775        parser.set_language(&super::LANGUAGE.into()).unwrap();
776        let source = "flow work(input?: Text[]) -> Text:\n  models += fast\n  async run worker\n  keep last 2\nagic messages:\n  user missing colon\n  user: After.\nflow next:\n  pass\n";
777        let tree = parser.parse(source, None).unwrap();
778        let root = tree.root_node();
779        assert!(root.has_error());
780        let flow = root.named_child(0).unwrap();
781        let parameter = flow
782            .child_by_field_name("params")
783            .unwrap()
784            .child_by_field_name("param")
785            .unwrap();
786        let body = flow.child_by_field_name("body").unwrap();
787        let directive = body.child_by_field_name("directive").unwrap();
788        let run = body.child_by_field_name("stmt").unwrap();
789        let mut cursor = body.walk();
790        let keep = body
791            .named_children(&mut cursor)
792            .find(|node| node.kind() == "keep_stmt")
793            .unwrap();
794        let selection = keep.child_by_field_name("selection").unwrap();
795        assert_eq!(selection.kind(), "selection");
796        let ty = parameter.child_by_field_name("type").unwrap();
797        for (owner, field, literal) in [
798            (flow, "keyword", "flow"),
799            (flow, "arrow", "->"),
800            (parameter, "optional", "?"),
801            (ty, "suffix", "[]"),
802            (directive, "operator", "+="),
803            (run, "async", "async"),
804            (selection, "side", "last"),
805        ] {
806            let token = owner.child_by_field_name(field).unwrap();
807            assert_eq!(token.kind(), literal);
808            assert!(!token.is_named());
809            assert_eq!(&source[token.byte_range()], literal);
810        }
811        let body = root
812            .named_child(1)
813            .unwrap()
814            .child_by_field_name("body")
815            .unwrap();
816        let mut cursor = body.walk();
817        let entries: Vec<_> = body
818            .children_by_field_name("message", &mut cursor)
819            .collect();
820        assert_eq!(
821            entries.iter().map(|node| node.kind()).collect::<Vec<_>>(),
822            ["message"]
823        );
824        let diagnostic = native_errors(body)
825            .into_iter()
826            .find(|node| node.is_error())
827            .unwrap();
828        assert_eq!(diagnostic.parent().unwrap(), body);
829        assert_eq!(
830            diagnostic.utf8_text(source.as_bytes()).unwrap().trim(),
831            "user missing colon"
832        );
833        assert!(entries[0].child_by_field_name("content").is_some());
834        assert_eq!(
835            root.named_child(2)
836                .unwrap()
837                .child_by_field_name("name")
838                .unwrap()
839                .utf8_text(source.as_bytes())
840                .unwrap(),
841            "next"
842        );
843    }
844
845    #[test]
846    fn invalid_parameter_annotations_and_directives_keep_their_owners() {
847        let mut parser = tree_sitter::Parser::new();
848        parser.set_language(&super::LANGUAGE.into()).unwrap();
849        for kind in ["agic", "flow"] {
850            for parameter in [": Text", "?: Text[]", "topic:", "topic?:"] {
851                let source = format!("{kind} work({parameter}, next: Text) -> Text:\n  models = fast\n  models after\n  tools = fast\n## Next.\nflow next:\n  pass\n");
852                let tree = parser.parse(&source, None).unwrap();
853                let root = tree.root_node();
854                assert!(!is_valid(root));
855                let owner = root.named_child(0).unwrap();
856                assert_eq!(owner.kind(), kind);
857                let params = owner.child_by_field_name("params").unwrap();
858                let bad = params.child_by_field_name("param").unwrap();
859                if parameter.starts_with(':') || parameter.starts_with('?') {
860                    assert_eq!(bad.kind(), "invalid_param");
861                    assert_eq!(bad.utf8_text(source.as_bytes()).unwrap(), parameter);
862                } else {
863                    assert_eq!(bad.kind(), "param");
864                    assert!(params.has_error());
865                    assert!(bad.child_by_field_name("type").is_none());
866                }
867                let body = owner.child_by_field_name("body").unwrap();
868                let mut cursor = body.walk();
869                let entries: Vec<_> = body
870                    .children_by_field_name("directive", &mut cursor)
871                    .collect();
872                assert_eq!(
873                    entries.iter().map(|node| node.kind()).collect::<Vec<_>>(),
874                    ["directive", "directive"]
875                );
876                assert_eq!(
877                    entries[1]
878                        .child_by_field_name("key")
879                        .unwrap()
880                        .utf8_text(source.as_bytes())
881                        .unwrap(),
882                    "tools"
883                );
884                assert_eq!(entries[1].parent().unwrap(), body);
885                assert!(native_errors(body).into_iter().any(|node| node.is_error()
886                    && node.utf8_text(source.as_bytes()).unwrap().trim() == "models after"));
887                assert_eq!(root.named_child(1).unwrap().kind(), "item_doc");
888                let next = root.named_child(2).unwrap();
889                assert_eq!(next.kind(), "flow");
890                assert_eq!(
891                    next.child_by_field_name("name")
892                        .unwrap()
893                        .utf8_text(source.as_bytes())
894                        .unwrap(),
895                    "next"
896                );
897                assert!(!next.has_error());
898            }
899        }
900    }
901
902    #[test]
903    fn return_annotation_recovery_keeps_headers_and_inline_content() {
904        let mut parser = tree_sitter::Parser::new();
905        parser.set_language(&super::LANGUAGE.into()).unwrap();
906        for (header, missing_type) in [
907            ("flow work(topic: Text -> Text: # ) ,", false),
908            ("flow work(: Text?) -> : # Header.", true),
909        ] {
910            let source = format!("{header}\n  ## First.\n  reduce -> :\n\n    Merge.\n    initial: Initial.\n  run after\n## Next.\nflow next:\n  pass\n");
911            let tree = parser.parse(&source, None).unwrap();
912            let root = tree.root_node();
913            assert!(!is_valid(root));
914            let flow = root.named_child(0).unwrap();
915            assert_eq!(flow.kind(), "flow");
916            assert_eq!(flow.child_by_field_name("return").is_none(), missing_type);
917            let body = flow.child_by_field_name("body").unwrap();
918            assert_eq!(body.named_child(0).unwrap().kind(), "item_doc");
919            let reducer = body.child_by_field_name("stmt").unwrap();
920            let inline = reducer.child_by_field_name("target").unwrap();
921            assert!(root.has_error());
922            assert!(native_errors(reducer)
923                .into_iter()
924                .any(|node| node.is_error() && node.utf8_text(source.as_bytes()).unwrap() == "->"));
925            assert!(inline.child_by_field_name("return").is_none());
926            assert_eq!(
927                inline
928                    .child_by_field_name("content")
929                    .unwrap()
930                    .utf8_text(source.as_bytes())
931                    .unwrap(),
932                "\n    Merge.\n"
933            );
934            assert_eq!(
935                reducer
936                    .child_by_field_name("initial")
937                    .unwrap()
938                    .utf8_text(source.as_bytes())
939                    .unwrap(),
940                " Initial."
941            );
942            assert_eq!(root.named_child(1).unwrap().kind(), "item_doc");
943            assert!(is_valid(root.named_child(2).unwrap()));
944        }
945    }
946
947    #[test]
948    fn sync_and_async_target_errors_keep_the_recognized_name() {
949        let mut parser = tree_sitter::Parser::new();
950        parser.set_language(&super::LANGUAGE.into()).unwrap();
951        for modifier in ["", "async "] {
952            for binding in ["", "let ", "let result = "] {
953                let source = format!("flow first:\n  {binding}{modifier}run worker extra\n  run after\nflow next:\n  pass\n");
954                let tree = parser.parse(&source, None).unwrap();
955                let root = tree.root_node();
956                assert!(root.has_error());
957                let body = root
958                    .named_child(0)
959                    .unwrap()
960                    .child_by_field_name("body")
961                    .unwrap();
962                let statement = body.child_by_field_name("stmt").unwrap();
963                let run = if binding.is_empty() {
964                    statement
965                } else {
966                    statement.child_by_field_name("stmt").unwrap()
967                };
968                assert_eq!(run.kind(), "run_stmt");
969                assert_eq!(
970                    run.child_by_field_name("target")
971                        .unwrap()
972                        .utf8_text(source.as_bytes())
973                        .unwrap(),
974                    "worker"
975                );
976                let errors = native_errors(run);
977                assert_eq!(errors.len(), 1);
978                assert!(errors[0].is_error());
979                assert_eq!(errors[0].utf8_text(source.as_bytes()).unwrap(), "extra");
980                assert_eq!(errors[0].start_byte(), source.find("extra").unwrap());
981                assert!(is_valid(root.named_child(1).unwrap()));
982            }
983        }
984    }
985
986    #[test]
987    fn missing_directive_values_are_visible_public_tokens() {
988        let mut parser = tree_sitter::Parser::new();
989        parser.set_language(&super::LANGUAGE.into()).unwrap();
990        for (key, kind) in [
991            ("models", "query_value"),
992            ("context", "identifier"),
993            ("instruct", "identifier"),
994        ] {
995            let source = format!("flow first:\n  {key} =\n  run after\nflow next:\n  pass\n");
996            let tree = parser.parse(&source, None).unwrap();
997            let root = tree.root_node();
998            assert!(root.has_error());
999            let errors = native_errors(root);
1000            assert_eq!(errors.len(), 1);
1001            assert_eq!(errors[0].kind(), kind);
1002            assert!(errors[0].is_missing());
1003            assert_eq!(errors[0].start_byte(), errors[0].end_byte());
1004            assert!(is_valid(root.named_child(1).unwrap()));
1005        }
1006    }
1007
1008    #[test]
1009    fn incomplete_header_terminators_preserve_following_items() {
1010        let mut parser = tree_sitter::Parser::new();
1011        parser.set_language(&super::LANGUAGE.into()).unwrap();
1012        for prefix in [
1013            "flow first:\n  repeat 2 times windowing 1,\n    run worker\n    until ready\n",
1014            "flow first:\n  repeat 2 times windowing 1\n\n    run worker\n    until ready\n",
1015            "agic first(_: Text) -> Json\n  models = fast\n  recall = near\n  tool: Body.\n",
1016            "agic first(_: Text) -> J[on:\n  models = fast\n  recall = near\n  tool: Body.\n",
1017            "flow first(topic?, Text[]\n other: Json) -> Text[]:\n  ## First.\n  run worker\n",
1018            "agic first(topic?: Text[], other\n Json) -> Text[]\n\n  ## First.\n  user: Body.\n",
1019        ] {
1020            for newline in ["\n", "\r\n"] {
1021                let prefix = prefix.replace('\n', newline);
1022                let suffix =
1023                    "## Next résumé.\nflow next:\n  run next_worker\nagic last:\n  user: Last.\n"
1024                        .replace('\n', newline);
1025                let source = format!("{prefix}{suffix}");
1026                let tree = parser.parse(&source, None).unwrap();
1027                let root = tree.root_node();
1028                assert!(root.has_error());
1029                assert!(native_errors(root)
1030                    .into_iter()
1031                    .all(|node| node.end_byte() <= prefix.len()));
1032                let mut cursor = root.walk();
1033                let peers: Vec<_> = root
1034                    .named_children(&mut cursor)
1035                    .filter(|node| node.start_byte() >= prefix.len())
1036                    .collect();
1037                assert_eq!(peers.len(), 3);
1038                assert_eq!(peers[0].kind(), "item_doc");
1039                assert_eq!(peers[1].kind(), "flow");
1040                assert_eq!(peers[1].start_byte(), source.find("flow next:").unwrap());
1041                assert_eq!(peers[2].kind(), "agic");
1042                assert!(peers[1..].iter().all(|node| is_valid(*node)));
1043            }
1044        }
1045    }
1046
1047    #[test]
1048    fn can_load_language() {
1049        let mut parser = tree_sitter::Parser::new();
1050        parser.set_language(&super::LANGUAGE.into()).unwrap();
1051        let source = "flow work:\n  repeat 2 times:\n    run improve\n  run publish\n";
1052        let tree = parser.parse(source, None).unwrap();
1053        assert!(!tree.root_node().has_error());
1054        let statements = tree
1055            .root_node()
1056            .named_child(0)
1057            .unwrap()
1058            .child_by_field_name("body")
1059            .unwrap();
1060        assert_eq!(statements.named_child_count(), 2);
1061        let body = statements
1062            .named_child(0)
1063            .unwrap()
1064            .child_by_field_name("body")
1065            .unwrap();
1066        assert_eq!(body.kind(), "repeat_body");
1067        assert_eq!(body.named_child_count(), 1);
1068    }
1069}