1use tree_sitter_language::LanguageFn;
2
3extern "C" {
4 fn tree_sitter_toolang() -> *const ();
5}
6
7pub const LANGUAGE: LanguageFn = unsafe { LanguageFn::from_raw(tree_sitter_toolang) };
8
9#[cfg(test)]
10mod tests {
11 fn native_errors(node: tree_sitter::Node<'_>) -> Vec<tree_sitter::Node<'_>> {
12 let mut pending = vec![node];
13 let mut errors = Vec::new();
14 while let Some(current) = pending.pop() {
15 if current.is_error() || current.is_missing() {
16 errors.push(current);
17 }
18 let mut cursor = current.walk();
19 pending.extend(current.children(&mut cursor));
20 }
21 errors
22 }
23
24 fn is_valid(node: tree_sitter::Node<'_>) -> bool {
25 if node.has_error() || node.is_missing() || node.kind().starts_with("invalid_") {
26 return false;
27 }
28 let mut cursor = node.walk();
29 let valid = node.children(&mut cursor).all(is_valid);
30 valid
31 }
32
33 #[test]
34 fn text_leaves_and_reducer_ranges_share_the_public_contract() {
35 let mut parser = tree_sitter::Parser::new();
36 parser.set_language(&super::LANGUAGE.into()).unwrap();
37 let source = "#@ Program.\n## @param topic Input # details.\nflow work(topic: Text):\n models = fast\n reduce:\n\n # Heading\n Merge.\n initial: Seed.\n run next\n";
38 let tree = parser.parse(source, None).unwrap();
39 let root = tree.root_node();
40 assert!(!root.has_error());
41 for (index, field) in [(0, "text"), (1, "description")] {
42 let text = root
43 .named_child(index)
44 .unwrap()
45 .child_by_field_name(field)
46 .unwrap();
47 assert_eq!(text.kind(), "text");
48 assert_eq!(text.child_count(), 0);
49 }
50 let body = root
51 .named_child(2)
52 .unwrap()
53 .child_by_field_name("body")
54 .unwrap();
55 let value = body
56 .child_by_field_name("directive")
57 .unwrap()
58 .child_by_field_name("value")
59 .unwrap();
60 assert_eq!(value.kind(), "query_value");
61 let reduce = body.child_by_field_name("stmt").unwrap();
62 let content = reduce
63 .child_by_field_name("target")
64 .unwrap()
65 .child_by_field_name("content")
66 .unwrap();
67 assert_eq!(
68 content.utf8_text(source.as_bytes()).unwrap(),
69 "\n # Heading\n Merge.\n"
70 );
71 assert_eq!(content.named_child(0).unwrap().kind(), "blank_line");
72 let text = content.named_child(1).unwrap();
73 assert_eq!(text.kind(), "text");
74 assert_eq!(text.child_count(), 0);
75 assert_eq!(
76 reduce.child_by_field_name("initial").unwrap().kind(),
77 "content"
78 );
79 }
80
81 #[test]
82 fn initial_fields_preserve_nested_literal_text() {
83 let mut parser = tree_sitter::Parser::new();
84 parser.set_language(&super::LANGUAGE.into()).unwrap();
85 let source = "flow work:\n reduce:\n Merge.\n initial: Nested literal.\n initial: \"Seed # literal\"\n run next\n";
86 let tree = parser.parse(source, None).unwrap();
87 let root = tree.root_node();
88 assert!(is_valid(root));
89 let body = root
90 .named_child(0)
91 .unwrap()
92 .child_by_field_name("body")
93 .unwrap();
94 let reduce = body.child_by_field_name("stmt").unwrap();
95 assert_eq!(reduce.kind(), "reduce_stmt");
96 let target = reduce.child_by_field_name("target").unwrap();
97 assert_eq!(
98 target
99 .child_by_field_name("content")
100 .unwrap()
101 .utf8_text(source.as_bytes())
102 .unwrap(),
103 " Merge.\n initial: Nested literal.\n"
104 );
105 assert_eq!(
106 reduce
107 .child_by_field_name("initial")
108 .unwrap()
109 .utf8_text(source.as_bytes())
110 .unwrap(),
111 " \"Seed # literal\""
112 );
113 assert!(target.end_byte() < reduce.child_by_field_name("initial").unwrap().start_byte());
114 }
115
116 #[test]
117 fn malformed_block_colons_cannot_reopen_enclosing_headers() {
118 let mut parser = tree_sitter::Parser::new();
119 parser.set_language(&super::LANGUAGE.into()).unwrap();
120 for header in ["generate :", "keep in 2 lanes:", "sort ascending:"] {
121 for binding in ["", "let value = ", "let "] {
122 for nested in [false, true] {
123 let indent = if nested { " " } else { " " };
124 let prefix = if nested {
125 " repeat:\n run inner\n"
126 } else {
127 ""
128 };
129 let source = format!("flow first:\n run before\n{prefix}{indent}{binding}{header}\n{indent} Orphaned text.\n{indent}## Following.\n{indent}run after\n## Next.\nflow next:\n pass\n");
130 let tree = parser.parse(&source, None).unwrap();
131 let root = tree.root_node();
132 assert!(root.has_error());
133 let flow = root.named_child(0).unwrap();
134 assert_eq!(
135 flow.child_by_field_name("colon").unwrap().start_position(),
136 tree_sitter::Point::new(0, 10)
137 );
138 let body = flow.child_by_field_name("body").unwrap();
139 assert!(!body.named_child(0).unwrap().has_error());
140 let body = if nested {
141 body.named_child(1)
142 .unwrap()
143 .child_by_field_name("body")
144 .unwrap()
145 } else {
146 body
147 };
148 let mut cursor = body.walk();
149 let after = body
150 .named_children(&mut cursor)
151 .find(|node| {
152 node.kind() == "run_stmt"
153 && node
154 .child_by_field_name("target")
155 .unwrap()
156 .utf8_text(source.as_bytes())
157 .unwrap()
158 == "after"
159 })
160 .unwrap();
161 assert!(!after.has_error());
162 let mut cursor = body.walk();
163 assert!(body
164 .named_children(&mut cursor)
165 .any(|node| node.kind() == "item_doc"));
166 assert_eq!(root.named_child(1).unwrap().kind(), "item_doc");
167 assert!(!root.named_child(2).unwrap().has_error());
168 }
169 }
170 }
171 }
172
173 #[test]
174 fn rejected_first_directives_keep_the_authored_body_baseline() {
175 let mut parser = tree_sitter::Parser::new();
176 parser.set_language(&super::LANGUAGE.into()).unwrap();
177 for (kind, following, entry) in [
178 ("flow", "run after", "run_stmt"),
179 ("agic", "user: Following.", "message"),
180 ] {
181 let source = format!("{kind} first:\n modles = fast\n Orphaned content.\n ## Following.\n {following}\n## Next.\nflow next:\n pass\n");
182 let tree = parser.parse(&source, None).unwrap();
183 let root = tree.root_node();
184 assert!(root.has_error());
185 let owner = root.named_child(0).unwrap();
186 assert_eq!(owner.kind(), kind);
187 let body = owner.child_by_field_name("body").unwrap();
188 assert_eq!(body.start_position(), tree_sitter::Point::new(1, 0));
189 let mut cursor = body.walk();
190 let after = body
191 .named_children(&mut cursor)
192 .find(|node| {
193 node.kind() == entry
194 && node.utf8_text(source.as_bytes()).unwrap().trim() == following
195 })
196 .unwrap();
197 assert!(!after.has_error());
198 let mut cursor = body.walk();
199 let doc = body
200 .named_children(&mut cursor)
201 .find(|node| node.kind() == "item_doc")
202 .unwrap();
203 assert_eq!(doc.utf8_text(source.as_bytes()).unwrap(), "## Following.\n");
204 assert_eq!(root.named_child(1).unwrap().kind(), "item_doc");
205 let next = root.named_child(2).unwrap();
206 assert_eq!(next.kind(), "flow");
207 assert!(!next.has_error());
208 }
209 }
210
211 #[test]
212 fn rejected_outer_prefix_docs_preserve_the_next_member_documentation() {
213 let mut parser = tree_sitter::Parser::new();
214 parser.set_language(&super::LANGUAGE.into()).unwrap();
215 for (header, member, body_kind) in [
216 ("flow first:", "run one", "flow_body"),
217 ("agic first:", "user: One.", "agic_body"),
218 ("struct First:", "one: Text", "struct_body"),
219 ("prompt first:", "one = Value", "cap_body"),
220 ("chore first:", "one = Value", "chore_body"),
221 ] {
222 let source = format!("{header}\n#@ Wrong scope.\n# Outer.\n\n ## First.\n {member}\n## Next.\nflow next:\n pass\n");
223 let tree = parser.parse(&source, None).unwrap();
224 let root = tree.root_node();
225 assert!(root.has_error());
226 let body = root
227 .named_child(0)
228 .unwrap()
229 .child_by_field_name("body")
230 .unwrap();
231 assert_eq!(body.kind(), body_kind);
232 assert_eq!(body.start_byte(), source.find(" ## First.").unwrap());
233 let doc = body.named_child(0).unwrap();
234 assert_eq!(doc.kind(), "item_doc");
235 assert!(!doc.has_error());
236 assert_eq!(doc.utf8_text(source.as_bytes()).unwrap(), "## First.\n");
237 assert_eq!(root.named_child(1).unwrap().kind(), "item_doc");
238 let next = root.named_child(2).unwrap();
239 assert!(!next.has_error());
240 assert_eq!(next.kind(), "flow");
241 }
242 }
243
244 #[test]
245 fn leading_member_docs_use_the_body_baseline_and_preserve_empty_boundaries() {
246 let mut parser = tree_sitter::Parser::new();
247 parser.set_language(&super::LANGUAGE.into()).unwrap();
248 let source = "flow first:\n ## Before outer.\n# Outer.\n\n ## First.\n ## Deeper.\n run one\n ## Second.\n run two\nprompt empty:\n## Next.\nflow next:\n pass\n";
249 let tree = parser.parse(source, None).unwrap();
250 let root = tree.root_node();
251 assert!(!root.has_error());
252 let flow = root.named_child(0).unwrap();
253 let body = flow.child_by_field_name("body").unwrap();
254 assert_eq!(body.start_byte(), source.find(" ## First.").unwrap());
255 assert_eq!(body.end_byte(), source.find("prompt empty:").unwrap());
256 let mut cursor = flow.walk();
257 let outer: Vec<_> = flow
258 .named_children(&mut cursor)
259 .filter(|node| node.kind() == "item_doc")
260 .map(|node| node.utf8_text(source.as_bytes()).unwrap())
261 .collect();
262 assert_eq!(outer, ["## Before outer.\n"]);
263 let mut cursor = body.walk();
264 let docs: Vec<_> = body
265 .named_children(&mut cursor)
266 .filter(|node| node.kind() == "item_doc")
267 .map(|node| node.utf8_text(source.as_bytes()).unwrap())
268 .collect();
269 assert_eq!(docs, ["## First.\n", "## Deeper.\n", "## Second.\n"]);
270 let empty = root.named_child(1).unwrap();
271 assert!(empty.child_by_field_name("body").is_none());
272 assert_eq!(empty.end_byte(), source.find("## Next.").unwrap());
273 assert_eq!(root.named_child(2).unwrap().kind(), "item_doc");
274 assert_eq!(root.named_child(3).unwrap().kind(), "flow");
275 }
276
277 #[test]
278 fn rejected_comments_pending_statements_and_until_errors_keep_their_owners() {
279 let mut parser = tree_sitter::Parser::new();
280 parser.set_language(&super::LANGUAGE.into()).unwrap();
281 let source = "flow bad_comment:\n run before\n #@ Wrong scope.\n run after\nflow missing:\n models = fast\n## Next.\nflow bad_until:\n repeat:\n run worker\n until 42\n run after_repeat\nflow next:\n pass\n";
282 let tree = parser.parse(source, None).unwrap();
283 let root = tree.root_node();
284 assert!(root.has_error());
285 let mut cursor = root.walk();
286 let items: Vec<_> = root.named_children(&mut cursor).collect();
287 assert_eq!(
288 items.iter().map(|node| node.kind()).collect::<Vec<_>>(),
289 ["flow", "flow", "item_doc", "flow", "flow"]
290 );
291 for (index, name) in [
292 (0, "bad_comment"),
293 (1, "missing"),
294 (3, "bad_until"),
295 (4, "next"),
296 ] {
297 assert_eq!(
298 items[index]
299 .child_by_field_name("name")
300 .unwrap()
301 .utf8_text(source.as_bytes())
302 .unwrap(),
303 name
304 );
305 }
306 assert!(items[1].end_byte() <= items[2].start_byte());
307 assert_eq!(items[2].utf8_text(source.as_bytes()).unwrap(), "## Next.\n");
308 assert!(!items[4].has_error());
309 for (index, expected) in [(0, vec!["before", "after"]), (3, vec!["after_repeat"])] {
310 let body = items[index].child_by_field_name("body").unwrap();
311 let mut cursor = body.walk();
312 let targets: Vec<_> = body
313 .named_children(&mut cursor)
314 .filter(|node| node.kind() == "run_stmt")
315 .map(|node| {
316 assert!(!node.has_error());
317 node.child_by_field_name("target")
318 .unwrap()
319 .utf8_text(source.as_bytes())
320 .unwrap()
321 })
322 .collect();
323 assert_eq!(targets, expected);
324 }
325 let repeat = items[3]
326 .child_by_field_name("body")
327 .unwrap()
328 .child_by_field_name("stmt")
329 .unwrap();
330 let body = repeat.child_by_field_name("body").unwrap();
331 let worker = body.child_by_field_name("stmt").unwrap();
332 assert_eq!(worker.kind(), "run_stmt");
333 assert_eq!(
334 worker.utf8_text(source.as_bytes()).unwrap().trim(),
335 "run worker"
336 );
337 assert_eq!(body.named_child(1).unwrap().kind(), "invalid_flow_stmt");
338 }
339
340 #[test]
341 fn native_recovery_synchronizes_at_statements_and_program_items() {
342 let mut parser = tree_sitter::Parser::new();
343 parser.set_language(&super::LANGUAGE.into()).unwrap();
344 for prefix in [
345 "flow broken:\n map in 2 lanes in 3 lanes using worker\n ## Following.\n run after\n",
346 "flow broken:\n hands += worker\n run after\n",
347 "agic broken:\n recall = wrong\n user: Following.\n",
348 "struct Broken:\n value Text\n",
349 "flow broken(\n",
350 ] {
351 let source = format!("{prefix}flow next:\n pass\n");
352 let tree = parser.parse(&source, None).unwrap();
353 let root = tree.root_node();
354 assert!(!is_valid(root));
355 let next = root.named_child(root.named_child_count() - 1).unwrap();
356 assert_eq!(next.kind(), "flow");
357 assert!(!next.has_error());
358 assert_eq!(next.start_byte(), prefix.len());
359 assert_eq!(
360 next.child_by_field_name("name")
361 .unwrap()
362 .utf8_text(source.as_bytes())
363 .unwrap(),
364 "next"
365 );
366 if prefix.starts_with("flow broken:") {
367 let body = root
368 .named_child(0)
369 .unwrap()
370 .child_by_field_name("body")
371 .unwrap();
372 assert_eq!(
373 body.child_by_field_name("stmt")
374 .unwrap()
375 .child_by_field_name("target")
376 .unwrap()
377 .utf8_text(source.as_bytes())
378 .unwrap(),
379 "after"
380 );
381 }
382 }
383 }
384
385 #[test]
386 fn public_cst_fields_and_body_ranges_keep_their_roles() {
387 let mut parser = tree_sitter::Parser::new();
388 parser.set_language(&super::LANGUAGE.into()).unwrap();
389 let source = "prompt empty:\n## Next.\nflow work:\n# Preface.\n instruct = named\n run worker\n repeat:\n run worker\n until: Ready.\n";
390 let tree = parser.parse(source, None).unwrap();
391 let root = tree.root_node();
392 assert!(!root.has_error());
393 assert!(root
394 .named_child(0)
395 .unwrap()
396 .child_by_field_name("body")
397 .is_none());
398 assert_eq!(root.named_child(1).unwrap().kind(), "item_doc");
399 let flow = root.named_child(2).unwrap();
400 let body = flow.child_by_field_name("body").unwrap();
401 assert_eq!(body.start_byte(), source.find(" instruct").unwrap());
402 let directive = body.child_by_field_name("directive").unwrap();
403 for (field, kind) in [
404 ("key", "directive_key"),
405 ("operator", "="),
406 ("value", "identifier"),
407 ] {
408 assert_eq!(directive.child_by_field_name(field).unwrap().kind(), kind);
409 }
410 let run = body.child_by_field_name("stmt").unwrap();
411 assert_eq!(
412 run.child_by_field_name("target").unwrap().kind(),
413 "runnable_name"
414 );
415 let condition = body
416 .named_child(body.named_child_count() - 1)
417 .unwrap()
418 .child_by_field_name("body")
419 .unwrap()
420 .child_by_field_name("until")
421 .unwrap();
422 let target = condition.child_by_field_name("target").unwrap();
423 assert_eq!(target.kind(), "inline_agic");
424 assert!(target.child_by_field_name("return").is_none());
425 assert_eq!(
426 target
427 .child_by_field_name("content")
428 .unwrap()
429 .utf8_text(source.as_bytes())
430 .unwrap(),
431 " Ready."
432 );
433 assert_eq!(
434 run.named_child(run.named_child_count() - 1).unwrap().kind(),
435 "newline"
436 );
437 }
438
439 #[test]
440 fn empty_body_and_content_recovery_preserve_peer_documentation() {
441 let mut parser = tree_sitter::Parser::new();
442 parser.set_language(&super::LANGUAGE.into()).unwrap();
443 for header in [
444 "prompt empty:",
445 "chore empty:",
446 "flow empty:",
447 "context empty:",
448 "flow empty:\n run:",
449 ] {
450 for ending in ["\n", "\r\n"] {
451 let source =
452 format!("{header}\n## Next.\nflow next:\n pass\n").replace('\n', ending);
453 let tree = parser.parse(&source, None).unwrap();
454 let root = tree.root_node();
455 assert!(!root.has_error());
456 assert_eq!(root.named_child_count(), 3);
457 let doc = root.named_child(1).unwrap();
458 assert_eq!(doc.kind(), "item_doc");
459 assert_eq!(doc.start_byte(), source.find("## Next.").unwrap());
460 assert!(root.named_child(0).unwrap().end_byte() <= doc.start_byte());
461 assert_eq!(
462 root.named_child(2)
463 .unwrap()
464 .child_by_field_name("name")
465 .unwrap()
466 .utf8_text(source.as_bytes())
467 .unwrap(),
468 "next"
469 );
470 }
471 }
472 }
473
474 #[test]
475 fn unit_spellings_preserve_integer_fields_and_require_prefix_lanes() {
476 let mut parser = tree_sitter::Parser::new();
477 parser.set_language(&super::LANGUAGE.into()).unwrap();
478 for number in [
479 "0",
480 "1",
481 "01",
482 "2",
483 "0002",
484 "9999999999999999999999999999999999999999",
485 ] {
486 for unit in ["lane", "lanes"] {
487 for (head, connector) in [
488 ("generate 3", "using"),
489 ("map", "using"),
490 ("keep", "if"),
491 ("drop", "if"),
492 ("sort ascending", "by"),
493 ] {
494 let source =
495 format!("flow work:\n {head} in {number} {unit} {connector} worker\n");
496 let tree = parser.parse(&source, None).unwrap();
497 assert!(!tree.root_node().has_error());
498 let statement = tree
499 .root_node()
500 .named_child(0)
501 .unwrap()
502 .child_by_field_name("body")
503 .unwrap()
504 .child_by_field_name("stmt")
505 .unwrap();
506 let lanes = statement.child_by_field_name("lanes").unwrap();
507 assert_eq!(lanes.kind(), "integer_literal");
508 assert_eq!(lanes.utf8_text(source.as_bytes()).unwrap(), number);
509 assert_eq!(lanes.child_count(), 0);
510 }
511 }
512 for unit in ["time", "times"] {
513 let source = format!("flow work:\n repeat {number} {unit}:\n run worker\n");
514 let tree = parser.parse(&source, None).unwrap();
515 assert!(!tree.root_node().has_error());
516 let statement = tree
517 .root_node()
518 .named_child(0)
519 .unwrap()
520 .child_by_field_name("body")
521 .unwrap()
522 .child_by_field_name("stmt")
523 .unwrap();
524 assert_eq!(
525 statement
526 .child_by_field_name("count")
527 .unwrap()
528 .utf8_text(source.as_bytes())
529 .unwrap(),
530 number
531 );
532 }
533 }
534 for statement in [
535 "keep if worker in 2 lanes",
536 "drop if worker in 1 lane",
537 "sort ascending by worker in 2 lanes",
538 ] {
539 let source = format!("flow work:\n {statement}\n run after\nflow next:\n pass\n");
540 let tree = parser.parse(source, None).unwrap();
541 assert!(tree.root_node().has_error());
542 assert_eq!(tree.root_node().named_child(1).unwrap().kind(), "flow");
543 assert!(!tree.root_node().named_child(1).unwrap().has_error());
544 }
545 }
546
547 #[test]
548 fn unknown_directives_fail_only_in_the_prefix() {
549 let mut parser = tree_sitter::Parser::new();
550 parser.set_language(&super::LANGUAGE.into()).unwrap();
551 for kind in ["agic", "flow"] {
552 let body = if kind == "agic" {
553 "user: Continue."
554 } else {
555 "run after"
556 };
557 let invalid = format!("{kind} work:\n modles = fast\n {body}\nflow next:\n pass\n");
558 let tree = parser.parse(&invalid, None).unwrap();
559 let root = tree.root_node();
560 assert!(root.has_error());
561 let next = root.named_child(root.named_child_count() - 1).unwrap();
562 assert_eq!(next.kind(), "flow");
563 assert!(!next.has_error());
564 assert_eq!(
565 next.child_by_field_name("name")
566 .unwrap()
567 .utf8_text(invalid.as_bytes())
568 .unwrap(),
569 "next"
570 );
571 let valid = format!("{kind} work:\n {body}\n modles = literal\n");
572 assert!(!parser.parse(valid, None).unwrap().root_node().has_error());
573 }
574 for kind in ["psyche", "skill", "service", "prompt", "chore"] {
575 let source =
576 format!("{kind} example:\n unknown = value\n Body.\n unknown = literal\n");
577 let tree = parser.parse(source, None).unwrap();
578 assert!(!tree.root_node().has_error());
579 let body = tree
580 .root_node()
581 .named_child(0)
582 .unwrap()
583 .child_by_field_name("body")
584 .unwrap();
585 let property = body.child_by_field_name("property").unwrap();
586 assert_eq!(property.kind(), "property");
587 assert!(body.child_by_field_name("content").is_some());
588 }
589 }
590
591 #[test]
592 fn documentation_fields_and_queries_are_available() {
593 let language = super::LANGUAGE.into();
594 let mut parser = tree_sitter::Parser::new();
595 parser.set_language(&language).unwrap();
596 let source = "#!/usr/bin/env too\n#@ Program.\n##! Important.\n## @param _ Input.\n";
597 let tree = parser.parse(source, None).unwrap();
598 let root = tree.root_node();
599 assert_eq!(root.kind(), "program");
600 assert!(!root.has_error());
601 assert_eq!(root.named_child(0).unwrap().kind(), "shebang_comment");
602 for (index, kind, text) in [
603 (1, "program_doc", "Program."),
604 (2, "item_doc", "! Important."),
605 ] {
606 let comment = root.named_child(index).unwrap();
607 assert_eq!(comment.kind(), kind);
608 assert_eq!(comment.child_by_field_name("text").unwrap().kind(), "text");
609 assert_eq!(
610 comment
611 .child_by_field_name("text")
612 .unwrap()
613 .utf8_text(source.as_bytes())
614 .unwrap(),
615 text
616 );
617 }
618 let comment = root.named_child(3).unwrap();
619 assert_eq!(comment.kind(), "param_doc");
620 assert_eq!(comment.start_byte(), source.find("## @param").unwrap());
621 assert_eq!(comment.end_byte(), source.len());
622 assert!(comment.child_by_field_name("parameter").is_none());
623 for (field, value) in [("param", "_"), ("description", "Input.")] {
624 assert_eq!(
625 comment
626 .child_by_field_name(field)
627 .unwrap()
628 .utf8_text(source.as_bytes())
629 .unwrap(),
630 value
631 );
632 }
633 for query in [
634 include_str!("../../queries/highlights.scm"),
635 include_str!("../../queries/injections.scm"),
636 include_str!("../../queries/indents.scm"),
637 include_str!("../../queries/outline.scm"),
638 include_str!("../../queries/tags.scm"),
639 ] {
640 tree_sitter::Query::new(&language, query).unwrap();
641 }
642 }
643
644 #[test]
645 fn spawn_targets_and_bindings_are_available() {
646 let language = super::LANGUAGE.into();
647 let mut parser = tree_sitter::Parser::new();
648 parser.set_language(&language).unwrap();
649 let source = "flow launch:\n spawn worker\n let job = spawn -> Text: Research.\n";
650 let tree = parser.parse(source, None).unwrap();
651 assert!(!tree.root_node().has_error());
652 let statements = tree
653 .root_node()
654 .named_child(0)
655 .unwrap()
656 .child_by_field_name("body")
657 .unwrap();
658 let bare = statements.named_child(0).unwrap();
659 assert_eq!(bare.kind(), "spawn_stmt");
660 assert_eq!(
661 bare.child_by_field_name("target").unwrap().kind(),
662 "runnable_name"
663 );
664 let bound = statements.named_child(1).unwrap();
665 assert_eq!(bound.kind(), "let_stmt");
666 assert_eq!(
667 bound
668 .child_by_field_name("local")
669 .unwrap()
670 .utf8_text(source.as_bytes())
671 .unwrap()
672 .trim(),
673 "job"
674 );
675 let spawn = bound.child_by_field_name("stmt").unwrap();
676 assert_eq!(spawn.kind(), "spawn_stmt");
677 assert_eq!(
678 spawn.child_by_field_name("target").unwrap().kind(),
679 "inline_agic"
680 );
681 }
682
683 #[test]
684 fn repeat_body_exposes_condition_in_source_order() {
685 let mut parser = tree_sitter::Parser::new();
686 parser.set_language(&super::LANGUAGE.into()).unwrap();
687 let source = "flow work:\n repeat:\n run first\n until done\n run last\n";
688 let tree = parser.parse(source, None).unwrap();
689 assert!(!tree.root_node().has_error());
690 let statement = tree
691 .root_node()
692 .named_child(0)
693 .unwrap()
694 .child_by_field_name("body")
695 .unwrap()
696 .named_child(0)
697 .unwrap();
698 assert!(statement.child_by_field_name("until").is_none());
699 let body = statement.child_by_field_name("body").unwrap();
700 assert_eq!(body.kind(), "repeat_body");
701 let condition = body.child_by_field_name("until").unwrap();
702 assert_eq!(condition.kind(), "until_clause");
703 assert_eq!(body.named_child(1).unwrap(), condition);
704 let target = condition.child_by_field_name("target").unwrap();
705 assert_eq!(target.kind(), "runnable_name");
706 assert_eq!(target.utf8_text(source.as_bytes()).unwrap().trim(), "done");
707 let mut cursor = body.walk();
708 let statements: Vec<_> = body.children_by_field_name("stmt", &mut cursor).collect();
709 assert_eq!(statements.len(), 2);
710 assert!(statements[0].end_byte() <= condition.start_byte());
711 assert!(condition.end_byte() <= statements[1].start_byte());
712 }
713
714 #[test]
715 fn launch_handles_share_await_fields() {
716 let language = super::LANGUAGE.into();
717 let mut parser = tree_sitter::Parser::new();
718 parser.set_language(&language).unwrap();
719 for (operation, kind, target) in [
720 ("async run", "run_stmt", "target"),
721 ("spawn", "spawn_stmt", "target"),
722 ] {
723 let source = format!("flow launch:\n let h = {operation} worker\n await h\n let x = await h\n let await h\n let h = await h\n");
724 let tree = parser.parse(&source, None).unwrap();
725 assert!(!tree.root_node().has_error());
726 let statements = tree
727 .root_node()
728 .named_child(0)
729 .unwrap()
730 .child_by_field_name("body")
731 .unwrap();
732 let launch = statements.named_child(0).unwrap();
733 let statement = launch.child_by_field_name("stmt").unwrap();
734 assert_eq!(statement.kind(), kind);
735 assert_eq!(
736 statement
737 .child_by_field_name("async")
738 .map(|node| node.kind()),
739 if operation == "async run" {
740 Some("async")
741 } else {
742 None
743 }
744 );
745 assert_eq!(
746 statement.child_by_field_name(target).unwrap().kind(),
747 "runnable_name"
748 );
749 for (index, name) in [(1, None), (2, Some("x")), (3, None), (4, Some("h"))] {
750 let node = statements.named_child(index).unwrap();
751 let await_node = if index == 1 {
752 node
753 } else {
754 assert_eq!(node.kind(), "let_stmt");
755 assert!(node.child_by_field_name("value").is_none());
756 assert_eq!(
757 node.child_by_field_name("local")
758 .map(|node| node.utf8_text(source.as_bytes()).unwrap().trim()),
759 name
760 );
761 node.child_by_field_name("stmt").unwrap()
762 };
763 assert_eq!(await_node.kind(), "await_stmt");
764 let handle = await_node.child_by_field_name("handle").unwrap();
765 assert_eq!(handle.kind(), "handle_name");
766 assert_eq!(handle.child_count(), 0);
767 assert_eq!(handle.utf8_text(source.as_bytes()).unwrap().trim(), "h");
768 }
769 }
770 }
771
772 #[test]
773 fn anonymous_terminals_keep_fields_and_native_message_errors_stay_local() {
774 let mut parser = tree_sitter::Parser::new();
775 parser.set_language(&super::LANGUAGE.into()).unwrap();
776 let source = "flow work(input?: Text[]) -> Text:\n models += fast\n async run worker\n keep last 2\nagic messages:\n user missing colon\n user: After.\nflow next:\n pass\n";
777 let tree = parser.parse(source, None).unwrap();
778 let root = tree.root_node();
779 assert!(root.has_error());
780 let flow = root.named_child(0).unwrap();
781 let parameter = flow
782 .child_by_field_name("params")
783 .unwrap()
784 .child_by_field_name("param")
785 .unwrap();
786 let body = flow.child_by_field_name("body").unwrap();
787 let directive = body.child_by_field_name("directive").unwrap();
788 let run = body.child_by_field_name("stmt").unwrap();
789 let mut cursor = body.walk();
790 let keep = body
791 .named_children(&mut cursor)
792 .find(|node| node.kind() == "keep_stmt")
793 .unwrap();
794 let selection = keep.child_by_field_name("selection").unwrap();
795 assert_eq!(selection.kind(), "selection");
796 let ty = parameter.child_by_field_name("type").unwrap();
797 for (owner, field, literal) in [
798 (flow, "keyword", "flow"),
799 (flow, "arrow", "->"),
800 (parameter, "optional", "?"),
801 (ty, "suffix", "[]"),
802 (directive, "operator", "+="),
803 (run, "async", "async"),
804 (selection, "side", "last"),
805 ] {
806 let token = owner.child_by_field_name(field).unwrap();
807 assert_eq!(token.kind(), literal);
808 assert!(!token.is_named());
809 assert_eq!(&source[token.byte_range()], literal);
810 }
811 let body = root
812 .named_child(1)
813 .unwrap()
814 .child_by_field_name("body")
815 .unwrap();
816 let mut cursor = body.walk();
817 let entries: Vec<_> = body
818 .children_by_field_name("message", &mut cursor)
819 .collect();
820 assert_eq!(
821 entries.iter().map(|node| node.kind()).collect::<Vec<_>>(),
822 ["message"]
823 );
824 let diagnostic = native_errors(body)
825 .into_iter()
826 .find(|node| node.is_error())
827 .unwrap();
828 assert_eq!(diagnostic.parent().unwrap(), body);
829 assert_eq!(
830 diagnostic.utf8_text(source.as_bytes()).unwrap().trim(),
831 "user missing colon"
832 );
833 assert!(entries[0].child_by_field_name("content").is_some());
834 assert_eq!(
835 root.named_child(2)
836 .unwrap()
837 .child_by_field_name("name")
838 .unwrap()
839 .utf8_text(source.as_bytes())
840 .unwrap(),
841 "next"
842 );
843 }
844
845 #[test]
846 fn invalid_parameter_annotations_and_directives_keep_their_owners() {
847 let mut parser = tree_sitter::Parser::new();
848 parser.set_language(&super::LANGUAGE.into()).unwrap();
849 for kind in ["agic", "flow"] {
850 for parameter in [": Text", "?: Text[]", "topic:", "topic?:"] {
851 let source = format!("{kind} work({parameter}, next: Text) -> Text:\n models = fast\n models after\n tools = fast\n## Next.\nflow next:\n pass\n");
852 let tree = parser.parse(&source, None).unwrap();
853 let root = tree.root_node();
854 assert!(!is_valid(root));
855 let owner = root.named_child(0).unwrap();
856 assert_eq!(owner.kind(), kind);
857 let params = owner.child_by_field_name("params").unwrap();
858 let bad = params.child_by_field_name("param").unwrap();
859 if parameter.starts_with(':') || parameter.starts_with('?') {
860 assert_eq!(bad.kind(), "invalid_param");
861 assert_eq!(bad.utf8_text(source.as_bytes()).unwrap(), parameter);
862 } else {
863 assert_eq!(bad.kind(), "param");
864 assert!(params.has_error());
865 assert!(bad.child_by_field_name("type").is_none());
866 }
867 let body = owner.child_by_field_name("body").unwrap();
868 let mut cursor = body.walk();
869 let entries: Vec<_> = body
870 .children_by_field_name("directive", &mut cursor)
871 .collect();
872 assert_eq!(
873 entries.iter().map(|node| node.kind()).collect::<Vec<_>>(),
874 ["directive", "directive"]
875 );
876 assert_eq!(
877 entries[1]
878 .child_by_field_name("key")
879 .unwrap()
880 .utf8_text(source.as_bytes())
881 .unwrap(),
882 "tools"
883 );
884 assert_eq!(entries[1].parent().unwrap(), body);
885 assert!(native_errors(body).into_iter().any(|node| node.is_error()
886 && node.utf8_text(source.as_bytes()).unwrap().trim() == "models after"));
887 assert_eq!(root.named_child(1).unwrap().kind(), "item_doc");
888 let next = root.named_child(2).unwrap();
889 assert_eq!(next.kind(), "flow");
890 assert_eq!(
891 next.child_by_field_name("name")
892 .unwrap()
893 .utf8_text(source.as_bytes())
894 .unwrap(),
895 "next"
896 );
897 assert!(!next.has_error());
898 }
899 }
900 }
901
902 #[test]
903 fn return_annotation_recovery_keeps_headers_and_inline_content() {
904 let mut parser = tree_sitter::Parser::new();
905 parser.set_language(&super::LANGUAGE.into()).unwrap();
906 for (header, missing_type) in [
907 ("flow work(topic: Text -> Text: # ) ,", false),
908 ("flow work(: Text?) -> : # Header.", true),
909 ] {
910 let source = format!("{header}\n ## First.\n reduce -> :\n\n Merge.\n initial: Initial.\n run after\n## Next.\nflow next:\n pass\n");
911 let tree = parser.parse(&source, None).unwrap();
912 let root = tree.root_node();
913 assert!(!is_valid(root));
914 let flow = root.named_child(0).unwrap();
915 assert_eq!(flow.kind(), "flow");
916 assert_eq!(flow.child_by_field_name("return").is_none(), missing_type);
917 let body = flow.child_by_field_name("body").unwrap();
918 assert_eq!(body.named_child(0).unwrap().kind(), "item_doc");
919 let reducer = body.child_by_field_name("stmt").unwrap();
920 let inline = reducer.child_by_field_name("target").unwrap();
921 assert!(root.has_error());
922 assert!(native_errors(reducer)
923 .into_iter()
924 .any(|node| node.is_error() && node.utf8_text(source.as_bytes()).unwrap() == "->"));
925 assert!(inline.child_by_field_name("return").is_none());
926 assert_eq!(
927 inline
928 .child_by_field_name("content")
929 .unwrap()
930 .utf8_text(source.as_bytes())
931 .unwrap(),
932 "\n Merge.\n"
933 );
934 assert_eq!(
935 reducer
936 .child_by_field_name("initial")
937 .unwrap()
938 .utf8_text(source.as_bytes())
939 .unwrap(),
940 " Initial."
941 );
942 assert_eq!(root.named_child(1).unwrap().kind(), "item_doc");
943 assert!(is_valid(root.named_child(2).unwrap()));
944 }
945 }
946
947 #[test]
948 fn sync_and_async_target_errors_keep_the_recognized_name() {
949 let mut parser = tree_sitter::Parser::new();
950 parser.set_language(&super::LANGUAGE.into()).unwrap();
951 for modifier in ["", "async "] {
952 for binding in ["", "let ", "let result = "] {
953 let source = format!("flow first:\n {binding}{modifier}run worker extra\n run after\nflow next:\n pass\n");
954 let tree = parser.parse(&source, None).unwrap();
955 let root = tree.root_node();
956 assert!(root.has_error());
957 let body = root
958 .named_child(0)
959 .unwrap()
960 .child_by_field_name("body")
961 .unwrap();
962 let statement = body.child_by_field_name("stmt").unwrap();
963 let run = if binding.is_empty() {
964 statement
965 } else {
966 statement.child_by_field_name("stmt").unwrap()
967 };
968 assert_eq!(run.kind(), "run_stmt");
969 assert_eq!(
970 run.child_by_field_name("target")
971 .unwrap()
972 .utf8_text(source.as_bytes())
973 .unwrap(),
974 "worker"
975 );
976 let errors = native_errors(run);
977 assert_eq!(errors.len(), 1);
978 assert!(errors[0].is_error());
979 assert_eq!(errors[0].utf8_text(source.as_bytes()).unwrap(), "extra");
980 assert_eq!(errors[0].start_byte(), source.find("extra").unwrap());
981 assert!(is_valid(root.named_child(1).unwrap()));
982 }
983 }
984 }
985
986 #[test]
987 fn missing_directive_values_are_visible_public_tokens() {
988 let mut parser = tree_sitter::Parser::new();
989 parser.set_language(&super::LANGUAGE.into()).unwrap();
990 for (key, kind) in [
991 ("models", "query_value"),
992 ("context", "identifier"),
993 ("instruct", "identifier"),
994 ] {
995 let source = format!("flow first:\n {key} =\n run after\nflow next:\n pass\n");
996 let tree = parser.parse(&source, None).unwrap();
997 let root = tree.root_node();
998 assert!(root.has_error());
999 let errors = native_errors(root);
1000 assert_eq!(errors.len(), 1);
1001 assert_eq!(errors[0].kind(), kind);
1002 assert!(errors[0].is_missing());
1003 assert_eq!(errors[0].start_byte(), errors[0].end_byte());
1004 assert!(is_valid(root.named_child(1).unwrap()));
1005 }
1006 }
1007
1008 #[test]
1009 fn incomplete_header_terminators_preserve_following_items() {
1010 let mut parser = tree_sitter::Parser::new();
1011 parser.set_language(&super::LANGUAGE.into()).unwrap();
1012 for prefix in [
1013 "flow first:\n repeat 2 times windowing 1,\n run worker\n until ready\n",
1014 "flow first:\n repeat 2 times windowing 1\n\n run worker\n until ready\n",
1015 "agic first(_: Text) -> Json\n models = fast\n recall = near\n tool: Body.\n",
1016 "agic first(_: Text) -> J[on:\n models = fast\n recall = near\n tool: Body.\n",
1017 "flow first(topic?, Text[]\n other: Json) -> Text[]:\n ## First.\n run worker\n",
1018 "agic first(topic?: Text[], other\n Json) -> Text[]\n\n ## First.\n user: Body.\n",
1019 ] {
1020 for newline in ["\n", "\r\n"] {
1021 let prefix = prefix.replace('\n', newline);
1022 let suffix =
1023 "## Next résumé.\nflow next:\n run next_worker\nagic last:\n user: Last.\n"
1024 .replace('\n', newline);
1025 let source = format!("{prefix}{suffix}");
1026 let tree = parser.parse(&source, None).unwrap();
1027 let root = tree.root_node();
1028 assert!(root.has_error());
1029 assert!(native_errors(root)
1030 .into_iter()
1031 .all(|node| node.end_byte() <= prefix.len()));
1032 let mut cursor = root.walk();
1033 let peers: Vec<_> = root
1034 .named_children(&mut cursor)
1035 .filter(|node| node.start_byte() >= prefix.len())
1036 .collect();
1037 assert_eq!(peers.len(), 3);
1038 assert_eq!(peers[0].kind(), "item_doc");
1039 assert_eq!(peers[1].kind(), "flow");
1040 assert_eq!(peers[1].start_byte(), source.find("flow next:").unwrap());
1041 assert_eq!(peers[2].kind(), "agic");
1042 assert!(peers[1..].iter().all(|node| is_valid(*node)));
1043 }
1044 }
1045 }
1046
1047 #[test]
1048 fn can_load_language() {
1049 let mut parser = tree_sitter::Parser::new();
1050 parser.set_language(&super::LANGUAGE.into()).unwrap();
1051 let source = "flow work:\n repeat 2 times:\n run improve\n run publish\n";
1052 let tree = parser.parse(source, None).unwrap();
1053 assert!(!tree.root_node().has_error());
1054 let statements = tree
1055 .root_node()
1056 .named_child(0)
1057 .unwrap()
1058 .child_by_field_name("body")
1059 .unwrap();
1060 assert_eq!(statements.named_child_count(), 2);
1061 let body = statements
1062 .named_child(0)
1063 .unwrap()
1064 .child_by_field_name("body")
1065 .unwrap();
1066 assert_eq!(body.kind(), "repeat_body");
1067 assert_eq!(body.named_child_count(), 1);
1068 }
1069}