big_code_analysis/ops.rs
1// Per-language metric and AST modules deliberately consume the macro-
2// generated tree-sitter token enums via `use crate::*` and `use Foo::*`
3// inside match expressions — explicit imports would list dozens of
4// variants per arm and obscure the per-language token sets that are the
5// point of these files. Allowed at the module level rather than per
6// function so the per-language impl blocks stay readable.
7#![allow(clippy::wildcard_imports, clippy::enum_glob_use)]
8
9use std::borrow::Cow;
10
11use crate::checker::Checker;
12use crate::error::MetricsError;
13use crate::getter::Getter;
14use crate::node::{Ancestors, Node};
15use crate::spaces::{SpaceKind, line_span, push_children};
16
17use crate::halstead::{Halstead, HalsteadMaps};
18
19use crate::MetricSuite;
20
21/// All operands and operators of a space.
22#[derive(Debug, Clone)]
23pub struct Ops {
24 /// The name of a function space.
25 ///
26 /// For the top-level (file-level) `Ops` the value is whatever
27 /// `Source::name` the caller supplied to the [`crate::Ast::ops`]
28 /// seam — `Some` or `None`.
29 ///
30 /// For nested spaces, `None` means an error occurred in parsing the
31 /// name of the function space from the AST.
32 pub name: Option<String>,
33 /// `true` when [`Ops::name`] was produced by lossy conversion (the
34 /// original path contained non-UTF-8 bytes and was rendered using
35 /// U+FFFD replacement characters). The explicit-name
36 /// [`crate::Ast::ops`] seam never sets it, since a caller-supplied
37 /// `String` name is UTF-8 by construction, so it is always `false`
38 /// in current code paths. Retained as a wire field for forward
39 /// compatibility; skipped from JSON output when `false` so existing
40 /// schemas keep their shape.
41 pub name_was_lossy: bool,
42 /// The first line of a function space.
43 pub start_line: usize,
44 /// The last line of a function space.
45 pub end_line: usize,
46 /// The space kind.
47 pub kind: SpaceKind,
48 /// All subspaces contained in a function space.
49 pub spaces: Vec<Ops>,
50 /// The **distinct** operands of a space — the deduplicated Halstead
51 /// operand vocabulary (`n2`), one entry per unique operand, not every
52 /// occurrence. Sorted in byte-lexicographic order, so the same input
53 /// always yields the same sequence (#1091).
54 pub operands: Vec<String>,
55 /// The **distinct** operators of a space — the deduplicated Halstead
56 /// operator vocabulary (`n1`), one entry per unique operator, not
57 /// every occurrence. Sorted in byte-lexicographic order, so the same
58 /// input always yields the same sequence (#1091).
59 pub operators: Vec<String>,
60}
61
62// Space nesting is caller-controlled, so the compiler-generated `Drop`
63// glue would recurse once per level and abort the process on a deep tree
64// (#1056). See [`crate::recursion`].
65crate::recursion::impl_iterative_drop!(Ops, spaces);
66
67impl Ops {
68 /// Project this tree into its [`crate::wire::Ops`] form — the
69 /// plain, `Deserialize`-capable record that defines the serialized
70 /// shape.
71 #[must_use]
72 pub fn to_wire(&self) -> crate::wire::Ops {
73 crate::wire::Ops::from(self)
74 }
75
76 fn new<'a, T: Getter>(
77 node: &Node<'a>,
78 code: &[u8],
79 ancestors: Ancestors<'a, '_>,
80 kind: SpaceKind,
81 ) -> Self {
82 let (start_position, end_position) = line_span(node, kind);
83 // The top-level Unit's name is overwritten by `ops_inner` with the
84 // caller-supplied name before returning, so computing it here is
85 // wasted work. Non-top-level Unit spaces have no resolvable name, so
86 // leaving `None` matches the documented "could not be resolved"
87 // semantics rather than inventing the `<anonymous>` placeholder the
88 // default getter returns. Other kinds keep the AST-derived name.
89 // Mirrors the `SpaceKind::Unit` handling in `FuncSpace::new`.
90 let name = (kind != SpaceKind::Unit)
91 .then(|| T::get_func_space_name(node, code, ancestors).map(str::to_owned))
92 .flatten();
93 Self {
94 name,
95 name_was_lossy: false,
96 spaces: Vec::new(),
97 kind,
98 start_line: start_position,
99 end_line: end_position,
100 operators: Vec::new(),
101 operands: Vec::new(),
102 }
103 }
104}
105
106#[derive(Debug, Clone)]
107struct State<'a> {
108 ops: Ops,
109 halstead_maps: HalsteadMaps<'a>,
110}
111
112/// Pushes a synthetic `Unit` root onto the state stack when the grammar
113/// hands us a non-`Unit` root.
114///
115/// Mirrors [`crate::spaces::push_synthetic_unit_root`] on the metrics
116/// seam: some grammars (e.g. tree-sitter-lua / tree-sitter-mozcpp on
117/// unparseable input) return an `ERROR` root that is not classified as a
118/// function space, so without this push the walk would never open a
119/// frame and `ops_inner` would return [`MetricsError::EmptyRoot`] for an
120/// input where `metrics()` succeeds (issue #789). A `Unit` root needs no
121/// wrapper, so nothing is pushed in that case.
122fn push_synthetic_unit_root<T: MetricSuite>(
123 state_stack: &mut Vec<State>,
124 node: &Node,
125 code: &[u8],
126) {
127 // `Ancestors::unknown()`: `node` is the tree root here, so it has
128 // no ancestors to hand over either way.
129 if T::Getter::get_space_kind_with_code(node, code, Ancestors::unknown()) != SpaceKind::Unit {
130 state_stack.push(State {
131 ops: Ops::new::<T::Getter>(node, code, Ancestors::unknown(), SpaceKind::Unit),
132 halstead_maps: HalsteadMaps::new(),
133 });
134 }
135}
136
137// Space-kind classifications `ops_inner` has performed on this thread.
138//
139// The classification only ever reaches `Ops::new`, so running it on a
140// node that opens no space is work thrown away — and nothing about the
141// walk's *output* can tell the two apart. The counter is the
142// observable: it reads one per space after the lookup was moved inside
143// the `func_space` branch, and one per *node* before, which is what
144// makes hoisting it back out a test failure rather than a silent
145// regression (#1110).
146crate::observation::counter!(space_kind_lookups);
147
148/// Classifies a node that is about to open a function space.
149///
150/// Classification happens after the decision that a space opens, the
151/// same way [`crate::spaces::compute`]'s `open_func_space` does it
152/// (#522), and through the same source-aware classifier, so a space
153/// both seams open carries the same [`SpaceKind`] in either walk.
154/// The `_with_code` variant is what lets Elixir's macro-shaped
155/// `defmodule` / `def` declarations — plain `Call` nodes distinguished
156/// only by their target identifier text — come back as `Class` /
157/// `Function` rather than `Unknown` (#275, #1130).
158///
159/// The lookup is a per-language `match` on the node's kind for most
160/// grammars, but C#'s reaches a child scan for a bodied indexer or
161/// property and Elixir's reads the `Call` target text and scans the
162/// ancestor chain for an enclosing `quote` block, so it is not free on
163/// every node either.
164fn classify_space_kind<'a, T: MetricSuite>(
165 node: &Node<'a>,
166 code: &[u8],
167 ancestors: Ancestors<'a, '_>,
168) -> SpaceKind {
169 space_kind_lookups::record();
170 T::Getter::get_space_kind_with_code(node, code, ancestors)
171}
172
173/// Render a space's vocabulary: byte-lexicographically ordered, one
174/// owned `String` per distinct entry.
175///
176/// # Order
177///
178/// `HashMap`'s hasher is randomly seeded per instance, so key order
179/// differs between two runs — and even between two parses in one
180/// process. Without a canonical order the same input renders and
181/// serializes differently every time, which makes `bca ops` output
182/// undiffable and unusable as a cache key (#1091).
183///
184/// Byte-lexicographic, not first-appearance, order: `finalize` merges
185/// a child space's maps into its parent, so an insertion-ordered map
186/// would give the parent "what it saw directly, then whatever each
187/// child contributed" — a walk artifact that shifts when nesting
188/// changes. Sorting is also stable across platforms and hasher
189/// versions, which insertion order via a different map type is not.
190///
191/// # Why the sort runs before the `String`s exist
192///
193/// Every ancestor of a space re-renders that space's whole vocabulary:
194/// `finalize` merges each child's Halstead maps into its parent, so a
195/// parent's key set is a superset of every descendant's and an entry
196/// nested `D` spaces deep is rendered `D + 1` times. Sorting the
197/// borrowed keys first makes each swap a fat pointer rather than a
198/// 24-byte `String`, and — because the `String`s are then allocated in
199/// output order — it hands every later pass over them (the wire
200/// projection, serialization, the `dump_ops` tree) a heap laid out in
201/// the order it reads. Issue #1110 measured both effects.
202///
203/// # Non-UTF-8 keys
204///
205/// Tree-sitter sources are expected to be valid UTF-8; non-UTF-8 bytes
206/// are replaced with the Unicode replacement character to keep the entry
207/// visible (rather than silently dropping it or using a sentinel string
208/// that could collide with a real identifier). That rendering is not
209/// order-preserving — `b"\xffA"` sorts after `"\u{fffd}B"` by raw bytes
210/// and before it once both are rendered — so a vocabulary that actually
211/// lost bytes is re-sorted on the rendered text, which is the order the
212/// pre-#1110 render-then-sort code produced. Valid UTF-8, which is every
213/// key in practice, takes the single sort.
214fn sorted_vocabulary(mut keys: Vec<&[u8]>) -> Vec<String> {
215 keys.sort_unstable();
216 let mut lossy = false;
217 let mut rendered: Vec<String> = keys
218 .into_iter()
219 .map(|key| match String::from_utf8_lossy(key) {
220 Cow::Borrowed(text) => text.to_owned(),
221 Cow::Owned(text) => {
222 lossy = true;
223 text
224 }
225 })
226 .collect();
227 if lossy {
228 rendered.sort_unstable();
229 }
230 rendered
231}
232
233fn compute_operators_and_operands<T: MetricSuite>(state: &mut State) {
234 let maps = &state.halstead_maps;
235
236 // Primitive-type operators live in a second map (keyed by text rather
237 // than by token id), so the operator vocabulary is the concatenation
238 // of both key sets.
239 let operators = maps
240 .operators
241 .keys()
242 .map(|k| T::Getter::get_operator_id_as_str(*k).as_bytes())
243 .chain(maps.primitive_operators.keys().copied())
244 .collect();
245
246 state.ops.operators = sorted_vocabulary(operators);
247 state.ops.operands = sorted_vocabulary(maps.operands.keys().copied().collect());
248}
249
250/// Close up to `diff_level` open spaces, folding each into its parent.
251///
252/// Only the states this pops get their vocabularies computed. The
253/// bottom state is never popped here, so the root's vocabulary is built
254/// once by [`ops_inner`] after the final drain — computing it on every
255/// call would rebuild (and, since #1091, re-sort) the whole file's
256/// vocabulary once per level-drop in the walk, and every result but the
257/// last would be overwritten.
258fn finalize<T: MetricSuite>(state_stack: &mut Vec<State>, diff_level: usize) {
259 for _ in 0..diff_level {
260 if state_stack.len() < 2 {
261 break;
262 }
263 let mut state = state_stack
264 .pop()
265 .expect("state_stack verified to have len >= 2");
266 let last_state = state_stack
267 .last_mut()
268 .expect("state_stack verified to have len >= 1 after pop");
269
270 // Populate the child's ops from its HalsteadMaps before
271 // recording it as a sub-space of the parent.
272 compute_operators_and_operands::<T>(&mut state);
273
274 // Merge child's Halstead maps into parent and record child space.
275 last_state.halstead_maps.merge(&state.halstead_maps);
276 last_state.ops.spaces.push(state.ops);
277 }
278}
279
280/// Context the ops walk carries down the tree alongside each node.
281///
282/// A named pair rather than a `(usize, usize)`: the two counts advance
283/// on different events and swapping them is silent — `level` only moves
284/// at space boundaries while `depth` counts every AST step. Mirrors
285/// [`crate::spaces::compute`]'s `Walk`, minus the comment flag this walk
286/// has no use for.
287#[derive(Clone, Copy)]
288struct Walk {
289 /// Nesting level, used to close op-spaces on the way back up.
290 level: usize,
291 /// AST depth — the number of ancestors this node has, so the root
292 /// sits at `0`. Indexes the ancestor chain the walk maintains.
293 depth: usize,
294}
295
296/// Explicit-name core of the operator/operand walk backing the
297/// [`crate::Ast::ops`] `Source`-based seam. The top-level [`Ops::name`]
298/// is whatever the caller passes in `name`; `name_was_lossy` is left at
299/// its `false` default because an explicit `String` name is never lossy.
300/// Mirrors [`crate::spaces::metrics_inner`].
301pub(crate) fn ops_inner<T: MetricSuite>(
302 parser: &T,
303 name: Option<String>,
304) -> Result<Ops, MetricsError> {
305 let code = parser.code();
306 let node = parser.root();
307 let mut cursor = node.cursor();
308 let mut stack = Vec::new();
309 // Ancestor chain of the node currently being visited, root first,
310 // maintained by the same truncate/push rule as
311 // `spaces::compute::metrics_inner` (#1084).
312 let mut chain: Vec<Node<'_>> = Vec::new();
313 let mut state_stack: Vec<State> = Vec::new();
314 let mut last_level = 0;
315
316 // Mirror `metrics_inner`: wrap a non-`Unit` (e.g. `ERROR`) root in a
317 // synthetic `Unit` frame so the walk always has a frame to populate.
318 // Without this, an `ERROR`-root parse drains the state stack and
319 // `ops_inner` returns `EmptyRoot` for inputs where `metrics()`
320 // succeeds (issue #789).
321 push_synthetic_unit_root::<T>(&mut state_stack, &node, code);
322
323 stack.push((node, Walk { level: 0, depth: 0 }));
324
325 while let Some((node, Walk { level, depth })) = stack.pop() {
326 chain.truncate(depth);
327
328 if level < last_level {
329 finalize::<T>(&mut state_stack, last_level - level);
330 last_level = level;
331 }
332
333 let ancestors = Ancestors::checked(&chain, &node);
334
335 // Same predicate `spaces::compute::metrics_inner` opens on, so
336 // the two walks agree on which nodes become spaces. The
337 // byte-less `is_func || is_func_space` this replaced could not
338 // see Elixir's macro-shaped declarations, which are `Call`
339 // nodes identified by their target text, so `bca ops` opened no
340 // space for a `defmodule` / `def` / `defp` / `defmacro` — only
341 // for `Source` and an explicit `fn … -> … end` (#1130).
342 let func_space = T::Checker::promotes_to_func_space_with_code(&node, code, ancestors);
343
344 let new_level = if func_space {
345 let kind = classify_space_kind::<T>(&node, code, ancestors);
346 let state = State {
347 ops: Ops::new::<T::Getter>(&node, code, ancestors, kind),
348 halstead_maps: HalsteadMaps::new(),
349 };
350 state_stack.push(state);
351 last_level = level + 1;
352 last_level
353 } else {
354 level
355 };
356
357 if let Some(state) = state_stack.last_mut() {
358 T::Halstead::compute(&node, code, ancestors, &mut state.halstead_maps);
359 }
360
361 chain.push(node);
362
363 // Shared with `metrics_inner` (issue #969): `push_children` is
364 // State-independent — it only moves the cursor over child nodes —
365 // so unlike the local `finalize` / `push_synthetic_unit_root`
366 // mirrors (which differ by `State` payload) it is reused directly
367 // rather than duplicated. The source-order-then-reverse ordering
368 // it encapsulates is load-bearing for suppression attribution.
369 // The returned child slice is only useful to `metrics_inner`,
370 // which seeds their cognitive nesting; `ops` just walks them.
371 push_children(
372 &mut cursor,
373 &node,
374 Walk {
375 level: new_level,
376 depth: depth + 1,
377 },
378 &mut stack,
379 );
380 }
381
382 finalize::<T>(&mut state_stack, usize::MAX);
383
384 // Reserved error path: `MetricsError::EmptyRoot` is unreachable
385 // today because the synthetic Unit push above (and every supported
386 // language's root being recognised as a `func_space`) keeps the
387 // state stack non-empty for every input, including ERROR-root,
388 // empty, whitespace-only, and comment-only sources — matching
389 // `metrics_inner`. The `ok_or` is retained so a future walker change
390 // that legitimately drains the stack surfaces a distinct error
391 // variant rather than a bare `None`. See `MetricsError::EmptyRoot`
392 // for the matching variant doc.
393 let mut state = state_stack.pop().ok_or(MetricsError::EmptyRoot)?;
394 // The root is the one state `finalize` never pops, so its vocabulary
395 // is built here — once, from the fully-merged maps.
396 compute_operators_and_operands::<T>(&mut state);
397 state.ops.name = name;
398 Ok(state.ops)
399}
400
401#[cfg(test)]
402#[allow(
403 clippy::float_cmp,
404 clippy::cast_precision_loss,
405 clippy::cast_possible_truncation,
406 clippy::cast_sign_loss,
407 clippy::similar_names,
408 clippy::doc_markdown,
409 clippy::needless_raw_string_hashes,
410 clippy::too_many_lines
411)]
412mod tests {
413 use super::Ops;
414 use crate::{Ast, LANG, Source};
415
416 #[cfg(any(
417 feature = "cpp",
418 feature = "java",
419 feature = "javascript",
420 feature = "mozjs",
421 feature = "perl",
422 feature = "python",
423 feature = "rust",
424 feature = "typescript",
425 ))]
426 #[inline]
427 fn check_ops(
428 lang: LANG,
429 source: &str,
430 file: &str,
431 correct_operators: &mut [&str],
432 correct_operands: &mut [&str],
433 ) {
434 let mut trimmed_bytes = source.trim_end().trim_matches('\n').as_bytes().to_vec();
435 trimmed_bytes.push(b'\n');
436 let ops = Ast::parse(Source::new(lang, &trimmed_bytes).with_name(Some(file.to_owned())))
437 .expect("language feature enabled")
438 .ops()
439 .expect("ops walk must yield a top-level Ops");
440
441 let operators_str: Vec<&str> = ops.operators.iter().map(AsRef::as_ref).collect();
442 let operands_str: Vec<&str> = ops.operands.iter().map(AsRef::as_ref).collect();
443
444 // Only the *expectations* are sorted here: `Ops` is documented to
445 // come back byte-lexicographically ordered (#1091), so comparing
446 // against a sorted expectation without re-sorting the actual value
447 // makes every `check_ops` caller an ordering regression test for
448 // its language.
449 correct_operators.sort_unstable();
450 assert_eq!(&operators_str[..], correct_operators);
451
452 correct_operands.sort_unstable();
453 assert_eq!(&operands_str[..], correct_operands);
454 }
455
456 #[cfg(feature = "python")]
457 #[test]
458 fn python_ops() {
459 check_ops(
460 LANG::Python,
461 "if True:
462 a = 1 + 2",
463 "foo.py",
464 &mut ["if", ":", "=", "+"],
465 &mut ["True", "a", "1", "2"],
466 );
467 }
468
469 #[cfg(feature = "perl")]
470 #[test]
471 fn perl_pattern_operations_render_as_source_spellings() {
472 // #1314 classifies `s///` and `tr///` as Halstead operators.
473 // They are *named* nodes rather than punctuation tokens, so the
474 // `get_operator!` macro's fallback would render each kind's own
475 // name — `substitution_pattern_s`, `transliteration_tr_or_y` —
476 // into `bca ops`, which reads as a bug rather than as Perl.
477 // `PerlCode::get_operator_id_as_str` is hand-written to map
478 // them, and this is what pins that mapping: the counts alone
479 // cannot see it.
480 //
481 // `y///` is a synonym of `tr///` and shares one kind, so the
482 // two source spellings collapse to a single `tr///` entry —
483 // asserted here by the *absence* of a `y///` row in an
484 // exhaustive expectation, not by a negative assertion.
485 //
486 // The trailing `/pat/` keeps a pattern *value* in the fixture,
487 // so the test also shows the split: the operation spellings are
488 // operators while the value is an operand.
489 check_ops(
490 LANG::Perl,
491 "$s =~ s/a/b/;\n$s =~ tr/c/d/;\n$s =~ y/e/f/;\n$t = /pat/;",
492 "foo.pl",
493 &mut ["$", ";", "=", "=~", "s///", "tr///"],
494 &mut ["$s", "$t", "/pat/"],
495 );
496 }
497
498 /// The fat comma is a named node, so the generic fallback rendered
499 /// its kind name, `fat_comma`; it lists as the `=>` written.
500 #[cfg(feature = "perl")]
501 #[test]
502 fn perl_fat_comma_renders_as_written() {
503 check_ops(
504 LANG::Perl,
505 "my %h = (a => 1);",
506 "foo.pl",
507 &mut ["()", ";", "=", "=>", "my"],
508 &mut ["%h", "1", "a"],
509 );
510 }
511
512 #[cfg(feature = "python")]
513 #[test]
514 fn python_function_ops() {
515 check_ops(
516 LANG::Python,
517 "def foo():
518 def bar():
519 def toto():
520 a = 1 + 1
521 b = 2 + a
522 c = 3 + 3",
523 "foo.py",
524 &mut ["def", "()", ":", "=", "+"],
525 &mut ["foo", "bar", "toto", "a", "b", "c", "1", "2", "3"],
526 );
527 }
528
529 /// #1486: Python bills a call's parentheses the way the JS family
530 /// does — one `()` entry for the pair, rendered as the pair. Before
531 /// it, Python reported no operator here at all.
532 #[cfg(all(feature = "javascript", feature = "python"))]
533 #[test]
534 fn python_call_parens_pair_like_javascript() {
535 check_ops(LANG::Python, "f(x)", "foo.py", &mut ["()"], &mut ["f", "x"]);
536 check_ops(
537 LANG::Javascript,
538 "f(x)",
539 "foo.js",
540 &mut ["()"],
541 &mut ["f", "x"],
542 );
543 }
544
545 #[cfg(feature = "cpp")]
546 #[test]
547 fn cpp_ops() {
548 check_ops(
549 LANG::Cpp,
550 "int a, b, c;
551 float avg;
552 avg = (a + b + c) / 3;",
553 "foo.c",
554 &mut ["int", "float", "()", "=", "+", "/", ",", ";"],
555 &mut ["a", "b", "c", "avg", "3"],
556 );
557 }
558
559 #[cfg(feature = "cpp")]
560 #[test]
561 fn cpp_function_ops() {
562 check_ops(
563 LANG::Cpp,
564 "main()
565 {
566 int a, b, c, avg;
567 scanf(\"%d %d %d\", &a, &b, &c);
568 avg = (a + b + c) / 3;
569 printf(\"avg = %d\", avg);
570 }",
571 "foo.c",
572 &mut ["()", "{}", "int", "&", "=", "+", "/", ",", ";"],
573 &mut [
574 "main",
575 "a",
576 "b",
577 "c",
578 "avg",
579 "scanf",
580 "\"%d %d %d\"",
581 "3",
582 "printf",
583 "\"avg = %d\"",
584 ],
585 );
586 }
587
588 #[cfg(feature = "rust")]
589 #[test]
590 fn rust_ops() {
591 check_ops(
592 LANG::Rust,
593 "let: usize a = 5; let b: f32 = 7.0; let c: i32 = 3;",
594 "foo.rs",
595 &mut ["let", ":", "usize", "=", ";", "f32", "i32"],
596 &mut ["a", "b", "c", "5", "7.0", "3"],
597 );
598 }
599
600 #[cfg(feature = "rust")]
601 #[test]
602 fn rust_function_ops() {
603 check_ops(
604 LANG::Rust,
605 "fn main() {
606 let a = 5; let b = 5; let c = 5;
607 let avg = (a + b + c) / 3;
608 println!(\"{}\", avg);
609 }",
610 "foo.rs",
611 &mut ["fn", "()", "{}", "let", "=", "+", "/", ";", "!", ","],
612 &mut ["main", "a", "b", "c", "avg", "5", "3", "println", "\"{}\""],
613 );
614 }
615
616 #[cfg(feature = "javascript")]
617 #[test]
618 fn javascript_ops() {
619 check_ops(
620 LANG::Javascript,
621 "var a, b, c, avg;
622 let x = 1;
623 a = 5; b = 5; c = 5;
624 avg = (a + b + c) / 3;
625 console.log(\"{}\", avg);",
626 "foo.js",
627 &mut ["()", "var", "let", "=", "+", "/", ",", ".", ";"],
628 &mut [
629 "a", "b", "c", "avg", "x", "1", "3", "5", "console", "log", "\"{}\"",
630 ],
631 );
632 }
633
634 #[cfg(feature = "javascript")]
635 #[test]
636 fn javascript_function_ops() {
637 check_ops(
638 LANG::Javascript,
639 "function main() {
640 var a, b, c, avg;
641 let x = 1;
642 a = 5; b = 5; c = 5;
643 avg = (a + b + c) / 3;
644 console.log(\"{}\", avg);
645 }",
646 "foo.js",
647 &mut [
648 "function", "()", "{}", "var", "let", "=", "+", "/", ",", ".", ";",
649 ],
650 &mut [
651 "main", "a", "b", "c", "avg", "x", "1", "3", "5", "console", "log", "\"{}\"",
652 ],
653 );
654 }
655
656 #[cfg(feature = "mozjs")]
657 #[test]
658 fn mozjs_ops() {
659 check_ops(
660 LANG::Mozjs,
661 "var a, b, c, avg;
662 let x = 1;
663 a = 5; b = 5; c = 5;
664 avg = (a + b + c) / 3;
665 console.log(\"{}\", avg);",
666 "foo.js",
667 &mut ["()", "var", "let", "=", "+", "/", ",", ".", ";"],
668 &mut [
669 "a", "b", "c", "avg", "x", "1", "3", "5", "console", "log", "\"{}\"",
670 ],
671 );
672 }
673
674 #[cfg(feature = "mozjs")]
675 #[test]
676 fn mozjs_function_ops() {
677 check_ops(
678 LANG::Mozjs,
679 "function main() {
680 var a, b, c, avg;
681 let x = 1;
682 a = 5; b = 5; c = 5;
683 avg = (a + b + c) / 3;
684 console.log(\"{}\", avg);
685 }",
686 "foo.js",
687 &mut [
688 "function", "()", "{}", "var", "let", "=", "+", "/", ",", ".", ";",
689 ],
690 &mut [
691 "main", "a", "b", "c", "avg", "x", "1", "3", "5", "console", "log", "\"{}\"",
692 ],
693 );
694 }
695
696 #[cfg(feature = "typescript")]
697 #[test]
698 fn typescript_ops() {
699 // Issue #1261: the `: string` annotation counts exactly once,
700 // as the text-keyed `string` operator (PredefinedType wrapper)
701 // — symmetric with `: number` / `: boolean`. Under #313 its
702 // `String2` child also emitted a `"string"` operand, so one
703 // source token tallied twice.
704 check_ops(
705 LANG::Typescript,
706 "var a, b, c, avg;
707 let age: number = 32;
708 let name: string = \"John\"; let isUpdated: boolean = true;
709 a = 5; b = 5; c = 5;
710 avg = (a + b + c) / 3;
711 console.log(\"{}\", avg);",
712 "foo.ts",
713 &mut [
714 "()", "var", "let", "string", "number", "boolean", ":", "=", "+", "/", ",", ".",
715 ";",
716 ],
717 &mut [
718 "a",
719 "b",
720 "c",
721 "avg",
722 "age",
723 "name",
724 "isUpdated",
725 "32",
726 "\"John\"",
727 "true",
728 "3",
729 "5",
730 "console",
731 "log",
732 "\"{}\"",
733 ],
734 );
735 }
736
737 #[cfg(feature = "typescript")]
738 #[test]
739 fn typescript_function_ops() {
740 // Issue #1261: see `typescript_ops` — the `string` type keyword
741 // contributes only the primitive-typed operator, never a
742 // `"string"` operand.
743 check_ops(
744 LANG::Typescript,
745 "function main() {
746 var a, b, c, avg;
747 let age: number = 32;
748 let name: string = \"John\"; let isUpdated: boolean = true;
749 a = 5; b = 5; c = 5;
750 avg = (a + b + c) / 3;
751 console.log(\"{}\", avg);
752 }",
753 "foo.ts",
754 &mut [
755 "function", "()", "{}", "var", "let", "string", "number", "boolean", ":", "=", "+",
756 "/", ",", ".", ";",
757 ],
758 &mut [
759 "main",
760 "a",
761 "b",
762 "c",
763 "avg",
764 "age",
765 "name",
766 "isUpdated",
767 "32",
768 "\"John\"",
769 "true",
770 "3",
771 "5",
772 "console",
773 "log",
774 "\"{}\"",
775 ],
776 );
777 }
778
779 #[cfg(feature = "typescript")]
780 #[test]
781 fn tsx_ops() {
782 // Issue #1261: TSX exposes the `: string` type-keyword child as
783 // `String3` (vs. TS's `String2`); like TS, the keyword counts
784 // only as the `string` operator, never as an operand.
785 check_ops(
786 LANG::Tsx,
787 "var a, b, c, avg;
788 let age: number = 32;
789 let name: string = \"John\"; let isUpdated: boolean = true;
790 a = 5; b = 5; c = 5;
791 avg = (a + b + c) / 3;
792 console.log(\"{}\", avg);",
793 "foo.ts",
794 &mut [
795 "()", "var", "let", "string", "number", "boolean", ":", "=", "+", "/", ",", ".",
796 ";",
797 ],
798 &mut [
799 "a",
800 "b",
801 "c",
802 "avg",
803 "age",
804 "name",
805 "isUpdated",
806 "32",
807 "\"John\"",
808 "true",
809 "3",
810 "5",
811 "console",
812 "log",
813 "\"{}\"",
814 ],
815 );
816 }
817
818 #[cfg(feature = "typescript")]
819 #[test]
820 fn tsx_function_ops() {
821 // Issue #1261: see `tsx_ops` — TSX::String3 (type-keyword
822 // `string`) is an operator only, never an operand.
823 check_ops(
824 LANG::Tsx,
825 "function main() {
826 var a, b, c, avg;
827 let age: number = 32;
828 let name: string = \"John\"; let isUpdated: boolean = true;
829 a = 5; b = 5; c = 5;
830 avg = (a + b + c) / 3;
831 console.log(\"{}\", avg);
832 }",
833 "foo.ts",
834 &mut [
835 "function", "()", "{}", "var", "let", "string", "number", "boolean", ":", "=", "+",
836 "/", ",", ".", ";",
837 ],
838 &mut [
839 "main",
840 "a",
841 "b",
842 "c",
843 "avg",
844 "age",
845 "name",
846 "isUpdated",
847 "32",
848 "\"John\"",
849 "true",
850 "3",
851 "5",
852 "console",
853 "log",
854 "\"{}\"",
855 ],
856 );
857 }
858
859 // Issue #453: a `void` return type (a `predefined_type` wrapper over a
860 // `void` token) and an expression `void` (`void 0`) must collapse to a
861 // single distinct `"void"` operator. `check_ops` asserts the exact
862 // operator list, so a duplicate `"void"` — the pre-fix symptom, where
863 // the wrapper keyed `primitive_operators["void"]` and the inner token
864 // keyed `operators[Void]` — trips the assertion. This pins the lesson-4
865 // `n1 == dedupe(ops.operators)` invariant for the two `void` forms in
866 // one file.
867 #[cfg(feature = "typescript")]
868 #[test]
869 fn typescript_void_return_and_expression_single_operator_453() {
870 check_ops(
871 LANG::Typescript,
872 "function f(): void { return void 0; }",
873 "foo.ts",
874 &mut ["function", "()", "{}", ":", "void", "return", ";"],
875 &mut ["f", "0"],
876 );
877 }
878
879 #[cfg(feature = "typescript")]
880 #[test]
881 fn tsx_void_return_and_expression_single_operator_453() {
882 check_ops(
883 LANG::Tsx,
884 "function f(): void { return void 0; }",
885 "foo.tsx",
886 &mut ["function", "()", "{}", ":", "void", "return", ";"],
887 &mut ["f", "0"],
888 );
889 }
890
891 #[cfg(feature = "java")]
892 #[test]
893 fn java_ops() {
894 check_ops(
895 LANG::Java,
896 "public class Main {
897 public static void main(string args[]) {
898 int a, b, c, avg;
899 a = 5; b = 5; c = 5;
900 avg = (a + b + c) / 3;
901 MessageFormat.format(\"{0}\", avg);
902 }
903 }",
904 "foo.java",
905 &mut [
906 "class", "{}", "void", "()", "[]", ",", ".", ";", "int", "=", "+", "/",
907 ],
908 &mut [
909 "Main",
910 "main",
911 // the parameter's type name (#1560)
912 "string",
913 "args",
914 "a",
915 "b",
916 "c",
917 "avg",
918 "5",
919 "3",
920 "MessageFormat",
921 "format",
922 "\"{0}\"",
923 ],
924 );
925 }
926
927 #[cfg(feature = "java")]
928 #[test]
929 fn java_primitive_ops() {
930 check_ops(
931 LANG::Java,
932 "public class Prims {
933 byte a = 1;
934 short b = 2;
935 int c = 3;
936 long d = 4;
937 char e = 'x';
938 float f = 1.0f;
939 double g = 2.0;
940 boolean h = true;
941 boolean i = false;
942 }",
943 "foo.java",
944 // All 8 primitive-type keywords must appear as distinct operators.
945 // true/false appear as operands; `class` is billed since #1552.
946 &mut [
947 "class",
948 "{}",
949 ";",
950 "=",
951 "byte",
952 "short",
953 "int",
954 "long",
955 "char",
956 "float",
957 "double",
958 "boolean_type",
959 ],
960 &mut [
961 "Prims", "a", "b", "c", "d", "e", "f", "g", "h", "i", "1", "2", "3", "4", "'x'",
962 "1.0f", "2.0", "true", "false",
963 ],
964 );
965 }
966
967 /// A `Unit` space must never carry the synthetic `<anonymous>`
968 /// placeholder that the default getter invents for nodes without a
969 /// `name` field. The public docs describe `None` as the
970 /// "name could not be resolved" state, and the metrics-side
971 /// `FuncSpace::new` already special-cases `SpaceKind::Unit` the same
972 /// way; this pins the `Ops::new` mirror so a regression to the old
973 /// `Some("<anonymous>")` initialisation fails here rather than only
974 /// surfacing for a (currently unreachable) non-top-level `Unit`
975 /// space, where `ops_inner`'s top-level override would not rescue it.
976 /// See issue #755.
977 #[cfg(feature = "rust")]
978 #[test]
979 fn unit_space_name_is_none_not_anonymous() {
980 use crate::getter::Getter;
981 use crate::node::Ancestors;
982 use crate::traits::ParserTrait;
983 use crate::{RustCode, RustParser, SpaceKind};
984
985 let code = b"fn f() {}\n";
986 let parser = RustParser::new(code.to_vec(), std::path::Path::new("foo.rs"), None);
987 let root = parser.root();
988 // The Rust `source_file` root is a `Unit` and has no `name`/`type`
989 // field, so the default getter would invent `<anonymous>`.
990 assert_eq!(SpaceKind::Unit, RustCode::get_space_kind(&root));
991
992 let ops = super::Ops::new::<RustCode>(&root, code, Ancestors::unknown(), SpaceKind::Unit);
993 assert_eq!(
994 ops.name, None,
995 "Unit space must preserve name = None, not invent <anonymous>"
996 );
997 }
998
999 /// Issue #789: an `ERROR`-root parse (here Lua partial input) where
1000 /// `metrics()` succeeds must make `ops()` succeed too — the two seams
1001 /// should agree. Before the synthetic-Unit-root mirror in `ops_inner`,
1002 /// `ops()` returned `Err(MetricsError::EmptyRoot)` because the ERROR
1003 /// root is not classified as a function space, so no frame was ever
1004 /// pushed. This pins their agreement: both succeed, and the resulting
1005 /// top-level `Ops` is a `Unit` whose name is the caller-supplied
1006 /// `Source::name` (the intrinsic Unit name stays `None` per #755 until
1007 /// `ops_inner` overrides the top-level name).
1008 #[cfg(feature = "lua")]
1009 #[test]
1010 fn lua_error_root_ops_agrees_with_metrics_789() {
1011 use crate::{MetricsOptions, SpaceKind};
1012
1013 // tree-sitter-lua surfaces an ERROR root for this partial input.
1014 let src = b"function foo(x)\n return x +\n".to_vec();
1015 let name = "partial.lua".to_owned();
1016
1017 let ast = Ast::parse(Source::new(LANG::Lua, &src).with_name(Some(name.clone())))
1018 .expect("lua feature enabled");
1019
1020 // metrics() must succeed (it already wrapped a synthetic Unit root).
1021 let space = ast
1022 .metrics(MetricsOptions::default())
1023 .expect("metrics must yield a top-level space");
1024 assert_eq!(space.kind, SpaceKind::Unit);
1025
1026 // ops() must now succeed in the same case rather than returning
1027 // Err(EmptyRoot).
1028 let ops = ast
1029 .ops()
1030 .expect("ops must agree with metrics and yield a top-level Ops");
1031 assert_eq!(ops.kind, SpaceKind::Unit);
1032 assert_eq!(
1033 ops.name.as_deref(),
1034 Some(name.as_str()),
1035 "top-level Ops name is the caller-supplied Source::name"
1036 );
1037 }
1038
1039 /// Issue #790: `Ops::operands` / `Ops::operators` are the *distinct*
1040 /// (deduplicated) Halstead operand/operator vocabularies (`n2` / `n1`),
1041 /// not every occurrence. Pin the documented dedup semantics: each
1042 /// vector's length equals its unique-element count. The fixture
1043 /// repeats `+`, `;`, and `=` operators and the `a` operand so a
1044 /// regression to non-deduplicated collection would make `len` exceed
1045 /// the unique count.
1046 #[cfg(feature = "rust")]
1047 #[test]
1048 fn ops_vocabularies_are_distinct_790() {
1049 use std::collections::HashSet;
1050
1051 let src = b"fn main() { let a = 1 + 1; let b = a + a; }\n".to_vec();
1052 let ops = Ast::parse(Source::new(LANG::Rust, &src).with_name(Some("foo.rs".to_owned())))
1053 .expect("rust feature enabled")
1054 .ops()
1055 .expect("ops walk must yield a top-level Ops");
1056
1057 let unique_operators: HashSet<&String> = ops.operators.iter().collect();
1058 assert_eq!(
1059 ops.operators.len(),
1060 unique_operators.len(),
1061 "Ops::operators must be the distinct operator vocabulary (n1)"
1062 );
1063
1064 let unique_operands: HashSet<&String> = ops.operands.iter().collect();
1065 assert_eq!(
1066 ops.operands.len(),
1067 unique_operands.len(),
1068 "Ops::operands must be the distinct operand vocabulary (n2)"
1069 );
1070 }
1071
1072 /// Assert that every space in the tree carries sorted vocabularies,
1073 /// and return how many spaces were checked so a caller can prove the
1074 /// walk actually descended.
1075 ///
1076 /// The length floor is what keeps this from going quietly vacuous:
1077 /// `is_sorted` is trivially true for an empty or single-entry
1078 /// vector, so without it a fixture that stopped producing real
1079 /// vocabularies would keep passing while covering nothing.
1080 fn assert_sorted_spaces(ops: &Ops, lang: LANG) -> usize {
1081 /// Smallest vocabulary in which an ordering is observable.
1082 const MIN_OBSERVABLE: usize = 2;
1083
1084 let mut stack = vec![ops];
1085 let mut visited = 0;
1086
1087 while let Some(space) = stack.pop() {
1088 visited += 1;
1089 for (field, values) in [
1090 ("operators", &space.operators),
1091 ("operands", &space.operands),
1092 ] {
1093 assert!(
1094 values.len() >= MIN_OBSERVABLE && values.is_sorted(),
1095 "{lang:?} {field} of space {:?} (@{}) must hold at least \
1096 {MIN_OBSERVABLE} entries and be sorted: {values:?}",
1097 space.name,
1098 space.start_line
1099 );
1100 }
1101 stack.extend(space.spaces.iter());
1102 }
1103
1104 visited
1105 }
1106
1107 /// Every space's vocabularies come back sorted, in every language.
1108 ///
1109 /// The operator vocabulary is the union of two maps — one keyed by
1110 /// token id, one keyed by text, which is where primitive types such
1111 /// as C++ `int` land — and sorting is what interleaves them rather
1112 /// than leaving the second concatenated onto the first (#1091). The
1113 /// `spans_both_maps` pair per case names one entry from each map, so
1114 /// a fixture that stopped exercising the text-keyed map fails here
1115 /// instead of silently narrowing the test's reach.
1116 #[test]
1117 fn ops_vocabularies_are_sorted_1091() {
1118 /// `(language, file name, source, (text-keyed operator,
1119 /// token-id-keyed operator that must sort after it))`.
1120 type Case = (
1121 LANG,
1122 &'static str,
1123 &'static str,
1124 (&'static str, &'static str),
1125 );
1126
1127 let cases: &[Case] = &[
1128 #[cfg(feature = "rust")]
1129 (
1130 LANG::Rust,
1131 "rust.rs",
1132 "fn zeta(quux: u32) -> u32 { let mid = quux + 1; \
1133 let alpha = |beta: u32| beta * mid; alpha(mid) - quux }\n",
1134 ("u32", "|"),
1135 ),
1136 #[cfg(feature = "cpp")]
1137 (
1138 LANG::Cpp,
1139 "cpp.cpp",
1140 "int zeta(int quux) { double mid = quux + 1; \
1141 char alpha = 'z'; return quux - mid + alpha; }\n",
1142 ("int", "return"),
1143 ),
1144 #[cfg(feature = "java")]
1145 (
1146 LANG::Java,
1147 "Java.java",
1148 "class Zeta { int quux(int mid) { long alpha = mid + 1; \
1149 boolean beta = alpha > 2; return beta ? mid : 0; } }\n",
1150 ("long", "return"),
1151 ),
1152 #[cfg(feature = "python")]
1153 (
1154 LANG::Python,
1155 "python.py",
1156 "def zeta(quux):\n mid = quux + 1\n \
1157 def alpha(beta):\n return beta * mid\n return alpha(mid) - quux\n",
1158 // Python has no primitive-type operators; both entries
1159 // come from the token-id map, so this pair only pins the
1160 // ordering, not the interleaving.
1161 ("def", "return"),
1162 ),
1163 #[cfg(feature = "typescript")]
1164 (
1165 LANG::Typescript,
1166 "ts.ts",
1167 "function zeta(quux: number): number { const mid: number = quux + 1; \
1168 const alpha = (beta: number) => beta * mid; return alpha(mid) - quux; }\n",
1169 ("number", "return"),
1170 ),
1171 ];
1172
1173 for (lang, file, source, (from_text_map, sorts_after)) in cases {
1174 let ops = Ast::parse(
1175 Source::new(*lang, source.as_bytes()).with_name(Some((*file).to_owned())),
1176 )
1177 .expect("language feature enabled")
1178 .ops()
1179 .expect("ops walk must yield a top-level Ops");
1180
1181 let position = |needle: &str| {
1182 ops.operators
1183 .iter()
1184 .position(|op| op == needle)
1185 .unwrap_or_else(|| {
1186 panic!(
1187 "{lang:?} operators must contain {needle:?}: {:?}",
1188 ops.operators
1189 )
1190 })
1191 };
1192 assert!(
1193 position(from_text_map) < position(sorts_after),
1194 "{lang:?} must order {from_text_map:?} before {sorts_after:?}: {:?}",
1195 ops.operators
1196 );
1197
1198 assert!(
1199 assert_sorted_spaces(&ops, *lang) > 1,
1200 "{lang:?} sample must nest at least one sub-space"
1201 );
1202 }
1203 }
1204
1205 /// Two parses of the same bytes in one process must agree exactly.
1206 ///
1207 /// `RandomState` bumps its thread-local seed per instance, so the
1208 /// pre-fix code could — and did — order two `HashMap`s built from
1209 /// identical keys differently within a single run. This is the
1210 /// in-process form of the cross-run churn in #1091.
1211 #[test]
1212 #[cfg(feature = "rust")]
1213 fn ops_are_stable_across_repeated_parses_1091() {
1214 use std::fmt::Write as _;
1215
1216 // Enough distinct operands, in enough distinct spaces, that
1217 // agreement by coincidence is not a plausible explanation for a
1218 // pass.
1219 let mut src = String::new();
1220 for i in 0..40 {
1221 writeln!(src, "fn name{i}(arg{i}: u32) -> u32 {{ arg{i} + {i} }}")
1222 .expect("writing to a String cannot fail");
1223 }
1224
1225 let parse = || {
1226 Ast::parse(Source::new(LANG::Rust, src.as_bytes()).with_name(Some("foo.rs".to_owned())))
1227 .expect("rust feature enabled")
1228 .ops()
1229 .expect("ops walk must yield a top-level Ops")
1230 };
1231
1232 let (first, second) = (parse(), parse());
1233 assert!(
1234 first.operands.len() >= 40 && first.spaces.len() >= 40,
1235 "sample must have a wide vocabulary across many spaces, got {} operands \
1236 in {} spaces",
1237 first.operands.len(),
1238 first.spaces.len()
1239 );
1240 // `Ops` has no `PartialEq`, and the nested spaces are the half a
1241 // top-level vector comparison would miss, so compare the whole
1242 // serialized tree.
1243 let render =
1244 |ops: &Ops| serde_json::to_string(&ops.to_wire()).expect("wire Ops serializes to JSON");
1245 assert_eq!(render(&first), render(&second));
1246 }
1247
1248 /// A vocabulary that lost bytes is ordered by its *rendered* text.
1249 ///
1250 /// #1110 moved the sort ahead of the lossy UTF-8 rendering, which is
1251 /// only order-preserving while every key is valid UTF-8. Here it is
1252 /// not: the two string operands are `"\xffA"` and `"\u{fffd}B"`, so
1253 /// by raw bytes the first sorts *after* the second (`0xff` > `0xef`)
1254 /// and by rendered text — both start `U+FFFD`, then `A` before `B` —
1255 /// it sorts before. The rendered order is what the pre-#1110
1256 /// render-then-sort code produced and what `Ops` documents, so
1257 /// dropping the fallback re-sort flips this pair and fails here.
1258 #[test]
1259 #[cfg(feature = "rust")]
1260 fn ops_vocabulary_orders_lossy_entries_by_rendered_text_1110() {
1261 let mut src = b"fn f() { let a = \"".to_vec();
1262 src.push(0xff);
1263 src.extend_from_slice(b"A\"; let b = \"");
1264 src.extend_from_slice(&[0xef, 0xbf, 0xbd]);
1265 src.extend_from_slice(b"B\"; }\n");
1266
1267 let ops = Ast::parse(Source::new(LANG::Rust, &src).with_name(Some("foo.rs".to_owned())))
1268 .expect("rust feature enabled")
1269 .ops()
1270 .expect("ops walk must yield a top-level Ops");
1271
1272 let position = |needle: &str| {
1273 ops.operands
1274 .iter()
1275 .position(|operand| operand == needle)
1276 .unwrap_or_else(|| panic!("operands must contain {needle:?}: {:?}", ops.operands))
1277 };
1278 assert!(
1279 position("\"\u{fffd}A\"") < position("\"\u{fffd}B\""),
1280 "lossy entries must be ordered by rendered text, got {:?}",
1281 ops.operands
1282 );
1283 assert!(
1284 ops.operands.is_sorted(),
1285 "the whole vocabulary must be sorted as rendered, got {:?}",
1286 ops.operands
1287 );
1288 }
1289
1290 /// The walk classifies a space kind once per space, not once per node.
1291 ///
1292 /// Nothing in the output distinguishes the two: the classification
1293 /// only ever reaches `Ops::new`, so running it on every node produces
1294 /// the same tree and merely throws the extra answers away. #1110
1295 /// moved the call inside the `func_space` branch, mirroring
1296 /// `spaces::compute::open_func_space`; the counter is what makes
1297 /// hoisting it back out a failure. The node-count assertion is what
1298 /// makes the counts distinguishable — a fixture whose nodes and
1299 /// spaces were equal in number could not tell the two apart.
1300 // test-lang-gates: hand-written(rust) — the one grammar that
1301 // makes the fixture list non-empty, so the non-vacuity
1302 // assertion cannot fire on a minimal build
1303 #[test]
1304 // Gated on the language that guarantees a non-empty case list, so
1305 // the emptiness assertion below cannot fire on a minimal build.
1306 #[cfg(feature = "rust")]
1307 fn ops_classifies_space_kind_once_per_space_1110() {
1308 let cases: &[(LANG, &str, &str)] = &[
1309 #[cfg(feature = "rust")]
1310 (
1311 LANG::Rust,
1312 "foo.rs",
1313 "fn outer(a: u32) -> u32 { fn inner(b: u32) -> u32 { b + 1 } inner(a) * 2 }\n",
1314 ),
1315 #[cfg(feature = "python")]
1316 (
1317 LANG::Python,
1318 "foo.py",
1319 "def outer(a):\n def inner(b):\n return b + 1\n return inner(a) * 2\n",
1320 ),
1321 #[cfg(feature = "cpp")]
1322 (
1323 LANG::Cpp,
1324 "foo.cpp",
1325 "struct S { int m(int a) { return a + 1; } }; int f(int b) { return b * 2; }\n",
1326 ),
1327 #[cfg(feature = "java")]
1328 (
1329 LANG::Java,
1330 "Foo.java",
1331 "class C { int m(int a) { return a + 1; } int n(int b) { return b * 2; } }\n",
1332 ),
1333 #[cfg(feature = "javascript")]
1334 (
1335 LANG::Javascript,
1336 "foo.js",
1337 "function outer(a) { function inner(b) { return b + 1; } return inner(a) * 2; }\n",
1338 ),
1339 ];
1340 crate::test_support::assert_fixtures_present(cases);
1341
1342 for (lang, file, source) in cases {
1343 let ast = crate::test_support::parse_named(*lang, file, source);
1344
1345 let before = super::space_kind_lookups::observed();
1346 let ops = ast.ops().expect("ops walk must yield a top-level Ops");
1347 let lookups = super::space_kind_lookups::observed() - before;
1348
1349 let mut spaces = 0;
1350 let mut stack = vec![&ops];
1351 while let Some(space) = stack.pop() {
1352 spaces += 1;
1353 stack.extend(space.spaces.iter());
1354 }
1355
1356 let mut nodes = 0;
1357 let mut cursor = vec![ast.as_tree_sitter().root_node()];
1358 while let Some(node) = cursor.pop() {
1359 nodes += 1;
1360 let mut walker = node.walk();
1361 cursor.extend(node.children(&mut walker));
1362 }
1363
1364 assert!(
1365 nodes > spaces * 4,
1366 "{lang:?} fixture must have many more nodes ({nodes}) than spaces ({spaces}) \
1367 for the two counts to be distinguishable"
1368 );
1369 assert_eq!(
1370 lookups, spaces,
1371 "{lang:?} must classify once per space, not once per node ({nodes} nodes)"
1372 );
1373 }
1374 }
1375
1376 /// One flattened space: `(depth, kind, name, start_line, end_line)`.
1377 #[cfg(feature = "elixir")]
1378 type FlatSpace = (usize, crate::SpaceKind, String, usize, usize);
1379
1380 /// Flattens an `Ops` tree in preorder, so a test can pin the whole
1381 /// tree in one `assert_eq!` and see the surrounding spaces when one
1382 /// is wrong.
1383 ///
1384 /// `end_line` is carried as well as `start_line` because a change to
1385 /// the promote predicate can move a space's *extent* without moving
1386 /// its head — a `def` that swallows its sibling would keep the same
1387 /// start line.
1388 #[cfg(feature = "elixir")]
1389 fn flatten(ops: &Ops, depth: usize, out: &mut Vec<FlatSpace>) {
1390 out.push((
1391 depth,
1392 ops.kind,
1393 ops.name.clone().unwrap_or_else(|| "<none>".to_owned()),
1394 ops.start_line,
1395 ops.end_line,
1396 ));
1397 for child in &ops.spaces {
1398 flatten(child, depth + 1, out);
1399 }
1400 }
1401
1402 #[cfg(feature = "elixir")]
1403 fn elixir_ops_tree(source: &str) -> Vec<FlatSpace> {
1404 let ops = crate::test_support::parse_named(LANG::Elixir, "foo.ex", source)
1405 .ops()
1406 .expect("ops walk must yield a top-level Ops");
1407 let mut flat = Vec::new();
1408 flatten(&ops, 0, &mut flat);
1409 flat
1410 }
1411
1412 /// Issue #1130: Elixir's `defmodule` / `def` are `Call` nodes whose
1413 /// target identifier text spells the keyword, so only the
1414 /// source-aware promote predicate can recognise them. Before the
1415 /// fix `ops()` returned the bare file-level `Unit` for this input
1416 /// while `metrics()` returned the full module/function tree.
1417 #[cfg(feature = "elixir")]
1418 #[test]
1419 fn elixir_ops_opens_module_and_function_spaces_1130() {
1420 use crate::SpaceKind::{Class, Function, Unit};
1421
1422 assert_eq!(
1423 elixir_ops_tree("defmodule Foo do\n def bar(x) do\n x + 1\n end\nend\n"),
1424 vec![
1425 (0, Unit, "foo.ex".to_owned(), 1, 5),
1426 (1, Class, "Foo".to_owned(), 1, 5),
1427 (2, Function, "bar".to_owned(), 2, 4),
1428 ],
1429 );
1430 }
1431
1432 /// An `AnonymousFunction` is the one Elixir space the byte-less
1433 /// predicate could already see, so this pins that the source-aware
1434 /// predicate did not lose it — and that it nests under the `def`
1435 /// space rather than being reparented to the file root.
1436 #[cfg(feature = "elixir")]
1437 #[test]
1438 fn elixir_ops_opens_anonymous_function_space() {
1439 use crate::SpaceKind::{Class, Function, Unit};
1440
1441 assert_eq!(
1442 elixir_ops_tree(
1443 "defmodule Foo do\n def bar(list) do\n \
1444 Enum.map(list, fn x -> x * 2 end)\n end\nend\n"
1445 ),
1446 vec![
1447 (0, Unit, "foo.ex".to_owned(), 1, 5),
1448 (1, Class, "Foo".to_owned(), 1, 5),
1449 (2, Function, "bar".to_owned(), 2, 4),
1450 (3, Function, "<anonymous>".to_owned(), 3, 3),
1451 ],
1452 );
1453 }
1454
1455 /// Issue #310: a `def` inside `quote do … end` is a code *template*
1456 /// emitted later by macro expansion, not a declaration of the
1457 /// enclosing module, so it opens no space. This is the case only the
1458 /// source-aware predicate can get right — the byte-less one never
1459 /// saw any `def` at all, so it was accidentally "correct" here while
1460 /// being wrong everywhere else.
1461 #[cfg(feature = "elixir")]
1462 #[test]
1463 fn elixir_ops_skips_def_inside_quote_block_310() {
1464 use crate::SpaceKind::{Class, Function, Unit};
1465
1466 // The quoted `def` heads line 4. Its name resolves to the
1467 // `<anonymous>` placeholder (the head is `unquote(name)`, not a
1468 // literal identifier), so the absence of a fourth entry — not a
1469 // name match — is what pins it out of the tree.
1470 assert_eq!(
1471 elixir_ops_tree(
1472 "defmodule Foo do\n defmacro gen(name) do\n quote do\n \
1473 def unquote(name)(x) do\n x + 1\n end\n end\n end\nend\n",
1474 ),
1475 vec![
1476 (0, Unit, "foo.ex".to_owned(), 1, 9),
1477 (1, Class, "Foo".to_owned(), 1, 9),
1478 (2, Function, "gen".to_owned(), 2, 8),
1479 ],
1480 );
1481 }
1482
1483 /// #1559: a generic list's angle brackets are one `<>` bracket pair,
1484 /// billed on the opener as `()` / `[]` / `{}` are, with the closer
1485 /// billing nothing. Every fixture also spells a comparison `<` and
1486 /// `>`, which must stay their own vocabulary entries: the grammars
1487 /// give the list brackets the comparison operators' kind ids, so the
1488 /// vocabulary is what shows the two apart and `N1` is what shows the
1489 /// closer is not billed.
1490 #[cfg(any(
1491 feature = "cpp",
1492 feature = "csharp",
1493 feature = "groovy",
1494 feature = "java",
1495 feature = "kotlin",
1496 feature = "mozcpp",
1497 feature = "objc",
1498 feature = "rust",
1499 feature = "typescript",
1500 ))]
1501 fn check_generic_pair(lang: LANG, file: &str, source: &str, operators: &[&str], n1: [u64; 2]) {
1502 let ast = crate::test_support::parse_named(lang, file, source);
1503 let ops = ast.ops().expect("ops walk must yield a top-level Ops");
1504 let mut expected = operators.to_vec();
1505 expected.sort_unstable();
1506 assert_eq!(ops.operators, expected, "{lang:?} vocabulary");
1507
1508 let space = ast
1509 .metrics(crate::MetricsOptions::default().with_only(&[crate::Metric::Halstead]))
1510 .expect("metrics walk");
1511 let halstead = &space.metrics.halstead;
1512 assert_eq!(
1513 [halstead.unique_operators(), halstead.total_operators()],
1514 n1,
1515 "{lang:?} [n1, N1]"
1516 );
1517 }
1518
1519 // expected: `<>` three times (one parameter list, two nested
1520 // argument lists), never `>`; `<` and `>` once each, the comparisons.
1521 #[cfg(feature = "rust")]
1522 #[test]
1523 fn rust_generic_angles_are_one_pair_1559() {
1524 check_generic_pair(
1525 LANG::Rust,
1526 "foo.rs",
1527 "fn f<T>(v: Vec<Vec<T>>) -> bool { a < b && c > d }\n",
1528 &["fn", "<>", "()", ":", "->", "bool", "{}", "<", "&&", ">"],
1529 [10, 12],
1530 );
1531 // A turbofish and a higher-ranked `for<'a>` (`for_lifetimes`)
1532 // are lists too, `<>` twice (`fn` is twice as well); the `>>` shift
1533 // is its own operator.
1534 check_generic_pair(
1535 LANG::Rust,
1536 "foo.rs",
1537 "fn g(f: for<'a> fn(&'a u8)) { h::<u8>(x >> 1); }\n",
1538 &[
1539 "fn", "()", ":", "for", "<>", "&", "u8", "{}", "::", ">>", ";",
1540 ],
1541 [11, 16],
1542 );
1543 // A precise-capturing `use<T>` (`use_bounds`) is a list as well.
1544 check_generic_pair(
1545 LANG::Rust,
1546 "foo.rs",
1547 "fn u<T>(x: T) -> impl Sized + use<T> { x }\n",
1548 &["fn", "<>", "()", ":", "->", "impl", "+", "use", "{}"],
1549 [9, 10],
1550 );
1551 }
1552
1553 // expected: `<>` three times, `;` and `{}` twice each.
1554 #[cfg(feature = "java")]
1555 #[test]
1556 fn java_generic_angles_are_one_pair_1559() {
1557 check_generic_pair(
1558 LANG::Java,
1559 "foo.java",
1560 "class A<T> { List<List<T>> v; boolean m() { return a < b && c > d; } }\n",
1561 &[
1562 "class",
1563 "<>",
1564 ";",
1565 "{}",
1566 "boolean_type",
1567 "()",
1568 "return",
1569 "<",
1570 "&&",
1571 ">",
1572 ],
1573 [10, 14],
1574 );
1575 }
1576
1577 // expected: as Java, with `bool` for `boolean_type`.
1578 #[cfg(feature = "csharp")]
1579 #[test]
1580 fn csharp_generic_angles_are_one_pair_1559() {
1581 check_generic_pair(
1582 LANG::Csharp,
1583 "foo.cs",
1584 "class A<T> { List<List<T>> v; bool M() { return a < b && c > d; } }\n",
1585 &[
1586 "class", "<>", ";", "{}", "bool", "()", "return", "<", "&&", ">",
1587 ],
1588 [10, 14],
1589 );
1590 }
1591
1592 // expected: `<>` three times, `:` twice. TSX shares the getter macro
1593 // and the same arm, so it is checked on the same input.
1594 #[cfg(feature = "typescript")]
1595 #[test]
1596 fn typescript_generic_angles_are_one_pair_1559() {
1597 const SOURCE: &str =
1598 "function f<T>(v: Array<Array<T>>): boolean { return a < b && c > d; }\n";
1599 const OPERATORS: &[&str] = &[
1600 "function", "<>", "()", ":", "boolean", "{}", "return", "<", "&&", ">", ";",
1601 ];
1602 check_generic_pair(LANG::Typescript, "foo.ts", SOURCE, OPERATORS, [11, 14]);
1603 check_generic_pair(LANG::Tsx, "foo.tsx", SOURCE, OPERATORS, [11, 14]);
1604 }
1605
1606 // expected: `<>` three times (the `template` parameter list and two
1607 // nested argument lists), `::` twice. The closer of the outer list
1608 // is spelled `>>` and still parses as two `>` tokens.
1609 #[cfg(feature = "cpp")]
1610 #[test]
1611 fn cpp_generic_angles_are_one_pair_1559() {
1612 check_generic_pair(
1613 LANG::Cpp,
1614 "foo.cpp",
1615 "template <class T> bool f(std::vector<std::vector<T>> v) { return a < b && c > d; }\n",
1616 &[
1617 "template", "<>", "class", "bool", "()", "::", "{}", "return", "<", "&&", ">", ";",
1618 ],
1619 [12, 15],
1620 );
1621 }
1622
1623 // expected: the Cpp row, through the Mozcpp grammar, which owns no
1624 // file extension and so has no corpus coverage of its own.
1625 #[cfg(feature = "mozcpp")]
1626 #[test]
1627 fn mozcpp_generic_angles_are_one_pair_1559() {
1628 check_generic_pair(
1629 LANG::Mozcpp,
1630 "foo.cpp",
1631 "template <class T> bool f(std::vector<std::vector<T>> v) { return a < b && c > d; }\n",
1632 &[
1633 "template", "<>", "class", "bool", "()", "::", "{}", "return", "<", "&&", ">", ";",
1634 ],
1635 [12, 15],
1636 );
1637 }
1638
1639 // expected: `<>` three times, `:` twice.
1640 #[cfg(feature = "kotlin")]
1641 #[test]
1642 fn kotlin_generic_angles_are_one_pair_1559() {
1643 check_generic_pair(
1644 LANG::Kotlin,
1645 "foo.kt",
1646 "fun <T> f(v: List<List<T>>): Boolean { return a < b && c > d }\n",
1647 &["fun", "<>", "()", ":", "{}", "return", "<", "&&", ">"],
1648 [9, 12],
1649 );
1650 }
1651
1652 // expected: `<>` four times — Groovy spells a method's parameter
1653 // list as its own `method_type_parameters` node.
1654 #[cfg(feature = "groovy")]
1655 #[test]
1656 fn groovy_generic_angles_are_one_pair_1559() {
1657 check_generic_pair(
1658 LANG::Groovy,
1659 "foo.groovy",
1660 "class A<T> { List<List<T>> v; def <U> boolean m() { return a < b && c > d } }\n",
1661 &[
1662 "class", "<>", ";", "{}", "def", "()", "return", "<", "&&", ">",
1663 ],
1664 [10, 14],
1665 );
1666 }
1667
1668 // expected: `<>` five times, once per list — `parameterized_arguments`
1669 // (`: NSObject <NSCopying>`), `protocol_reference_list`
1670 // (`id<NSCopying>`), two nested `generic_specifier`s, and the
1671 // `argument_list` type form `g(NSArray<NSString *>)` — and never `>`.
1672 // `*` four times, `()` and `||` twice; `@end` and `id` bill nothing.
1673 // Without the pair, `<` and `>` would each bill six times: [10, 25].
1674 // expected: the spaced twin `g(a < b, c > d)`, two comparisons.
1675 // Unspaced, tree-sitter-objc reads the arguments as an
1676 // `argument_list`'s `Type<…>` form with an `ERROR` after the `>`, and
1677 // they must keep billing `<` and `>`; the clean `g(NSArray<NSString *>)`
1678 // stays one pair.
1679 #[cfg(feature = "objc")]
1680 #[test]
1681 fn objc_misparsed_comparisons_stay_comparisons_1559() {
1682 const COMPARISONS: &[&str] = &["()", ",", ";", "<", ">", "int", "return", "void", "{}"];
1683 check_generic_pair(
1684 LANG::Objc,
1685 "foo.m",
1686 "int f(void) { return g(a < b, c > d); }\n",
1687 COMPARISONS,
1688 [9, 10],
1689 );
1690 check_generic_pair(
1691 LANG::Objc,
1692 "foo.m",
1693 "int f(void) { return g(a<b, c>d); }\n",
1694 COMPARISONS,
1695 [9, 10],
1696 );
1697 check_generic_pair(
1698 LANG::Objc,
1699 "foo.m",
1700 "int f(void) { return g(NSArray<NSString *>); }\n",
1701 &["()", "*", ";", "<>", "int", "return", "void", "{}"],
1702 [8, 9],
1703 );
1704 }
1705
1706 // expected: the reading before #1559, which billed the two
1707 // comparisons as `<` and `>`. By the C# specification the identifier
1708 // after the `>` makes `F(a < b, c > d)` two arguments, though the
1709 // grammar parses a declaration typed `a<b, c>`. An `out` declaration
1710 // is a real generic and stays one pair.
1711 #[cfg(feature = "csharp")]
1712 #[test]
1713 fn csharp_misparsed_comparisons_stay_comparisons_1559() {
1714 check_generic_pair(
1715 LANG::Csharp,
1716 "foo.cs",
1717 "class A { void M() { F(a < b, c > d); } }\n",
1718 &["()", ",", ";", "<", ">", "class", "void", "{}"],
1719 [8, 10],
1720 );
1721 // A member access before the `<` wraps the generic name in a
1722 // `qualified_name`; the comparisons are the same.
1723 check_generic_pair(
1724 LANG::Csharp,
1725 "foo.cs",
1726 "class A { void M() { F(x.a < b, c > d); } }\n",
1727 &["()", ",", ".", ";", "<", ">", "class", "void", "{}"],
1728 [9, 11],
1729 );
1730 check_generic_pair(
1731 LANG::Csharp,
1732 "foo.cs",
1733 "class A { void M() { F(out List<int> x); } }\n",
1734 &["()", ";", "<>", "class", "int", "out", "void", "{}"],
1735 [8, 10],
1736 );
1737 // A deconstruction's declarations are `argument`s of a
1738 // `tuple_expression`, not of an argument list: a real generic.
1739 check_generic_pair(
1740 LANG::Csharp,
1741 "foo.cs",
1742 "class A { void M() { (List<int> p, int q) = (1, 2); } }\n",
1743 &["()", ",", ";", "<>", "=", "class", "int", "void", "{}"],
1744 [9, 14],
1745 );
1746 // A using alias's generic name sits two levels under the root, too
1747 // shallow for the argument-list shape.
1748 check_generic_pair(
1749 LANG::Csharp,
1750 "foo.cs",
1751 "using X = List<int>;\n",
1752 &[";", "<>", "=", "int", "using"],
1753 [5, 5],
1754 );
1755 }
1756
1757 #[cfg(feature = "objc")]
1758 #[test]
1759 fn objc_generic_angles_are_one_pair_1559() {
1760 check_generic_pair(
1761 LANG::Objc,
1762 "foo.m",
1763 "@interface A : NSObject <NSCopying>\n@end\n\
1764 id<NSCopying> f(NSArray<NSArray<NSString *> *> *v) \
1765 { return g(NSArray<NSString *>) || a < b || c > d; }\n",
1766 &[
1767 "@interface",
1768 ":",
1769 "<>",
1770 "()",
1771 "*",
1772 "{}",
1773 "return",
1774 "||",
1775 "<",
1776 ">",
1777 ";",
1778 ],
1779 [11, 20],
1780 );
1781 }
1782}