Skip to main content

big_code_analysis/
ops.rs

1// Per-language metric and AST modules deliberately consume the macro-
2// generated tree-sitter token enums via `use crate::*` and `use Foo::*`
3// inside match expressions — explicit imports would list dozens of
4// variants per arm and obscure the per-language token sets that are the
5// point of these files. Allowed at the module level rather than per
6// function so the per-language impl blocks stay readable.
7#![allow(clippy::wildcard_imports, clippy::enum_glob_use)]
8
9use std::borrow::Cow;
10
11use crate::checker::Checker;
12use crate::error::MetricsError;
13use crate::getter::Getter;
14use crate::node::{Ancestors, Node};
15use crate::spaces::{SpaceKind, line_span, push_children};
16
17use crate::halstead::{Halstead, HalsteadMaps};
18
19use crate::MetricSuite;
20
21/// All operands and operators of a space.
22#[derive(Debug, Clone)]
23pub struct Ops {
24    /// The name of a function space.
25    ///
26    /// For the top-level (file-level) `Ops` the value is whatever
27    /// `Source::name` the caller supplied to the [`crate::Ast::ops`]
28    /// seam — `Some` or `None`.
29    ///
30    /// For nested spaces, `None` means an error occurred in parsing the
31    /// name of the function space from the AST.
32    pub name: Option<String>,
33    /// `true` when [`Ops::name`] was produced by lossy conversion (the
34    /// original path contained non-UTF-8 bytes and was rendered using
35    /// U+FFFD replacement characters). The explicit-name
36    /// [`crate::Ast::ops`] seam never sets it, since a caller-supplied
37    /// `String` name is UTF-8 by construction, so it is always `false`
38    /// in current code paths. Retained as a wire field for forward
39    /// compatibility; skipped from JSON output when `false` so existing
40    /// schemas keep their shape.
41    pub name_was_lossy: bool,
42    /// The first line of a function space.
43    pub start_line: usize,
44    /// The last line of a function space.
45    pub end_line: usize,
46    /// The space kind.
47    pub kind: SpaceKind,
48    /// All subspaces contained in a function space.
49    pub spaces: Vec<Ops>,
50    /// The **distinct** operands of a space — the deduplicated Halstead
51    /// operand vocabulary (`n2`), one entry per unique operand, not every
52    /// occurrence. Sorted in byte-lexicographic order, so the same input
53    /// always yields the same sequence (#1091).
54    pub operands: Vec<String>,
55    /// The **distinct** operators of a space — the deduplicated Halstead
56    /// operator vocabulary (`n1`), one entry per unique operator, not
57    /// every occurrence. Sorted in byte-lexicographic order, so the same
58    /// input always yields the same sequence (#1091).
59    pub operators: Vec<String>,
60}
61
62// Space nesting is caller-controlled, so the compiler-generated `Drop`
63// glue would recurse once per level and abort the process on a deep tree
64// (#1056). See [`crate::recursion`].
65crate::recursion::impl_iterative_drop!(Ops, spaces);
66
67impl Ops {
68    /// Project this tree into its [`crate::wire::Ops`] form — the
69    /// plain, `Deserialize`-capable record that defines the serialized
70    /// shape.
71    #[must_use]
72    pub fn to_wire(&self) -> crate::wire::Ops {
73        crate::wire::Ops::from(self)
74    }
75
76    fn new<'a, T: Getter>(
77        node: &Node<'a>,
78        code: &[u8],
79        ancestors: Ancestors<'a, '_>,
80        kind: SpaceKind,
81    ) -> Self {
82        let (start_position, end_position) = line_span(node, kind);
83        // The top-level Unit's name is overwritten by `ops_inner` with the
84        // caller-supplied name before returning, so computing it here is
85        // wasted work. Non-top-level Unit spaces have no resolvable name, so
86        // leaving `None` matches the documented "could not be resolved"
87        // semantics rather than inventing the `<anonymous>` placeholder the
88        // default getter returns. Other kinds keep the AST-derived name.
89        // Mirrors the `SpaceKind::Unit` handling in `FuncSpace::new`.
90        let name = (kind != SpaceKind::Unit)
91            .then(|| T::get_func_space_name(node, code, ancestors).map(str::to_owned))
92            .flatten();
93        Self {
94            name,
95            name_was_lossy: false,
96            spaces: Vec::new(),
97            kind,
98            start_line: start_position,
99            end_line: end_position,
100            operators: Vec::new(),
101            operands: Vec::new(),
102        }
103    }
104}
105
106#[derive(Debug, Clone)]
107struct State<'a> {
108    ops: Ops,
109    halstead_maps: HalsteadMaps<'a>,
110}
111
112/// Pushes a synthetic `Unit` root onto the state stack when the grammar
113/// hands us a non-`Unit` root.
114///
115/// Mirrors [`crate::spaces::push_synthetic_unit_root`] on the metrics
116/// seam: some grammars (e.g. tree-sitter-lua / tree-sitter-mozcpp on
117/// unparseable input) return an `ERROR` root that is not classified as a
118/// function space, so without this push the walk would never open a
119/// frame and `ops_inner` would return [`MetricsError::EmptyRoot`] for an
120/// input where `metrics()` succeeds (issue #789). A `Unit` root needs no
121/// wrapper, so nothing is pushed in that case.
122fn push_synthetic_unit_root<T: MetricSuite>(
123    state_stack: &mut Vec<State>,
124    node: &Node,
125    code: &[u8],
126) {
127    // `Ancestors::unknown()`: `node` is the tree root here, so it has
128    // no ancestors to hand over either way.
129    if T::Getter::get_space_kind_with_code(node, code, Ancestors::unknown()) != SpaceKind::Unit {
130        state_stack.push(State {
131            ops: Ops::new::<T::Getter>(node, code, Ancestors::unknown(), SpaceKind::Unit),
132            halstead_maps: HalsteadMaps::new(),
133        });
134    }
135}
136
137// Space-kind classifications `ops_inner` has performed on this thread.
138//
139// The classification only ever reaches `Ops::new`, so running it on a
140// node that opens no space is work thrown away — and nothing about the
141// walk's *output* can tell the two apart. The counter is the
142// observable: it reads one per space after the lookup was moved inside
143// the `func_space` branch, and one per *node* before, which is what
144// makes hoisting it back out a test failure rather than a silent
145// regression (#1110).
146crate::observation::counter!(space_kind_lookups);
147
148/// Classifies a node that is about to open a function space.
149///
150/// Classification happens after the decision that a space opens, the
151/// same way [`crate::spaces::compute`]'s `open_func_space` does it
152/// (#522), and through the same source-aware classifier, so a space
153/// both seams open carries the same [`SpaceKind`] in either walk.
154/// The `_with_code` variant is what lets Elixir's macro-shaped
155/// `defmodule` / `def` declarations — plain `Call` nodes distinguished
156/// only by their target identifier text — come back as `Class` /
157/// `Function` rather than `Unknown` (#275, #1130).
158///
159/// The lookup is a per-language `match` on the node's kind for most
160/// grammars, but C#'s reaches a child scan for a bodied indexer or
161/// property and Elixir's reads the `Call` target text and scans the
162/// ancestor chain for an enclosing `quote` block, so it is not free on
163/// every node either.
164fn classify_space_kind<'a, T: MetricSuite>(
165    node: &Node<'a>,
166    code: &[u8],
167    ancestors: Ancestors<'a, '_>,
168) -> SpaceKind {
169    space_kind_lookups::record();
170    T::Getter::get_space_kind_with_code(node, code, ancestors)
171}
172
173/// Render a space's vocabulary: byte-lexicographically ordered, one
174/// owned `String` per distinct entry.
175///
176/// # Order
177///
178/// `HashMap`'s hasher is randomly seeded per instance, so key order
179/// differs between two runs — and even between two parses in one
180/// process. Without a canonical order the same input renders and
181/// serializes differently every time, which makes `bca ops` output
182/// undiffable and unusable as a cache key (#1091).
183///
184/// Byte-lexicographic, not first-appearance, order: `finalize` merges
185/// a child space's maps into its parent, so an insertion-ordered map
186/// would give the parent "what it saw directly, then whatever each
187/// child contributed" — a walk artifact that shifts when nesting
188/// changes. Sorting is also stable across platforms and hasher
189/// versions, which insertion order via a different map type is not.
190///
191/// # Why the sort runs before the `String`s exist
192///
193/// Every ancestor of a space re-renders that space's whole vocabulary:
194/// `finalize` merges each child's Halstead maps into its parent, so a
195/// parent's key set is a superset of every descendant's and an entry
196/// nested `D` spaces deep is rendered `D + 1` times. Sorting the
197/// borrowed keys first makes each swap a fat pointer rather than a
198/// 24-byte `String`, and — because the `String`s are then allocated in
199/// output order — it hands every later pass over them (the wire
200/// projection, serialization, the `dump_ops` tree) a heap laid out in
201/// the order it reads. Issue #1110 measured both effects.
202///
203/// # Non-UTF-8 keys
204///
205/// Tree-sitter sources are expected to be valid UTF-8; non-UTF-8 bytes
206/// are replaced with the Unicode replacement character to keep the entry
207/// visible (rather than silently dropping it or using a sentinel string
208/// that could collide with a real identifier). That rendering is not
209/// order-preserving — `b"\xffA"` sorts after `"\u{fffd}B"` by raw bytes
210/// and before it once both are rendered — so a vocabulary that actually
211/// lost bytes is re-sorted on the rendered text, which is the order the
212/// pre-#1110 render-then-sort code produced. Valid UTF-8, which is every
213/// key in practice, takes the single sort.
214fn sorted_vocabulary(mut keys: Vec<&[u8]>) -> Vec<String> {
215    keys.sort_unstable();
216    let mut lossy = false;
217    let mut rendered: Vec<String> = keys
218        .into_iter()
219        .map(|key| match String::from_utf8_lossy(key) {
220            Cow::Borrowed(text) => text.to_owned(),
221            Cow::Owned(text) => {
222                lossy = true;
223                text
224            }
225        })
226        .collect();
227    if lossy {
228        rendered.sort_unstable();
229    }
230    rendered
231}
232
233fn compute_operators_and_operands<T: MetricSuite>(state: &mut State) {
234    let maps = &state.halstead_maps;
235
236    // Primitive-type operators live in a second map (keyed by text rather
237    // than by token id), so the operator vocabulary is the concatenation
238    // of both key sets.
239    let operators = maps
240        .operators
241        .keys()
242        .map(|k| T::Getter::get_operator_id_as_str(*k).as_bytes())
243        .chain(maps.primitive_operators.keys().copied())
244        .collect();
245
246    state.ops.operators = sorted_vocabulary(operators);
247    state.ops.operands = sorted_vocabulary(maps.operands.keys().copied().collect());
248}
249
250/// Close up to `diff_level` open spaces, folding each into its parent.
251///
252/// Only the states this pops get their vocabularies computed. The
253/// bottom state is never popped here, so the root's vocabulary is built
254/// once by [`ops_inner`] after the final drain — computing it on every
255/// call would rebuild (and, since #1091, re-sort) the whole file's
256/// vocabulary once per level-drop in the walk, and every result but the
257/// last would be overwritten.
258fn finalize<T: MetricSuite>(state_stack: &mut Vec<State>, diff_level: usize) {
259    for _ in 0..diff_level {
260        if state_stack.len() < 2 {
261            break;
262        }
263        let mut state = state_stack
264            .pop()
265            .expect("state_stack verified to have len >= 2");
266        let last_state = state_stack
267            .last_mut()
268            .expect("state_stack verified to have len >= 1 after pop");
269
270        // Populate the child's ops from its HalsteadMaps before
271        // recording it as a sub-space of the parent.
272        compute_operators_and_operands::<T>(&mut state);
273
274        // Merge child's Halstead maps into parent and record child space.
275        last_state.halstead_maps.merge(&state.halstead_maps);
276        last_state.ops.spaces.push(state.ops);
277    }
278}
279
280/// Context the ops walk carries down the tree alongside each node.
281///
282/// A named pair rather than a `(usize, usize)`: the two counts advance
283/// on different events and swapping them is silent — `level` only moves
284/// at space boundaries while `depth` counts every AST step. Mirrors
285/// [`crate::spaces::compute`]'s `Walk`, minus the comment flag this walk
286/// has no use for.
287#[derive(Clone, Copy)]
288struct Walk {
289    /// Nesting level, used to close op-spaces on the way back up.
290    level: usize,
291    /// AST depth — the number of ancestors this node has, so the root
292    /// sits at `0`. Indexes the ancestor chain the walk maintains.
293    depth: usize,
294}
295
296/// Explicit-name core of the operator/operand walk backing the
297/// [`crate::Ast::ops`] `Source`-based seam. The top-level [`Ops::name`]
298/// is whatever the caller passes in `name`; `name_was_lossy` is left at
299/// its `false` default because an explicit `String` name is never lossy.
300/// Mirrors [`crate::spaces::metrics_inner`].
301pub(crate) fn ops_inner<T: MetricSuite>(
302    parser: &T,
303    name: Option<String>,
304) -> Result<Ops, MetricsError> {
305    let code = parser.code();
306    let node = parser.root();
307    let mut cursor = node.cursor();
308    let mut stack = Vec::new();
309    // Ancestor chain of the node currently being visited, root first,
310    // maintained by the same truncate/push rule as
311    // `spaces::compute::metrics_inner` (#1084).
312    let mut chain: Vec<Node<'_>> = Vec::new();
313    let mut state_stack: Vec<State> = Vec::new();
314    let mut last_level = 0;
315
316    // Mirror `metrics_inner`: wrap a non-`Unit` (e.g. `ERROR`) root in a
317    // synthetic `Unit` frame so the walk always has a frame to populate.
318    // Without this, an `ERROR`-root parse drains the state stack and
319    // `ops_inner` returns `EmptyRoot` for inputs where `metrics()`
320    // succeeds (issue #789).
321    push_synthetic_unit_root::<T>(&mut state_stack, &node, code);
322
323    stack.push((node, Walk { level: 0, depth: 0 }));
324
325    while let Some((node, Walk { level, depth })) = stack.pop() {
326        chain.truncate(depth);
327
328        if level < last_level {
329            finalize::<T>(&mut state_stack, last_level - level);
330            last_level = level;
331        }
332
333        let ancestors = Ancestors::checked(&chain, &node);
334
335        // Same predicate `spaces::compute::metrics_inner` opens on, so
336        // the two walks agree on which nodes become spaces. The
337        // byte-less `is_func || is_func_space` this replaced could not
338        // see Elixir's macro-shaped declarations, which are `Call`
339        // nodes identified by their target text, so `bca ops` opened no
340        // space for a `defmodule` / `def` / `defp` / `defmacro` — only
341        // for `Source` and an explicit `fn … -> … end` (#1130).
342        let func_space = T::Checker::promotes_to_func_space_with_code(&node, code, ancestors);
343
344        let new_level = if func_space {
345            let kind = classify_space_kind::<T>(&node, code, ancestors);
346            let state = State {
347                ops: Ops::new::<T::Getter>(&node, code, ancestors, kind),
348                halstead_maps: HalsteadMaps::new(),
349            };
350            state_stack.push(state);
351            last_level = level + 1;
352            last_level
353        } else {
354            level
355        };
356
357        if let Some(state) = state_stack.last_mut() {
358            T::Halstead::compute(&node, code, ancestors, &mut state.halstead_maps);
359        }
360
361        chain.push(node);
362
363        // Shared with `metrics_inner` (issue #969): `push_children` is
364        // State-independent — it only moves the cursor over child nodes —
365        // so unlike the local `finalize` / `push_synthetic_unit_root`
366        // mirrors (which differ by `State` payload) it is reused directly
367        // rather than duplicated. The source-order-then-reverse ordering
368        // it encapsulates is load-bearing for suppression attribution.
369        // The returned child slice is only useful to `metrics_inner`,
370        // which seeds their cognitive nesting; `ops` just walks them.
371        push_children(
372            &mut cursor,
373            &node,
374            Walk {
375                level: new_level,
376                depth: depth + 1,
377            },
378            &mut stack,
379        );
380    }
381
382    finalize::<T>(&mut state_stack, usize::MAX);
383
384    // Reserved error path: `MetricsError::EmptyRoot` is unreachable
385    // today because the synthetic Unit push above (and every supported
386    // language's root being recognised as a `func_space`) keeps the
387    // state stack non-empty for every input, including ERROR-root,
388    // empty, whitespace-only, and comment-only sources — matching
389    // `metrics_inner`. The `ok_or` is retained so a future walker change
390    // that legitimately drains the stack surfaces a distinct error
391    // variant rather than a bare `None`. See `MetricsError::EmptyRoot`
392    // for the matching variant doc.
393    let mut state = state_stack.pop().ok_or(MetricsError::EmptyRoot)?;
394    // The root is the one state `finalize` never pops, so its vocabulary
395    // is built here — once, from the fully-merged maps.
396    compute_operators_and_operands::<T>(&mut state);
397    state.ops.name = name;
398    Ok(state.ops)
399}
400
401#[cfg(test)]
402#[allow(
403    clippy::float_cmp,
404    clippy::cast_precision_loss,
405    clippy::cast_possible_truncation,
406    clippy::cast_sign_loss,
407    clippy::similar_names,
408    clippy::doc_markdown,
409    clippy::needless_raw_string_hashes,
410    clippy::too_many_lines
411)]
412mod tests {
413    use super::Ops;
414    use crate::{Ast, LANG, Source};
415
416    #[cfg(any(
417        feature = "cpp",
418        feature = "java",
419        feature = "javascript",
420        feature = "mozjs",
421        feature = "perl",
422        feature = "python",
423        feature = "rust",
424        feature = "typescript",
425    ))]
426    #[inline]
427    fn check_ops(
428        lang: LANG,
429        source: &str,
430        file: &str,
431        correct_operators: &mut [&str],
432        correct_operands: &mut [&str],
433    ) {
434        let mut trimmed_bytes = source.trim_end().trim_matches('\n').as_bytes().to_vec();
435        trimmed_bytes.push(b'\n');
436        let ops = Ast::parse(Source::new(lang, &trimmed_bytes).with_name(Some(file.to_owned())))
437            .expect("language feature enabled")
438            .ops()
439            .expect("ops walk must yield a top-level Ops");
440
441        let operators_str: Vec<&str> = ops.operators.iter().map(AsRef::as_ref).collect();
442        let operands_str: Vec<&str> = ops.operands.iter().map(AsRef::as_ref).collect();
443
444        // Only the *expectations* are sorted here: `Ops` is documented to
445        // come back byte-lexicographically ordered (#1091), so comparing
446        // against a sorted expectation without re-sorting the actual value
447        // makes every `check_ops` caller an ordering regression test for
448        // its language.
449        correct_operators.sort_unstable();
450        assert_eq!(&operators_str[..], correct_operators);
451
452        correct_operands.sort_unstable();
453        assert_eq!(&operands_str[..], correct_operands);
454    }
455
456    #[cfg(feature = "python")]
457    #[test]
458    fn python_ops() {
459        check_ops(
460            LANG::Python,
461            "if True:
462                 a = 1 + 2",
463            "foo.py",
464            &mut ["if", ":", "=", "+"],
465            &mut ["True", "a", "1", "2"],
466        );
467    }
468
469    #[cfg(feature = "perl")]
470    #[test]
471    fn perl_pattern_operations_render_as_source_spellings() {
472        // #1314 classifies `s///` and `tr///` as Halstead operators.
473        // They are *named* nodes rather than punctuation tokens, so the
474        // `get_operator!` macro's fallback would render each kind's own
475        // name — `substitution_pattern_s`, `transliteration_tr_or_y` —
476        // into `bca ops`, which reads as a bug rather than as Perl.
477        // `PerlCode::get_operator_id_as_str` is hand-written to map
478        // them, and this is what pins that mapping: the counts alone
479        // cannot see it.
480        //
481        // `y///` is a synonym of `tr///` and shares one kind, so the
482        // two source spellings collapse to a single `tr///` entry —
483        // asserted here by the *absence* of a `y///` row in an
484        // exhaustive expectation, not by a negative assertion.
485        //
486        // The trailing `/pat/` keeps a pattern *value* in the fixture,
487        // so the test also shows the split: the operation spellings are
488        // operators while the value is an operand.
489        check_ops(
490            LANG::Perl,
491            "$s =~ s/a/b/;\n$s =~ tr/c/d/;\n$s =~ y/e/f/;\n$t = /pat/;",
492            "foo.pl",
493            &mut ["$", ";", "=", "=~", "s///", "tr///"],
494            &mut ["$s", "$t", "/pat/"],
495        );
496    }
497
498    /// The fat comma is a named node, so the generic fallback rendered
499    /// its kind name, `fat_comma`; it lists as the `=>` written.
500    #[cfg(feature = "perl")]
501    #[test]
502    fn perl_fat_comma_renders_as_written() {
503        check_ops(
504            LANG::Perl,
505            "my %h = (a => 1);",
506            "foo.pl",
507            &mut ["()", ";", "=", "=>", "my"],
508            &mut ["%h", "1", "a"],
509        );
510    }
511
512    #[cfg(feature = "python")]
513    #[test]
514    fn python_function_ops() {
515        check_ops(
516            LANG::Python,
517            "def foo():
518                 def bar():
519                     def toto():
520                        a = 1 + 1
521                     b = 2 + a
522                 c = 3 + 3",
523            "foo.py",
524            &mut ["def", "()", ":", "=", "+"],
525            &mut ["foo", "bar", "toto", "a", "b", "c", "1", "2", "3"],
526        );
527    }
528
529    /// #1486: Python bills a call's parentheses the way the JS family
530    /// does — one `()` entry for the pair, rendered as the pair. Before
531    /// it, Python reported no operator here at all.
532    #[cfg(all(feature = "javascript", feature = "python"))]
533    #[test]
534    fn python_call_parens_pair_like_javascript() {
535        check_ops(LANG::Python, "f(x)", "foo.py", &mut ["()"], &mut ["f", "x"]);
536        check_ops(
537            LANG::Javascript,
538            "f(x)",
539            "foo.js",
540            &mut ["()"],
541            &mut ["f", "x"],
542        );
543    }
544
545    #[cfg(feature = "cpp")]
546    #[test]
547    fn cpp_ops() {
548        check_ops(
549            LANG::Cpp,
550            "int a, b, c;
551             float avg;
552             avg = (a + b + c) / 3;",
553            "foo.c",
554            &mut ["int", "float", "()", "=", "+", "/", ",", ";"],
555            &mut ["a", "b", "c", "avg", "3"],
556        );
557    }
558
559    #[cfg(feature = "cpp")]
560    #[test]
561    fn cpp_function_ops() {
562        check_ops(
563            LANG::Cpp,
564            "main()
565            {
566              int a, b, c, avg;
567              scanf(\"%d %d %d\", &a, &b, &c);
568              avg = (a + b + c) / 3;
569              printf(\"avg = %d\", avg);
570            }",
571            "foo.c",
572            &mut ["()", "{}", "int", "&", "=", "+", "/", ",", ";"],
573            &mut [
574                "main",
575                "a",
576                "b",
577                "c",
578                "avg",
579                "scanf",
580                "\"%d %d %d\"",
581                "3",
582                "printf",
583                "\"avg = %d\"",
584            ],
585        );
586    }
587
588    #[cfg(feature = "rust")]
589    #[test]
590    fn rust_ops() {
591        check_ops(
592            LANG::Rust,
593            "let: usize a = 5; let b: f32 = 7.0; let c: i32 = 3;",
594            "foo.rs",
595            &mut ["let", ":", "usize", "=", ";", "f32", "i32"],
596            &mut ["a", "b", "c", "5", "7.0", "3"],
597        );
598    }
599
600    #[cfg(feature = "rust")]
601    #[test]
602    fn rust_function_ops() {
603        check_ops(
604            LANG::Rust,
605            "fn main() {
606              let a = 5; let b = 5; let c = 5;
607              let avg = (a + b + c) / 3;
608              println!(\"{}\", avg);
609            }",
610            "foo.rs",
611            &mut ["fn", "()", "{}", "let", "=", "+", "/", ";", "!", ","],
612            &mut ["main", "a", "b", "c", "avg", "5", "3", "println", "\"{}\""],
613        );
614    }
615
616    #[cfg(feature = "javascript")]
617    #[test]
618    fn javascript_ops() {
619        check_ops(
620            LANG::Javascript,
621            "var a, b, c, avg;
622             let x = 1;
623             a = 5; b = 5; c = 5;
624             avg = (a + b + c) / 3;
625             console.log(\"{}\", avg);",
626            "foo.js",
627            &mut ["()", "var", "let", "=", "+", "/", ",", ".", ";"],
628            &mut [
629                "a", "b", "c", "avg", "x", "1", "3", "5", "console", "log", "\"{}\"",
630            ],
631        );
632    }
633
634    #[cfg(feature = "javascript")]
635    #[test]
636    fn javascript_function_ops() {
637        check_ops(
638            LANG::Javascript,
639            "function main() {
640              var a, b, c, avg;
641              let x = 1;
642              a = 5; b = 5; c = 5;
643              avg = (a + b + c) / 3;
644              console.log(\"{}\", avg);
645            }",
646            "foo.js",
647            &mut [
648                "function", "()", "{}", "var", "let", "=", "+", "/", ",", ".", ";",
649            ],
650            &mut [
651                "main", "a", "b", "c", "avg", "x", "1", "3", "5", "console", "log", "\"{}\"",
652            ],
653        );
654    }
655
656    #[cfg(feature = "mozjs")]
657    #[test]
658    fn mozjs_ops() {
659        check_ops(
660            LANG::Mozjs,
661            "var a, b, c, avg;
662             let x = 1;
663             a = 5; b = 5; c = 5;
664             avg = (a + b + c) / 3;
665             console.log(\"{}\", avg);",
666            "foo.js",
667            &mut ["()", "var", "let", "=", "+", "/", ",", ".", ";"],
668            &mut [
669                "a", "b", "c", "avg", "x", "1", "3", "5", "console", "log", "\"{}\"",
670            ],
671        );
672    }
673
674    #[cfg(feature = "mozjs")]
675    #[test]
676    fn mozjs_function_ops() {
677        check_ops(
678            LANG::Mozjs,
679            "function main() {
680              var a, b, c, avg;
681              let x = 1;
682              a = 5; b = 5; c = 5;
683              avg = (a + b + c) / 3;
684              console.log(\"{}\", avg);
685            }",
686            "foo.js",
687            &mut [
688                "function", "()", "{}", "var", "let", "=", "+", "/", ",", ".", ";",
689            ],
690            &mut [
691                "main", "a", "b", "c", "avg", "x", "1", "3", "5", "console", "log", "\"{}\"",
692            ],
693        );
694    }
695
696    #[cfg(feature = "typescript")]
697    #[test]
698    fn typescript_ops() {
699        // Issue #1261: the `: string` annotation counts exactly once,
700        // as the text-keyed `string` operator (PredefinedType wrapper)
701        // — symmetric with `: number` / `: boolean`. Under #313 its
702        // `String2` child also emitted a `"string"` operand, so one
703        // source token tallied twice.
704        check_ops(
705            LANG::Typescript,
706            "var a, b, c, avg;
707             let age: number = 32;
708             let name: string = \"John\"; let isUpdated: boolean = true;
709             a = 5; b = 5; c = 5;
710             avg = (a + b + c) / 3;
711             console.log(\"{}\", avg);",
712            "foo.ts",
713            &mut [
714                "()", "var", "let", "string", "number", "boolean", ":", "=", "+", "/", ",", ".",
715                ";",
716            ],
717            &mut [
718                "a",
719                "b",
720                "c",
721                "avg",
722                "age",
723                "name",
724                "isUpdated",
725                "32",
726                "\"John\"",
727                "true",
728                "3",
729                "5",
730                "console",
731                "log",
732                "\"{}\"",
733            ],
734        );
735    }
736
737    #[cfg(feature = "typescript")]
738    #[test]
739    fn typescript_function_ops() {
740        // Issue #1261: see `typescript_ops` — the `string` type keyword
741        // contributes only the primitive-typed operator, never a
742        // `"string"` operand.
743        check_ops(
744            LANG::Typescript,
745            "function main() {
746              var a, b, c, avg;
747              let age: number = 32;
748              let name: string = \"John\"; let isUpdated: boolean = true;
749              a = 5; b = 5; c = 5;
750              avg = (a + b + c) / 3;
751              console.log(\"{}\", avg);
752            }",
753            "foo.ts",
754            &mut [
755                "function", "()", "{}", "var", "let", "string", "number", "boolean", ":", "=", "+",
756                "/", ",", ".", ";",
757            ],
758            &mut [
759                "main",
760                "a",
761                "b",
762                "c",
763                "avg",
764                "age",
765                "name",
766                "isUpdated",
767                "32",
768                "\"John\"",
769                "true",
770                "3",
771                "5",
772                "console",
773                "log",
774                "\"{}\"",
775            ],
776        );
777    }
778
779    #[cfg(feature = "typescript")]
780    #[test]
781    fn tsx_ops() {
782        // Issue #1261: TSX exposes the `: string` type-keyword child as
783        // `String3` (vs. TS's `String2`); like TS, the keyword counts
784        // only as the `string` operator, never as an operand.
785        check_ops(
786            LANG::Tsx,
787            "var a, b, c, avg;
788             let age: number = 32;
789             let name: string = \"John\"; let isUpdated: boolean = true;
790             a = 5; b = 5; c = 5;
791             avg = (a + b + c) / 3;
792             console.log(\"{}\", avg);",
793            "foo.ts",
794            &mut [
795                "()", "var", "let", "string", "number", "boolean", ":", "=", "+", "/", ",", ".",
796                ";",
797            ],
798            &mut [
799                "a",
800                "b",
801                "c",
802                "avg",
803                "age",
804                "name",
805                "isUpdated",
806                "32",
807                "\"John\"",
808                "true",
809                "3",
810                "5",
811                "console",
812                "log",
813                "\"{}\"",
814            ],
815        );
816    }
817
818    #[cfg(feature = "typescript")]
819    #[test]
820    fn tsx_function_ops() {
821        // Issue #1261: see `tsx_ops` — TSX::String3 (type-keyword
822        // `string`) is an operator only, never an operand.
823        check_ops(
824            LANG::Tsx,
825            "function main() {
826              var a, b, c, avg;
827              let age: number = 32;
828              let name: string = \"John\"; let isUpdated: boolean = true;
829              a = 5; b = 5; c = 5;
830              avg = (a + b + c) / 3;
831              console.log(\"{}\", avg);
832            }",
833            "foo.ts",
834            &mut [
835                "function", "()", "{}", "var", "let", "string", "number", "boolean", ":", "=", "+",
836                "/", ",", ".", ";",
837            ],
838            &mut [
839                "main",
840                "a",
841                "b",
842                "c",
843                "avg",
844                "age",
845                "name",
846                "isUpdated",
847                "32",
848                "\"John\"",
849                "true",
850                "3",
851                "5",
852                "console",
853                "log",
854                "\"{}\"",
855            ],
856        );
857    }
858
859    // Issue #453: a `void` return type (a `predefined_type` wrapper over a
860    // `void` token) and an expression `void` (`void 0`) must collapse to a
861    // single distinct `"void"` operator. `check_ops` asserts the exact
862    // operator list, so a duplicate `"void"` — the pre-fix symptom, where
863    // the wrapper keyed `primitive_operators["void"]` and the inner token
864    // keyed `operators[Void]` — trips the assertion. This pins the lesson-4
865    // `n1 == dedupe(ops.operators)` invariant for the two `void` forms in
866    // one file.
867    #[cfg(feature = "typescript")]
868    #[test]
869    fn typescript_void_return_and_expression_single_operator_453() {
870        check_ops(
871            LANG::Typescript,
872            "function f(): void { return void 0; }",
873            "foo.ts",
874            &mut ["function", "()", "{}", ":", "void", "return", ";"],
875            &mut ["f", "0"],
876        );
877    }
878
879    #[cfg(feature = "typescript")]
880    #[test]
881    fn tsx_void_return_and_expression_single_operator_453() {
882        check_ops(
883            LANG::Tsx,
884            "function f(): void { return void 0; }",
885            "foo.tsx",
886            &mut ["function", "()", "{}", ":", "void", "return", ";"],
887            &mut ["f", "0"],
888        );
889    }
890
891    #[cfg(feature = "java")]
892    #[test]
893    fn java_ops() {
894        check_ops(
895            LANG::Java,
896            "public class Main {
897                public static void main(string args[]) {
898                      int a, b, c, avg;
899                      a = 5; b = 5; c = 5;
900                      avg = (a + b + c) / 3;
901                      MessageFormat.format(\"{0}\", avg);
902                    }
903                }",
904            "foo.java",
905            &mut [
906                "class", "{}", "void", "()", "[]", ",", ".", ";", "int", "=", "+", "/",
907            ],
908            &mut [
909                "Main",
910                "main",
911                // the parameter's type name (#1560)
912                "string",
913                "args",
914                "a",
915                "b",
916                "c",
917                "avg",
918                "5",
919                "3",
920                "MessageFormat",
921                "format",
922                "\"{0}\"",
923            ],
924        );
925    }
926
927    #[cfg(feature = "java")]
928    #[test]
929    fn java_primitive_ops() {
930        check_ops(
931            LANG::Java,
932            "public class Prims {
933                byte a = 1;
934                short b = 2;
935                int c = 3;
936                long d = 4;
937                char e = 'x';
938                float f = 1.0f;
939                double g = 2.0;
940                boolean h = true;
941                boolean i = false;
942            }",
943            "foo.java",
944            // All 8 primitive-type keywords must appear as distinct operators.
945            // true/false appear as operands; `class` is billed since #1552.
946            &mut [
947                "class",
948                "{}",
949                ";",
950                "=",
951                "byte",
952                "short",
953                "int",
954                "long",
955                "char",
956                "float",
957                "double",
958                "boolean_type",
959            ],
960            &mut [
961                "Prims", "a", "b", "c", "d", "e", "f", "g", "h", "i", "1", "2", "3", "4", "'x'",
962                "1.0f", "2.0", "true", "false",
963            ],
964        );
965    }
966
967    /// A `Unit` space must never carry the synthetic `<anonymous>`
968    /// placeholder that the default getter invents for nodes without a
969    /// `name` field. The public docs describe `None` as the
970    /// "name could not be resolved" state, and the metrics-side
971    /// `FuncSpace::new` already special-cases `SpaceKind::Unit` the same
972    /// way; this pins the `Ops::new` mirror so a regression to the old
973    /// `Some("<anonymous>")` initialisation fails here rather than only
974    /// surfacing for a (currently unreachable) non-top-level `Unit`
975    /// space, where `ops_inner`'s top-level override would not rescue it.
976    /// See issue #755.
977    #[cfg(feature = "rust")]
978    #[test]
979    fn unit_space_name_is_none_not_anonymous() {
980        use crate::getter::Getter;
981        use crate::node::Ancestors;
982        use crate::traits::ParserTrait;
983        use crate::{RustCode, RustParser, SpaceKind};
984
985        let code = b"fn f() {}\n";
986        let parser = RustParser::new(code.to_vec(), std::path::Path::new("foo.rs"), None);
987        let root = parser.root();
988        // The Rust `source_file` root is a `Unit` and has no `name`/`type`
989        // field, so the default getter would invent `<anonymous>`.
990        assert_eq!(SpaceKind::Unit, RustCode::get_space_kind(&root));
991
992        let ops = super::Ops::new::<RustCode>(&root, code, Ancestors::unknown(), SpaceKind::Unit);
993        assert_eq!(
994            ops.name, None,
995            "Unit space must preserve name = None, not invent <anonymous>"
996        );
997    }
998
999    /// Issue #789: an `ERROR`-root parse (here Lua partial input) where
1000    /// `metrics()` succeeds must make `ops()` succeed too — the two seams
1001    /// should agree. Before the synthetic-Unit-root mirror in `ops_inner`,
1002    /// `ops()` returned `Err(MetricsError::EmptyRoot)` because the ERROR
1003    /// root is not classified as a function space, so no frame was ever
1004    /// pushed. This pins their agreement: both succeed, and the resulting
1005    /// top-level `Ops` is a `Unit` whose name is the caller-supplied
1006    /// `Source::name` (the intrinsic Unit name stays `None` per #755 until
1007    /// `ops_inner` overrides the top-level name).
1008    #[cfg(feature = "lua")]
1009    #[test]
1010    fn lua_error_root_ops_agrees_with_metrics_789() {
1011        use crate::{MetricsOptions, SpaceKind};
1012
1013        // tree-sitter-lua surfaces an ERROR root for this partial input.
1014        let src = b"function foo(x)\n  return x +\n".to_vec();
1015        let name = "partial.lua".to_owned();
1016
1017        let ast = Ast::parse(Source::new(LANG::Lua, &src).with_name(Some(name.clone())))
1018            .expect("lua feature enabled");
1019
1020        // metrics() must succeed (it already wrapped a synthetic Unit root).
1021        let space = ast
1022            .metrics(MetricsOptions::default())
1023            .expect("metrics must yield a top-level space");
1024        assert_eq!(space.kind, SpaceKind::Unit);
1025
1026        // ops() must now succeed in the same case rather than returning
1027        // Err(EmptyRoot).
1028        let ops = ast
1029            .ops()
1030            .expect("ops must agree with metrics and yield a top-level Ops");
1031        assert_eq!(ops.kind, SpaceKind::Unit);
1032        assert_eq!(
1033            ops.name.as_deref(),
1034            Some(name.as_str()),
1035            "top-level Ops name is the caller-supplied Source::name"
1036        );
1037    }
1038
1039    /// Issue #790: `Ops::operands` / `Ops::operators` are the *distinct*
1040    /// (deduplicated) Halstead operand/operator vocabularies (`n2` / `n1`),
1041    /// not every occurrence. Pin the documented dedup semantics: each
1042    /// vector's length equals its unique-element count. The fixture
1043    /// repeats `+`, `;`, and `=` operators and the `a` operand so a
1044    /// regression to non-deduplicated collection would make `len` exceed
1045    /// the unique count.
1046    #[cfg(feature = "rust")]
1047    #[test]
1048    fn ops_vocabularies_are_distinct_790() {
1049        use std::collections::HashSet;
1050
1051        let src = b"fn main() { let a = 1 + 1; let b = a + a; }\n".to_vec();
1052        let ops = Ast::parse(Source::new(LANG::Rust, &src).with_name(Some("foo.rs".to_owned())))
1053            .expect("rust feature enabled")
1054            .ops()
1055            .expect("ops walk must yield a top-level Ops");
1056
1057        let unique_operators: HashSet<&String> = ops.operators.iter().collect();
1058        assert_eq!(
1059            ops.operators.len(),
1060            unique_operators.len(),
1061            "Ops::operators must be the distinct operator vocabulary (n1)"
1062        );
1063
1064        let unique_operands: HashSet<&String> = ops.operands.iter().collect();
1065        assert_eq!(
1066            ops.operands.len(),
1067            unique_operands.len(),
1068            "Ops::operands must be the distinct operand vocabulary (n2)"
1069        );
1070    }
1071
1072    /// Assert that every space in the tree carries sorted vocabularies,
1073    /// and return how many spaces were checked so a caller can prove the
1074    /// walk actually descended.
1075    ///
1076    /// The length floor is what keeps this from going quietly vacuous:
1077    /// `is_sorted` is trivially true for an empty or single-entry
1078    /// vector, so without it a fixture that stopped producing real
1079    /// vocabularies would keep passing while covering nothing.
1080    fn assert_sorted_spaces(ops: &Ops, lang: LANG) -> usize {
1081        /// Smallest vocabulary in which an ordering is observable.
1082        const MIN_OBSERVABLE: usize = 2;
1083
1084        let mut stack = vec![ops];
1085        let mut visited = 0;
1086
1087        while let Some(space) = stack.pop() {
1088            visited += 1;
1089            for (field, values) in [
1090                ("operators", &space.operators),
1091                ("operands", &space.operands),
1092            ] {
1093                assert!(
1094                    values.len() >= MIN_OBSERVABLE && values.is_sorted(),
1095                    "{lang:?} {field} of space {:?} (@{}) must hold at least \
1096                     {MIN_OBSERVABLE} entries and be sorted: {values:?}",
1097                    space.name,
1098                    space.start_line
1099                );
1100            }
1101            stack.extend(space.spaces.iter());
1102        }
1103
1104        visited
1105    }
1106
1107    /// Every space's vocabularies come back sorted, in every language.
1108    ///
1109    /// The operator vocabulary is the union of two maps — one keyed by
1110    /// token id, one keyed by text, which is where primitive types such
1111    /// as C++ `int` land — and sorting is what interleaves them rather
1112    /// than leaving the second concatenated onto the first (#1091). The
1113    /// `spans_both_maps` pair per case names one entry from each map, so
1114    /// a fixture that stopped exercising the text-keyed map fails here
1115    /// instead of silently narrowing the test's reach.
1116    #[test]
1117    fn ops_vocabularies_are_sorted_1091() {
1118        /// `(language, file name, source, (text-keyed operator,
1119        /// token-id-keyed operator that must sort after it))`.
1120        type Case = (
1121            LANG,
1122            &'static str,
1123            &'static str,
1124            (&'static str, &'static str),
1125        );
1126
1127        let cases: &[Case] = &[
1128            #[cfg(feature = "rust")]
1129            (
1130                LANG::Rust,
1131                "rust.rs",
1132                "fn zeta(quux: u32) -> u32 { let mid = quux + 1; \
1133                 let alpha = |beta: u32| beta * mid; alpha(mid) - quux }\n",
1134                ("u32", "|"),
1135            ),
1136            #[cfg(feature = "cpp")]
1137            (
1138                LANG::Cpp,
1139                "cpp.cpp",
1140                "int zeta(int quux) { double mid = quux + 1; \
1141                 char alpha = 'z'; return quux - mid + alpha; }\n",
1142                ("int", "return"),
1143            ),
1144            #[cfg(feature = "java")]
1145            (
1146                LANG::Java,
1147                "Java.java",
1148                "class Zeta { int quux(int mid) { long alpha = mid + 1; \
1149                 boolean beta = alpha > 2; return beta ? mid : 0; } }\n",
1150                ("long", "return"),
1151            ),
1152            #[cfg(feature = "python")]
1153            (
1154                LANG::Python,
1155                "python.py",
1156                "def zeta(quux):\n    mid = quux + 1\n    \
1157                 def alpha(beta):\n        return beta * mid\n    return alpha(mid) - quux\n",
1158                // Python has no primitive-type operators; both entries
1159                // come from the token-id map, so this pair only pins the
1160                // ordering, not the interleaving.
1161                ("def", "return"),
1162            ),
1163            #[cfg(feature = "typescript")]
1164            (
1165                LANG::Typescript,
1166                "ts.ts",
1167                "function zeta(quux: number): number { const mid: number = quux + 1; \
1168                 const alpha = (beta: number) => beta * mid; return alpha(mid) - quux; }\n",
1169                ("number", "return"),
1170            ),
1171        ];
1172
1173        for (lang, file, source, (from_text_map, sorts_after)) in cases {
1174            let ops = Ast::parse(
1175                Source::new(*lang, source.as_bytes()).with_name(Some((*file).to_owned())),
1176            )
1177            .expect("language feature enabled")
1178            .ops()
1179            .expect("ops walk must yield a top-level Ops");
1180
1181            let position = |needle: &str| {
1182                ops.operators
1183                    .iter()
1184                    .position(|op| op == needle)
1185                    .unwrap_or_else(|| {
1186                        panic!(
1187                            "{lang:?} operators must contain {needle:?}: {:?}",
1188                            ops.operators
1189                        )
1190                    })
1191            };
1192            assert!(
1193                position(from_text_map) < position(sorts_after),
1194                "{lang:?} must order {from_text_map:?} before {sorts_after:?}: {:?}",
1195                ops.operators
1196            );
1197
1198            assert!(
1199                assert_sorted_spaces(&ops, *lang) > 1,
1200                "{lang:?} sample must nest at least one sub-space"
1201            );
1202        }
1203    }
1204
1205    /// Two parses of the same bytes in one process must agree exactly.
1206    ///
1207    /// `RandomState` bumps its thread-local seed per instance, so the
1208    /// pre-fix code could — and did — order two `HashMap`s built from
1209    /// identical keys differently within a single run. This is the
1210    /// in-process form of the cross-run churn in #1091.
1211    #[test]
1212    #[cfg(feature = "rust")]
1213    fn ops_are_stable_across_repeated_parses_1091() {
1214        use std::fmt::Write as _;
1215
1216        // Enough distinct operands, in enough distinct spaces, that
1217        // agreement by coincidence is not a plausible explanation for a
1218        // pass.
1219        let mut src = String::new();
1220        for i in 0..40 {
1221            writeln!(src, "fn name{i}(arg{i}: u32) -> u32 {{ arg{i} + {i} }}")
1222                .expect("writing to a String cannot fail");
1223        }
1224
1225        let parse = || {
1226            Ast::parse(Source::new(LANG::Rust, src.as_bytes()).with_name(Some("foo.rs".to_owned())))
1227                .expect("rust feature enabled")
1228                .ops()
1229                .expect("ops walk must yield a top-level Ops")
1230        };
1231
1232        let (first, second) = (parse(), parse());
1233        assert!(
1234            first.operands.len() >= 40 && first.spaces.len() >= 40,
1235            "sample must have a wide vocabulary across many spaces, got {} operands \
1236             in {} spaces",
1237            first.operands.len(),
1238            first.spaces.len()
1239        );
1240        // `Ops` has no `PartialEq`, and the nested spaces are the half a
1241        // top-level vector comparison would miss, so compare the whole
1242        // serialized tree.
1243        let render =
1244            |ops: &Ops| serde_json::to_string(&ops.to_wire()).expect("wire Ops serializes to JSON");
1245        assert_eq!(render(&first), render(&second));
1246    }
1247
1248    /// A vocabulary that lost bytes is ordered by its *rendered* text.
1249    ///
1250    /// #1110 moved the sort ahead of the lossy UTF-8 rendering, which is
1251    /// only order-preserving while every key is valid UTF-8. Here it is
1252    /// not: the two string operands are `"\xffA"` and `"\u{fffd}B"`, so
1253    /// by raw bytes the first sorts *after* the second (`0xff` > `0xef`)
1254    /// and by rendered text — both start `U+FFFD`, then `A` before `B` —
1255    /// it sorts before. The rendered order is what the pre-#1110
1256    /// render-then-sort code produced and what `Ops` documents, so
1257    /// dropping the fallback re-sort flips this pair and fails here.
1258    #[test]
1259    #[cfg(feature = "rust")]
1260    fn ops_vocabulary_orders_lossy_entries_by_rendered_text_1110() {
1261        let mut src = b"fn f() { let a = \"".to_vec();
1262        src.push(0xff);
1263        src.extend_from_slice(b"A\"; let b = \"");
1264        src.extend_from_slice(&[0xef, 0xbf, 0xbd]);
1265        src.extend_from_slice(b"B\"; }\n");
1266
1267        let ops = Ast::parse(Source::new(LANG::Rust, &src).with_name(Some("foo.rs".to_owned())))
1268            .expect("rust feature enabled")
1269            .ops()
1270            .expect("ops walk must yield a top-level Ops");
1271
1272        let position = |needle: &str| {
1273            ops.operands
1274                .iter()
1275                .position(|operand| operand == needle)
1276                .unwrap_or_else(|| panic!("operands must contain {needle:?}: {:?}", ops.operands))
1277        };
1278        assert!(
1279            position("\"\u{fffd}A\"") < position("\"\u{fffd}B\""),
1280            "lossy entries must be ordered by rendered text, got {:?}",
1281            ops.operands
1282        );
1283        assert!(
1284            ops.operands.is_sorted(),
1285            "the whole vocabulary must be sorted as rendered, got {:?}",
1286            ops.operands
1287        );
1288    }
1289
1290    /// The walk classifies a space kind once per space, not once per node.
1291    ///
1292    /// Nothing in the output distinguishes the two: the classification
1293    /// only ever reaches `Ops::new`, so running it on every node produces
1294    /// the same tree and merely throws the extra answers away. #1110
1295    /// moved the call inside the `func_space` branch, mirroring
1296    /// `spaces::compute::open_func_space`; the counter is what makes
1297    /// hoisting it back out a failure. The node-count assertion is what
1298    /// makes the counts distinguishable — a fixture whose nodes and
1299    /// spaces were equal in number could not tell the two apart.
1300    // test-lang-gates: hand-written(rust) — the one grammar that
1301    //     makes the fixture list non-empty, so the non-vacuity
1302    //     assertion cannot fire on a minimal build
1303    #[test]
1304    // Gated on the language that guarantees a non-empty case list, so
1305    // the emptiness assertion below cannot fire on a minimal build.
1306    #[cfg(feature = "rust")]
1307    fn ops_classifies_space_kind_once_per_space_1110() {
1308        let cases: &[(LANG, &str, &str)] = &[
1309            #[cfg(feature = "rust")]
1310            (
1311                LANG::Rust,
1312                "foo.rs",
1313                "fn outer(a: u32) -> u32 { fn inner(b: u32) -> u32 { b + 1 } inner(a) * 2 }\n",
1314            ),
1315            #[cfg(feature = "python")]
1316            (
1317                LANG::Python,
1318                "foo.py",
1319                "def outer(a):\n    def inner(b):\n        return b + 1\n    return inner(a) * 2\n",
1320            ),
1321            #[cfg(feature = "cpp")]
1322            (
1323                LANG::Cpp,
1324                "foo.cpp",
1325                "struct S { int m(int a) { return a + 1; } }; int f(int b) { return b * 2; }\n",
1326            ),
1327            #[cfg(feature = "java")]
1328            (
1329                LANG::Java,
1330                "Foo.java",
1331                "class C { int m(int a) { return a + 1; } int n(int b) { return b * 2; } }\n",
1332            ),
1333            #[cfg(feature = "javascript")]
1334            (
1335                LANG::Javascript,
1336                "foo.js",
1337                "function outer(a) { function inner(b) { return b + 1; } return inner(a) * 2; }\n",
1338            ),
1339        ];
1340        crate::test_support::assert_fixtures_present(cases);
1341
1342        for (lang, file, source) in cases {
1343            let ast = crate::test_support::parse_named(*lang, file, source);
1344
1345            let before = super::space_kind_lookups::observed();
1346            let ops = ast.ops().expect("ops walk must yield a top-level Ops");
1347            let lookups = super::space_kind_lookups::observed() - before;
1348
1349            let mut spaces = 0;
1350            let mut stack = vec![&ops];
1351            while let Some(space) = stack.pop() {
1352                spaces += 1;
1353                stack.extend(space.spaces.iter());
1354            }
1355
1356            let mut nodes = 0;
1357            let mut cursor = vec![ast.as_tree_sitter().root_node()];
1358            while let Some(node) = cursor.pop() {
1359                nodes += 1;
1360                let mut walker = node.walk();
1361                cursor.extend(node.children(&mut walker));
1362            }
1363
1364            assert!(
1365                nodes > spaces * 4,
1366                "{lang:?} fixture must have many more nodes ({nodes}) than spaces ({spaces}) \
1367                 for the two counts to be distinguishable"
1368            );
1369            assert_eq!(
1370                lookups, spaces,
1371                "{lang:?} must classify once per space, not once per node ({nodes} nodes)"
1372            );
1373        }
1374    }
1375
1376    /// One flattened space: `(depth, kind, name, start_line, end_line)`.
1377    #[cfg(feature = "elixir")]
1378    type FlatSpace = (usize, crate::SpaceKind, String, usize, usize);
1379
1380    /// Flattens an `Ops` tree in preorder, so a test can pin the whole
1381    /// tree in one `assert_eq!` and see the surrounding spaces when one
1382    /// is wrong.
1383    ///
1384    /// `end_line` is carried as well as `start_line` because a change to
1385    /// the promote predicate can move a space's *extent* without moving
1386    /// its head — a `def` that swallows its sibling would keep the same
1387    /// start line.
1388    #[cfg(feature = "elixir")]
1389    fn flatten(ops: &Ops, depth: usize, out: &mut Vec<FlatSpace>) {
1390        out.push((
1391            depth,
1392            ops.kind,
1393            ops.name.clone().unwrap_or_else(|| "<none>".to_owned()),
1394            ops.start_line,
1395            ops.end_line,
1396        ));
1397        for child in &ops.spaces {
1398            flatten(child, depth + 1, out);
1399        }
1400    }
1401
1402    #[cfg(feature = "elixir")]
1403    fn elixir_ops_tree(source: &str) -> Vec<FlatSpace> {
1404        let ops = crate::test_support::parse_named(LANG::Elixir, "foo.ex", source)
1405            .ops()
1406            .expect("ops walk must yield a top-level Ops");
1407        let mut flat = Vec::new();
1408        flatten(&ops, 0, &mut flat);
1409        flat
1410    }
1411
1412    /// Issue #1130: Elixir's `defmodule` / `def` are `Call` nodes whose
1413    /// target identifier text spells the keyword, so only the
1414    /// source-aware promote predicate can recognise them. Before the
1415    /// fix `ops()` returned the bare file-level `Unit` for this input
1416    /// while `metrics()` returned the full module/function tree.
1417    #[cfg(feature = "elixir")]
1418    #[test]
1419    fn elixir_ops_opens_module_and_function_spaces_1130() {
1420        use crate::SpaceKind::{Class, Function, Unit};
1421
1422        assert_eq!(
1423            elixir_ops_tree("defmodule Foo do\n  def bar(x) do\n    x + 1\n  end\nend\n"),
1424            vec![
1425                (0, Unit, "foo.ex".to_owned(), 1, 5),
1426                (1, Class, "Foo".to_owned(), 1, 5),
1427                (2, Function, "bar".to_owned(), 2, 4),
1428            ],
1429        );
1430    }
1431
1432    /// An `AnonymousFunction` is the one Elixir space the byte-less
1433    /// predicate could already see, so this pins that the source-aware
1434    /// predicate did not lose it — and that it nests under the `def`
1435    /// space rather than being reparented to the file root.
1436    #[cfg(feature = "elixir")]
1437    #[test]
1438    fn elixir_ops_opens_anonymous_function_space() {
1439        use crate::SpaceKind::{Class, Function, Unit};
1440
1441        assert_eq!(
1442            elixir_ops_tree(
1443                "defmodule Foo do\n  def bar(list) do\n    \
1444                 Enum.map(list, fn x -> x * 2 end)\n  end\nend\n"
1445            ),
1446            vec![
1447                (0, Unit, "foo.ex".to_owned(), 1, 5),
1448                (1, Class, "Foo".to_owned(), 1, 5),
1449                (2, Function, "bar".to_owned(), 2, 4),
1450                (3, Function, "<anonymous>".to_owned(), 3, 3),
1451            ],
1452        );
1453    }
1454
1455    /// Issue #310: a `def` inside `quote do … end` is a code *template*
1456    /// emitted later by macro expansion, not a declaration of the
1457    /// enclosing module, so it opens no space. This is the case only the
1458    /// source-aware predicate can get right — the byte-less one never
1459    /// saw any `def` at all, so it was accidentally "correct" here while
1460    /// being wrong everywhere else.
1461    #[cfg(feature = "elixir")]
1462    #[test]
1463    fn elixir_ops_skips_def_inside_quote_block_310() {
1464        use crate::SpaceKind::{Class, Function, Unit};
1465
1466        // The quoted `def` heads line 4. Its name resolves to the
1467        // `<anonymous>` placeholder (the head is `unquote(name)`, not a
1468        // literal identifier), so the absence of a fourth entry — not a
1469        // name match — is what pins it out of the tree.
1470        assert_eq!(
1471            elixir_ops_tree(
1472                "defmodule Foo do\n  defmacro gen(name) do\n    quote do\n      \
1473                 def unquote(name)(x) do\n        x + 1\n      end\n    end\n  end\nend\n",
1474            ),
1475            vec![
1476                (0, Unit, "foo.ex".to_owned(), 1, 9),
1477                (1, Class, "Foo".to_owned(), 1, 9),
1478                (2, Function, "gen".to_owned(), 2, 8),
1479            ],
1480        );
1481    }
1482
1483    /// #1559: a generic list's angle brackets are one `<>` bracket pair,
1484    /// billed on the opener as `()` / `[]` / `{}` are, with the closer
1485    /// billing nothing. Every fixture also spells a comparison `<` and
1486    /// `>`, which must stay their own vocabulary entries: the grammars
1487    /// give the list brackets the comparison operators' kind ids, so the
1488    /// vocabulary is what shows the two apart and `N1` is what shows the
1489    /// closer is not billed.
1490    #[cfg(any(
1491        feature = "cpp",
1492        feature = "csharp",
1493        feature = "groovy",
1494        feature = "java",
1495        feature = "kotlin",
1496        feature = "mozcpp",
1497        feature = "objc",
1498        feature = "rust",
1499        feature = "typescript",
1500    ))]
1501    fn check_generic_pair(lang: LANG, file: &str, source: &str, operators: &[&str], n1: [u64; 2]) {
1502        let ast = crate::test_support::parse_named(lang, file, source);
1503        let ops = ast.ops().expect("ops walk must yield a top-level Ops");
1504        let mut expected = operators.to_vec();
1505        expected.sort_unstable();
1506        assert_eq!(ops.operators, expected, "{lang:?} vocabulary");
1507
1508        let space = ast
1509            .metrics(crate::MetricsOptions::default().with_only(&[crate::Metric::Halstead]))
1510            .expect("metrics walk");
1511        let halstead = &space.metrics.halstead;
1512        assert_eq!(
1513            [halstead.unique_operators(), halstead.total_operators()],
1514            n1,
1515            "{lang:?} [n1, N1]"
1516        );
1517    }
1518
1519    // expected: `<>` three times (one parameter list, two nested
1520    // argument lists), never `>`; `<` and `>` once each, the comparisons.
1521    #[cfg(feature = "rust")]
1522    #[test]
1523    fn rust_generic_angles_are_one_pair_1559() {
1524        check_generic_pair(
1525            LANG::Rust,
1526            "foo.rs",
1527            "fn f<T>(v: Vec<Vec<T>>) -> bool { a < b && c > d }\n",
1528            &["fn", "<>", "()", ":", "->", "bool", "{}", "<", "&&", ">"],
1529            [10, 12],
1530        );
1531        // A turbofish and a higher-ranked `for<'a>` (`for_lifetimes`)
1532        // are lists too, `<>` twice (`fn` is twice as well); the `>>` shift
1533        // is its own operator.
1534        check_generic_pair(
1535            LANG::Rust,
1536            "foo.rs",
1537            "fn g(f: for<'a> fn(&'a u8)) { h::<u8>(x >> 1); }\n",
1538            &[
1539                "fn", "()", ":", "for", "<>", "&", "u8", "{}", "::", ">>", ";",
1540            ],
1541            [11, 16],
1542        );
1543        // A precise-capturing `use<T>` (`use_bounds`) is a list as well.
1544        check_generic_pair(
1545            LANG::Rust,
1546            "foo.rs",
1547            "fn u<T>(x: T) -> impl Sized + use<T> { x }\n",
1548            &["fn", "<>", "()", ":", "->", "impl", "+", "use", "{}"],
1549            [9, 10],
1550        );
1551    }
1552
1553    // expected: `<>` three times, `;` and `{}` twice each.
1554    #[cfg(feature = "java")]
1555    #[test]
1556    fn java_generic_angles_are_one_pair_1559() {
1557        check_generic_pair(
1558            LANG::Java,
1559            "foo.java",
1560            "class A<T> { List<List<T>> v; boolean m() { return a < b && c > d; } }\n",
1561            &[
1562                "class",
1563                "<>",
1564                ";",
1565                "{}",
1566                "boolean_type",
1567                "()",
1568                "return",
1569                "<",
1570                "&&",
1571                ">",
1572            ],
1573            [10, 14],
1574        );
1575    }
1576
1577    // expected: as Java, with `bool` for `boolean_type`.
1578    #[cfg(feature = "csharp")]
1579    #[test]
1580    fn csharp_generic_angles_are_one_pair_1559() {
1581        check_generic_pair(
1582            LANG::Csharp,
1583            "foo.cs",
1584            "class A<T> { List<List<T>> v; bool M() { return a < b && c > d; } }\n",
1585            &[
1586                "class", "<>", ";", "{}", "bool", "()", "return", "<", "&&", ">",
1587            ],
1588            [10, 14],
1589        );
1590    }
1591
1592    // expected: `<>` three times, `:` twice. TSX shares the getter macro
1593    // and the same arm, so it is checked on the same input.
1594    #[cfg(feature = "typescript")]
1595    #[test]
1596    fn typescript_generic_angles_are_one_pair_1559() {
1597        const SOURCE: &str =
1598            "function f<T>(v: Array<Array<T>>): boolean { return a < b && c > d; }\n";
1599        const OPERATORS: &[&str] = &[
1600            "function", "<>", "()", ":", "boolean", "{}", "return", "<", "&&", ">", ";",
1601        ];
1602        check_generic_pair(LANG::Typescript, "foo.ts", SOURCE, OPERATORS, [11, 14]);
1603        check_generic_pair(LANG::Tsx, "foo.tsx", SOURCE, OPERATORS, [11, 14]);
1604    }
1605
1606    // expected: `<>` three times (the `template` parameter list and two
1607    // nested argument lists), `::` twice. The closer of the outer list
1608    // is spelled `>>` and still parses as two `>` tokens.
1609    #[cfg(feature = "cpp")]
1610    #[test]
1611    fn cpp_generic_angles_are_one_pair_1559() {
1612        check_generic_pair(
1613            LANG::Cpp,
1614            "foo.cpp",
1615            "template <class T> bool f(std::vector<std::vector<T>> v) { return a < b && c > d; }\n",
1616            &[
1617                "template", "<>", "class", "bool", "()", "::", "{}", "return", "<", "&&", ">", ";",
1618            ],
1619            [12, 15],
1620        );
1621    }
1622
1623    // expected: the Cpp row, through the Mozcpp grammar, which owns no
1624    // file extension and so has no corpus coverage of its own.
1625    #[cfg(feature = "mozcpp")]
1626    #[test]
1627    fn mozcpp_generic_angles_are_one_pair_1559() {
1628        check_generic_pair(
1629            LANG::Mozcpp,
1630            "foo.cpp",
1631            "template <class T> bool f(std::vector<std::vector<T>> v) { return a < b && c > d; }\n",
1632            &[
1633                "template", "<>", "class", "bool", "()", "::", "{}", "return", "<", "&&", ">", ";",
1634            ],
1635            [12, 15],
1636        );
1637    }
1638
1639    // expected: `<>` three times, `:` twice.
1640    #[cfg(feature = "kotlin")]
1641    #[test]
1642    fn kotlin_generic_angles_are_one_pair_1559() {
1643        check_generic_pair(
1644            LANG::Kotlin,
1645            "foo.kt",
1646            "fun <T> f(v: List<List<T>>): Boolean { return a < b && c > d }\n",
1647            &["fun", "<>", "()", ":", "{}", "return", "<", "&&", ">"],
1648            [9, 12],
1649        );
1650    }
1651
1652    // expected: `<>` four times — Groovy spells a method's parameter
1653    // list as its own `method_type_parameters` node.
1654    #[cfg(feature = "groovy")]
1655    #[test]
1656    fn groovy_generic_angles_are_one_pair_1559() {
1657        check_generic_pair(
1658            LANG::Groovy,
1659            "foo.groovy",
1660            "class A<T> { List<List<T>> v; def <U> boolean m() { return a < b && c > d } }\n",
1661            &[
1662                "class", "<>", ";", "{}", "def", "()", "return", "<", "&&", ">",
1663            ],
1664            [10, 14],
1665        );
1666    }
1667
1668    // expected: `<>` five times, once per list — `parameterized_arguments`
1669    // (`: NSObject <NSCopying>`), `protocol_reference_list`
1670    // (`id<NSCopying>`), two nested `generic_specifier`s, and the
1671    // `argument_list` type form `g(NSArray<NSString *>)` — and never `>`.
1672    // `*` four times, `()` and `||` twice; `@end` and `id` bill nothing.
1673    // Without the pair, `<` and `>` would each bill six times: [10, 25].
1674    // expected: the spaced twin `g(a < b, c > d)`, two comparisons.
1675    // Unspaced, tree-sitter-objc reads the arguments as an
1676    // `argument_list`'s `Type<…>` form with an `ERROR` after the `>`, and
1677    // they must keep billing `<` and `>`; the clean `g(NSArray<NSString *>)`
1678    // stays one pair.
1679    #[cfg(feature = "objc")]
1680    #[test]
1681    fn objc_misparsed_comparisons_stay_comparisons_1559() {
1682        const COMPARISONS: &[&str] = &["()", ",", ";", "<", ">", "int", "return", "void", "{}"];
1683        check_generic_pair(
1684            LANG::Objc,
1685            "foo.m",
1686            "int f(void) { return g(a < b, c > d); }\n",
1687            COMPARISONS,
1688            [9, 10],
1689        );
1690        check_generic_pair(
1691            LANG::Objc,
1692            "foo.m",
1693            "int f(void) { return g(a<b, c>d); }\n",
1694            COMPARISONS,
1695            [9, 10],
1696        );
1697        check_generic_pair(
1698            LANG::Objc,
1699            "foo.m",
1700            "int f(void) { return g(NSArray<NSString *>); }\n",
1701            &["()", "*", ";", "<>", "int", "return", "void", "{}"],
1702            [8, 9],
1703        );
1704    }
1705
1706    // expected: the reading before #1559, which billed the two
1707    // comparisons as `<` and `>`. By the C# specification the identifier
1708    // after the `>` makes `F(a < b, c > d)` two arguments, though the
1709    // grammar parses a declaration typed `a<b, c>`. An `out` declaration
1710    // is a real generic and stays one pair.
1711    #[cfg(feature = "csharp")]
1712    #[test]
1713    fn csharp_misparsed_comparisons_stay_comparisons_1559() {
1714        check_generic_pair(
1715            LANG::Csharp,
1716            "foo.cs",
1717            "class A { void M() { F(a < b, c > d); } }\n",
1718            &["()", ",", ";", "<", ">", "class", "void", "{}"],
1719            [8, 10],
1720        );
1721        // A member access before the `<` wraps the generic name in a
1722        // `qualified_name`; the comparisons are the same.
1723        check_generic_pair(
1724            LANG::Csharp,
1725            "foo.cs",
1726            "class A { void M() { F(x.a < b, c > d); } }\n",
1727            &["()", ",", ".", ";", "<", ">", "class", "void", "{}"],
1728            [9, 11],
1729        );
1730        check_generic_pair(
1731            LANG::Csharp,
1732            "foo.cs",
1733            "class A { void M() { F(out List<int> x); } }\n",
1734            &["()", ";", "<>", "class", "int", "out", "void", "{}"],
1735            [8, 10],
1736        );
1737        // A deconstruction's declarations are `argument`s of a
1738        // `tuple_expression`, not of an argument list: a real generic.
1739        check_generic_pair(
1740            LANG::Csharp,
1741            "foo.cs",
1742            "class A { void M() { (List<int> p, int q) = (1, 2); } }\n",
1743            &["()", ",", ";", "<>", "=", "class", "int", "void", "{}"],
1744            [9, 14],
1745        );
1746        // A using alias's generic name sits two levels under the root, too
1747        // shallow for the argument-list shape.
1748        check_generic_pair(
1749            LANG::Csharp,
1750            "foo.cs",
1751            "using X = List<int>;\n",
1752            &[";", "<>", "=", "int", "using"],
1753            [5, 5],
1754        );
1755    }
1756
1757    #[cfg(feature = "objc")]
1758    #[test]
1759    fn objc_generic_angles_are_one_pair_1559() {
1760        check_generic_pair(
1761            LANG::Objc,
1762            "foo.m",
1763            "@interface A : NSObject <NSCopying>\n@end\n\
1764             id<NSCopying> f(NSArray<NSArray<NSString *> *> *v) \
1765             { return g(NSArray<NSString *>) || a < b || c > d; }\n",
1766            &[
1767                "@interface",
1768                ":",
1769                "<>",
1770                "()",
1771                "*",
1772                "{}",
1773                "return",
1774                "||",
1775                "<",
1776                ">",
1777                ";",
1778            ],
1779            [11, 20],
1780        );
1781    }
1782}