big_code_analysis/metrics/

tokens.rs

// Per-language metric and AST modules deliberately consume the macro-
// generated tree-sitter token enums via `use crate::*` and `use Foo::*`
// inside match expressions — explicit imports would list dozens of
// variants per arm and obscure the per-language token sets that are the
// point of these files. Allowed at the module level rather than per
// function so the per-language impl blocks stay readable.
#![allow(clippy::wildcard_imports, clippy::enum_glob_use)]
// Metric counts (token, function, branch, argument, etc.) are stored as
// `usize` and crossed with `f64` averages, ratios, and Halstead scores
// across the cyclomatic / MI / Halstead computations. The `usize as f64`
// and `f64 as usize` casts are intentional and snapshot-anchored — every
// site is bounded by the count it came from. Allowing the lints at the
// module level keeps the metric arithmetic legible.
#![allow(
    clippy::cast_precision_loss,
    clippy::cast_possible_truncation,
    clippy::cast_sign_loss
)]

use std::fmt;

use crate::checker::Checker;
use crate::macros::implement_metric_trait;

use crate::*;

/// The `Tokens` metric: per-function and per-file count of tree-sitter
/// leaf tokens, excluding any leaf that is itself a comment or has a
/// comment among its ancestors. Both halves matter: most grammars emit
/// comments as bare leaves, while some (Rust doc comments, Groovy
/// groovydoc, JSX `html_comment`) give them structured children whose
/// own leaves are not comment kinds.
///
/// This is a token-based size proxy: it counts the lexer's tokens
/// (identifiers, literals, keywords, punctuation) rather than lines or
/// Halstead operators/operands. Punctuation that Halstead skips
/// (parentheses, semicolons, separators) does contribute, so
/// `tokens` ≠ Halstead `N1 + N2`.
#[derive(Clone, Debug, PartialEq)]
#[non_exhaustive]
pub struct Stats {
    tokens: usize,
    tokens_sum: usize,
    tokens_min: usize,
    tokens_max: usize,
    space_count: usize,
}

impl Default for Stats {
    fn default() -> Self {
        Self {
            tokens: 0,
            tokens_sum: 0,
            tokens_min: usize::MAX,
            tokens_max: 0,
            space_count: 1,
        }
    }
}

impl fmt::Display for Stats {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(
            f,
            "tokens: {}, \
             tokens_average: {}, \
             tokens_min: {}, \
             tokens_max: {}",
            self.tokens_sum(),
            self.tokens_average(),
            self.tokens_min(),
            self.tokens_max(),
        )
    }
}

impl Stats {
    /// Merges a second `Tokens` metric suite into the first one.
    pub fn merge(&mut self, other: &Stats) {
        self.tokens_min = self.tokens_min.min(other.tokens_min);
        self.tokens_max = self.tokens_max.max(other.tokens_max);
        self.tokens_sum += other.tokens_sum;
        self.space_count += other.space_count;
    }

    /// Returns the total token count across all merged spaces.
    #[inline]
    #[must_use]
    pub fn tokens_sum(&self) -> u64 {
        self.tokens_sum as u64
    }

    /// Returns the average tokens per space.
    #[inline]
    #[must_use]
    pub fn tokens_average(&self) -> f64 {
        crate::metrics::average(self.tokens_sum() as f64, self.space_count)
    }

    /// Returns the smallest single-space token count.
    ///
    /// Diverges intentionally from `nom::Stats::functions_min`, which
    /// surfaces the raw `usize::MAX` sentinel for a never-observed
    /// space. We collapse the sentinel to `0` so a `Stats::default()`
    /// that bypasses the metric pipeline serializes to a meaningful
    /// number rather than `18446744073709551615`.
    #[inline]
    #[must_use]
    pub fn tokens_min(&self) -> u64 {
        if self.tokens_min == usize::MAX {
            0
        } else {
            self.tokens_min as u64
        }
    }

    /// Returns the largest single-space token count.
    #[inline]
    #[must_use]
    pub fn tokens_max(&self) -> u64 {
        self.tokens_max as u64
    }

    #[inline]
    pub(crate) fn compute_sum(&mut self) {
        self.tokens_sum += self.tokens;
    }

    #[inline]
    pub(crate) fn compute_minmax(&mut self) {
        self.tokens_min = self.tokens_min.min(self.tokens);
        self.tokens_max = self.tokens_max.max(self.tokens);
        self.compute_sum();
    }
}

#[doc(hidden)]
/// Per-language counting of tokens.
pub(crate) trait Tokens
where
    Self: Checker,
{
    /// Walk `node` and update `stats` with this metric for the language
    /// implementing the trait.
    ///
    /// `in_comment` is true when the node itself or any ancestor is a
    /// comment, so grammars whose comments have internal structure (e.g.
    /// Rust doc comments split into markers and content) exclude their
    /// inner leaves too. The walker propagates it down the traversal
    /// rather than having this function rediscover it per leaf: the old
    /// ancestor walk was `O(depth)` per leaf over an `O(depth)`
    /// `Node::parent`, which made the metric `O(leaves × depth²)` and let
    /// a few kilobytes of nested source burn minutes of CPU (#1052).
    fn compute(node: &Node, stats: &mut Stats, in_comment: bool) {
        if in_comment || node.child_count() != 0 {
            return;
        }
        stats.tokens += 1;
    }
}

implement_metric_trait!(
    [Tokens],
    PythonCode,
    MozjsCode,
    JavascriptCode,
    TypescriptCode,
    TsxCode,
    CppCode,
    MozcppCode,
    CCode,
    ObjcCode,
    RustCode,
    PreprocCode,
    CcommentCode,
    JavaCode,
    KotlinCode,
    GoCode,
    PerlCode,
    BashCode,
    LuaCode,
    TclCode,
    PhpCode,
    CsharpCode,
    ElixirCode,
    RubyCode,
    GroovyCode,
    IrulesCode
);

#[cfg(test)]
#[allow(
    clippy::float_cmp,
    clippy::cast_precision_loss,
    clippy::cast_possible_truncation,
    clippy::cast_sign_loss,
    clippy::similar_names,
    clippy::doc_markdown,
    clippy::needless_raw_string_hashes,
    clippy::too_many_lines
)]
mod tests {
    use crate::test_support::{check_metrics_only_shim, metrics_verbatim};

    use super::*;

    check_metrics_only_shim!(check_metrics, Tokens);
    // `*_tokens_distinct_from_halstead` compares `tokens_sum()` against
    // Halstead's `N1 + N2`. Deselecting Halstead leaves that side at 0,
    // where `tokens_sum() > 0` holds for the wrong reason — so these two
    // ask for Halstead rather than passing vacuously.
    check_metrics_only_shim!(check_tokens_and_halstead, Tokens, Halstead);

    /// `def foo(x): return x` → leaves: `def`, `foo`, `(`, `x`, `)`,
    /// `:`, `return`, `x` = 8 tokens, hand-counted.
    #[cfg(feature = "python")]
    #[test]
    fn python_tokens_exact_count() {
        check_metrics::<PythonParser>("def foo(x): return x", "foo.py", |metric| {
            assert_eq!(metric.tokens.tokens_sum(), 8);
            assert!(metric.tokens.tokens_max() >= 7);
        });
    }

    /// Adding a Python comment must not change the token count.
    #[cfg(feature = "python")]
    #[test]
    fn python_tokens_comments_excluded() {
        check_metrics::<PythonParser>(
            "def foo(x): return x  # explanation\n# header\n",
            "foo.py",
            |metric| {
                assert_eq!(metric.tokens.tokens_sum(), 8);
            },
        );
    }

    /// Blank lines and indentation must not change the token count.
    #[cfg(feature = "python")]
    #[test]
    fn python_tokens_whitespace_excluded() {
        check_metrics::<PythonParser>(
            "\n\n    def foo(x):\n        return x\n\n",
            "foo.py",
            |metric| {
                assert_eq!(metric.tokens.tokens_sum(), 8);
            },
        );
    }

    /// Tokens must exceed Halstead `N1 + N2` for code containing
    /// punctuation Halstead skips. Guards against accidental Halstead
    /// reuse.
    #[cfg(feature = "python")]
    #[test]
    fn python_tokens_distinct_from_halstead() {
        check_tokens_and_halstead::<PythonParser>(
            "def foo(x): return (x + 1)",
            "foo.py",
            |metric| {
                let halstead_total =
                    metric.halstead.total_operators() + metric.halstead.total_operands();
                assert!(
                    metric.tokens.tokens_sum() > halstead_total,
                    "expected tokens ({}) > halstead N1+N2 ({}); punctuation \
                     like `(`, `)`, `:` should contribute to tokens but not Halstead",
                    metric.tokens.tokens_sum(),
                    halstead_total,
                );
            },
        );
    }

    /// Inner functions get attributed to their innermost scope. For
    /// `def outer(): def inner(): return 1`, the inner scope owns
    /// `def, inner, (, ), :, return, 1` = 7 tokens; the outer scope
    /// owns `def, outer, (, ), :` = 5; the unit owns 0 directly.
    /// Asserting the exact `tokens_max` is what catches an attribution
    /// regression — a broken implementation that credited all 12
    /// tokens to one scope would still pass `max <= sum`.
    #[cfg(feature = "python")]
    #[test]
    fn python_tokens_nested_attribution() {
        check_metrics::<PythonParser>(
            "def outer():\n    def inner():\n        return 1\n",
            "foo.py",
            |metric| {
                assert_eq!(metric.tokens.tokens_sum(), 12);
                assert_eq!(metric.tokens.tokens_max(), 7);
                assert_eq!(metric.tokens.tokens_min(), 0);
            },
        );
    }

    /// C++ `/* … */` block comments must not contribute.
    /// Same fixture with and without comment yields the same count.
    #[cfg(feature = "cpp")]
    #[test]
    fn cpp_tokens_block_comments_excluded() {
        check_metrics::<CppParser>(
            "int foo(int x) { /* multi\n   line */ return x; }",
            "foo.cpp",
            |m| {
                // Leaves outside the comment:
                // int, foo, (, int, x, ), {, return, x, ;, } = 11.
                assert_eq!(m.tokens.tokens_sum(), 11);
            },
        );
        check_metrics::<CppParser>("int foo(int x) { return x; }", "foo.cpp", |m| {
            assert_eq!(m.tokens.tokens_sum(), 11);
        });
    }

    /// C++ `// …` line comments must not contribute, matching the Python
    /// hand-counted style.  Leaves outside the comment:
    /// `int`, `x`, `=`, `1`, `;` = 5.
    #[cfg(feature = "cpp")]
    #[test]
    fn cpp_tokens_line_comments_excluded() {
        check_metrics::<CppParser>("int x = 1; // a one-line comment\n", "foo.cpp", |m| {
            assert_eq!(m.tokens.tokens_sum(), 5);
        });
        check_metrics::<CppParser>("int x = 1;\n", "foo.cpp", |m| {
            assert_eq!(m.tokens.tokens_sum(), 5);
        });
    }

    /// Whitespace and blank lines must not contribute to the token count
    /// (mirrors `python_tokens_whitespace_excluded`).
    #[cfg(feature = "cpp")]
    #[test]
    fn cpp_tokens_whitespace_excluded() {
        check_metrics::<CppParser>("\n\nint foo(int x) {\n    return x;\n}\n", "foo.cpp", |m| {
            // int, foo, (, int, x, ), {, return, x, ;, } = 11.
            assert_eq!(m.tokens.tokens_sum(), 11);
        });
    }

    /// Tokens count punctuation that Halstead skips (parentheses, braces,
    /// semicolons), so `tokens_sum` must exceed `N1 + N2` for a fixture
    /// with significant punctuation.  Mirrors
    /// `python_tokens_distinct_from_halstead`.
    #[cfg(feature = "cpp")]
    #[test]
    fn cpp_tokens_distinct_from_halstead() {
        check_tokens_and_halstead::<CppParser>(
            "int foo(int x) { return (x + 1); }",
            "foo.cpp",
            |m| {
                let halstead_total = m.halstead.total_operators() + m.halstead.total_operands();
                assert!(
                    m.tokens.tokens_sum() > halstead_total,
                    "expected tokens ({}) > halstead N1+N2 ({}); punctuation like \
                     `(`, `)`, `{{`, `}}` and `;` should contribute to tokens but not Halstead",
                    m.tokens.tokens_sum(),
                    halstead_total,
                );
            },
        );
    }

    /// A C++ struct method and a free function each open their own
    /// `FuncSpace`, so the leaves of
    /// `struct S { int method(int a) { return a + 1; } }; int outer() { return 2; }`
    /// split across scopes: `method` owns 13 and `outer` owns 9, with the
    /// `struct`/unit framing owning the rest of the 27-token total.
    /// Asserting the exact `tokens_max` (13 — the largest single scope,
    /// strictly below the sum of 27) is what catches an attribution
    /// regression: a broken implementation that credited every leaf to one
    /// scope would raise `tokens_max` to 27 while still passing
    /// `max <= sum`, mirroring the Python sibling's exact-max guard.
    #[cfg(feature = "cpp")]
    #[test]
    fn cpp_tokens_nested_attribution() {
        check_metrics::<CppParser>(
            "struct S {\n    int method(int a) { return a + 1; }\n};\nint outer() { return 2; }\n",
            "foo.cpp",
            |m| {
                assert_eq!(m.tokens.tokens_sum(), 27);
                assert_eq!(m.tokens.tokens_max(), 13);
                assert_eq!(m.tokens.tokens_min(), 1);
            },
        );
    }

    /// Java `// …` line comments must not contribute.
    #[cfg(feature = "java")]
    #[test]
    fn java_tokens_line_comments_excluded() {
        check_metrics::<JavaParser>(
            "class A { void foo() { // hi\n return; } }",
            "A.java",
            |m| {
                // class, A, {, void, foo, (, ), {, return, ;, }, } = 12.
                assert_eq!(m.tokens.tokens_sum(), 12);
            },
        );
        check_metrics::<JavaParser>("class A { void foo() { return; } }", "A.java", |m| {
            assert_eq!(m.tokens.tokens_sum(), 12);
        });
    }

    #[cfg(feature = "groovy")]
    #[test]
    fn groovy_tokens_line_comments_excluded() {
        // Groovy mirror — `// …` line comments must not contribute.
        check_metrics::<GroovyParser>(
            "class A { void foo() { // hi\n return\n } }",
            "A.groovy",
            |m| {
                // class, A, {, void, foo, (, ), {, return, newline,
                // }, } = 11 tokens (Groovy's newline acts as the
                // statement terminator that Java spells `;`).
                assert_eq!(m.tokens.tokens_sum(), 11);
            },
        );
    }

    /// JS-family `<!-- -->` Annex-B `html_comment` leaves must not
    /// contribute tokens — they classify as comments now (#697). The
    /// count must match the comment-free source exactly.
    #[cfg(feature = "javascript")]
    #[test]
    fn javascript_tokens_html_comment_excluded() {
        check_metrics::<JavascriptParser>("<!-- hi -->\nlet x = 1;\n", "foo.js", |m| {
            // let, x, =, 1, ; = 5.
            assert_eq!(m.tokens.tokens_sum(), 5);
        });
        check_metrics::<JavascriptParser>("let x = 1;\n", "foo.js", |m| {
            assert_eq!(m.tokens.tokens_sum(), 5);
        });
    }

    /// Groovy `/** … */` `groovydoc_comment` leaves must not contribute
    /// tokens (#697 — `is_comment` previously missed this kind even
    /// though `Loc` counted it).
    #[cfg(feature = "groovy")]
    #[test]
    fn groovy_tokens_groovydoc_excluded() {
        check_metrics::<GroovyParser>(
            "/** doc */\nclass A { void f() { return\n } }\n",
            "A.groovy",
            |m| {
                // class, A, {, void, f, (, ), {, return, newline,
                // }, } = 11.
                assert_eq!(m.tokens.tokens_sum(), 11);
            },
        );
        check_metrics::<GroovyParser>("class A { void f() { return\n } }\n", "A.groovy", |m| {
            assert_eq!(m.tokens.tokens_sum(), 11);
        });
    }

    /// Total token count for `source`, analysed byte-for-byte.
    ///
    /// Restricted to `Tokens` so the deep-nesting case below measures
    /// this metric and not the rest of the walk. When it was written,
    /// `cognitive`'s own parent lookups (#1062) dominated that case and
    /// would have misattributed a regression; they are linear now, but
    /// isolating the metric under test is still what makes the reading
    /// mean something.
    #[cfg(feature = "rust")]
    fn tokens_of(source: &str) -> u64 {
        metrics_verbatim(
            crate::LANG::Rust,
            source.as_bytes(),
            crate::MetricsOptions::default().with_only(&[crate::Metric::Tokens]),
        )
        .tokens
        .tokens_sum()
    }

    #[cfg(feature = "rust")]
    fn nested_parens(depth: usize) -> String {
        format!(
            "fn f() -> i32 {{ {}1{} }}\n",
            "(".repeat(depth),
            ")".repeat(depth)
        )
    }

    /// The token count holds thousands of levels deep (#1052).
    ///
    /// The metric used to walk each leaf's ancestor chain to decide
    /// comment membership. `Node::parent` is itself `O(depth)`, so that
    /// cost `O(leaves × depth²)`: depth 1000 took ~19 s and depth 2000
    /// over two minutes. The walker now propagates the flag down the
    /// traversal in `O(1)` per node, and this runs in milliseconds.
    ///
    /// The count is asserted by formula rather than hand-counted: each
    /// added paren pair contributes exactly two leaves, so the delta
    /// between consecutive depths pins the metric without depending on
    /// how the grammar tokenises the surrounding function.
    ///
    /// Counts only. The pre-#1052 implementation produced these *same*
    /// counts, only quadratically, so this test cannot tell the two
    /// apart — that job belongs to the `tokens/nested-paren` probe in
    /// the benchmark harness (#1068), which fits an exponent across
    /// three depths: `cargo bench -p big-code-analysis-bench --bench
    /// scaling`. The wall-clock budget that used to sit here was
    /// calibrated to one machine and this suite also runs under `cargo
    /// llvm-cov` and on shared Windows / macOS runners; the equivalent
    /// assertion in `cognitive` produced false failures in four
    /// separate environments before it was retired.
    #[cfg(feature = "rust")]
    #[test]
    fn tokens_count_holds_at_depth() {
        let shallow = tokens_of(&nested_parens(1));
        assert_eq!(tokens_of(&nested_parens(2)), shallow + 2);
        assert_eq!(tokens_of(&nested_parens(2000)), shallow + 2 * 1999);
    }

    /// A comment deep in the tree still contributes nothing.
    ///
    /// Honest scope: the comment's *own* subtree is only one level deep
    /// (`line_comment` → marker / `doc_comment`), so the enclosing block
    /// nesting exercises the walker's inheritance rather than any extra
    /// level of comment-internal structure —
    /// `rust_tokens_doc_comments_excluded` already covers the latter at
    /// depth 1. What this adds is the anchored differential below: a
    /// count that would not survive an `in_comment` wired to a constant.
    #[cfg(feature = "rust")]
    #[test]
    fn rust_tokens_comment_excluded_at_depth() {
        let deep_block = format!(
            "fn f() {{ {}let x = 1;{} }}\n",
            "{ ".repeat(50),
            " }".repeat(50)
        );
        let with_doc = deep_block.replace("let x = 1;", "/// doc\nlet x = 1;");
        let baseline = tokens_of(&deep_block);
        // Anchor the differential: without this, an `in_comment` wired
        // to always-true would return 0 on both sides and pass.
        assert_eq!(
            baseline, 111,
            // 4 (`fn f ( )`) + 2 (outer braces) + 100 (50 nested brace
            // pairs) + 5 (`let x = 1 ;`).
            "expected 111 tokens for the comment-free baseline"
        );
        assert_eq!(
            tokens_of(&with_doc),
            baseline,
            "a doc comment 50 blocks deep must contribute no tokens, \
             including its structured inner leaves"
        );
    }

    /// Rust doc comments split into structured children whose leaves are
    /// not themselves comment kinds (`//`, `outer_doc_comment_marker`,
    /// `doc_comment`), so excluding only the comment node is not enough
    /// — every leaf beneath it must be filtered too.
    #[cfg(feature = "rust")]
    #[test]
    fn rust_tokens_doc_comments_excluded() {
        check_metrics::<RustParser>(
            "/// outer doc\n/// more doc\nfn f() { let x = 1; }",
            "foo.rs",
            |m| {
                // fn, f, (, ), {, let, x, =, 1, ;, } = 11.
                assert_eq!(m.tokens.tokens_sum(), 11);
            },
        );
        check_metrics::<RustParser>("fn f() { let x = 1; }", "foo.rs", |m| {
            assert_eq!(m.tokens.tokens_sum(), 11);
        });
    }

    // -- Per-language smoke tests --------------------------------------
    //
    // Lesson 1 (`docs/development/lessons_learned.md`): every supported
    // language must have a positive test that asserts non-zero tokens
    // on real source. Catches the silent-zero regression where a
    // metric is registered but never fires. `check_metrics` takes a
    // `fn` pointer so each test inlines its assertion directly.

    #[cfg(feature = "python")]
    #[test]
    fn smoke_python() {
        check_metrics::<PythonParser>("x = 1\n", "foo.py", |m| {
            assert!(m.tokens.tokens_sum() > 0);
        });
    }

    #[cfg(feature = "rust")]
    #[test]
    fn smoke_rust() {
        check_metrics::<RustParser>("fn f() { let x = 1; }", "foo.rs", |m| {
            assert!(m.tokens.tokens_sum() > 0);
        });
    }

    #[cfg(feature = "cpp")]
    #[test]
    fn smoke_cpp() {
        check_metrics::<CppParser>("int x = 1;", "foo.cpp", |m| {
            assert!(m.tokens.tokens_sum() > 0);
        });
    }

    #[cfg(feature = "java")]
    #[test]
    fn smoke_java() {
        check_metrics::<JavaParser>("class A { int x = 1; }", "A.java", |m| {
            assert!(m.tokens.tokens_sum() > 0);
        });
    }

    #[cfg(feature = "csharp")]
    #[test]
    fn smoke_csharp() {
        check_metrics::<CsharpParser>("class A { int X = 1; }", "A.cs", |m| {
            assert!(m.tokens.tokens_sum() > 0);
        });
    }

    #[cfg(feature = "javascript")]
    #[test]
    fn smoke_javascript() {
        check_metrics::<JavascriptParser>("let x = 1;", "foo.js", |m| {
            assert!(m.tokens.tokens_sum() > 0);
        });
    }

    #[cfg(feature = "mozjs")]
    #[test]
    fn smoke_mozjs() {
        check_metrics::<MozjsParser>("let x = 1;", "foo.js", |m| {
            assert!(m.tokens.tokens_sum() > 0);
        });
    }

    #[cfg(feature = "typescript")]
    #[test]
    fn smoke_typescript() {
        check_metrics::<TypescriptParser>("const x: number = 1;", "foo.ts", |m| {
            assert!(m.tokens.tokens_sum() > 0);
        });
    }

    #[cfg(feature = "typescript")]
    #[test]
    fn smoke_tsx() {
        check_metrics::<TsxParser>("const x: number = 1;", "foo.tsx", |m| {
            assert!(m.tokens.tokens_sum() > 0);
        });
    }

    #[cfg(feature = "go")]
    #[test]
    fn smoke_go() {
        check_metrics::<GoParser>("package main\nfunc f() {}", "foo.go", |m| {
            assert!(m.tokens.tokens_sum() > 0);
        });
    }

    #[cfg(feature = "kotlin")]
    #[test]
    fn smoke_kotlin() {
        check_metrics::<KotlinParser>("fun f(): Int = 1", "foo.kt", |m| {
            assert!(m.tokens.tokens_sum() > 0);
        });
    }

    #[cfg(feature = "lua")]
    #[test]
    fn smoke_lua() {
        check_metrics::<LuaParser>("local x = 1", "foo.lua", |m| {
            assert!(m.tokens.tokens_sum() > 0);
        });
    }

    #[cfg(feature = "bash")]
    #[test]
    fn smoke_bash() {
        check_metrics::<BashParser>("x=1", "foo.sh", |m| {
            assert!(m.tokens.tokens_sum() > 0);
        });
    }

    #[cfg(feature = "tcl")]
    #[test]
    fn smoke_tcl() {
        check_metrics::<TclParser>("set x 1", "foo.tcl", |m| {
            assert!(m.tokens.tokens_sum() > 0);
        });
    }

    #[cfg(feature = "perl")]
    #[test]
    fn smoke_perl() {
        check_metrics::<PerlParser>("my $x = 1;", "foo.pl", |m| {
            assert!(m.tokens.tokens_sum() > 0);
        });
    }

    #[cfg(feature = "php")]
    #[test]
    fn smoke_php() {
        check_metrics::<PhpParser>("<?php $x = 1;", "foo.php", |m| {
            assert!(m.tokens.tokens_sum() > 0);
        });
    }

    #[cfg(any(
        feature = "c",
        feature = "c-family-helpers",
        feature = "cpp",
        feature = "mozcpp"
    ))]
    #[test]
    fn smoke_preproc() {
        check_metrics::<PreprocParser>("#define FOO 1\n", "foo.h", |m| {
            assert!(m.tokens.tokens_sum() > 0);
        });
    }

    #[cfg(any(
        feature = "c",
        feature = "c-family-helpers",
        feature = "cpp",
        feature = "mozcpp"
    ))]
    #[test]
    fn smoke_ccomment() {
        // Ccomment's grammar parses bare C source; non-comment text
        // produces non-comment leaves.
        check_metrics::<CcommentParser>("int x = 1;", "foo.c", |m| {
            assert!(m.tokens.tokens_sum() > 0);
        });
    }

    #[cfg(feature = "c")]
    #[test]
    fn smoke_c() {
        check_metrics::<CParser>("int x = 1;\n", "foo.c", |m| {
            assert!(m.tokens.tokens_sum() > 0);
        });
    }

    #[cfg(feature = "objc")]
    #[test]
    fn smoke_objc() {
        check_metrics::<ObjcParser>("int x = 1;\n", "foo.m", |m| {
            assert!(m.tokens.tokens_sum() > 0);
        });
    }

    #[cfg(feature = "elixir")]
    #[test]
    fn smoke_elixir() {
        check_metrics::<ElixirParser>("defmodule Foo do\n  :ok\nend\n", "foo.ex", |m| {
            assert!(m.tokens.tokens_sum() > 0);
        });
    }

    #[cfg(feature = "ruby")]
    #[test]
    fn smoke_ruby() {
        check_metrics::<RubyParser>("def foo\n  a = 1\nend\n", "foo.rb", |m| {
            assert!(m.tokens.tokens_sum() > 0);
        });
    }

    #[cfg(feature = "irules")]
    #[test]
    fn smoke_irules() {
        check_metrics::<IrulesParser>("when X {\n    set x 1\n}\n", "foo.irule", |m| {
            assert!(m.tokens.tokens_sum() > 0);
        });
    }
}