tokenix 0.67.2

Symbol graphs, lexical code search, secrets scanning, output filters, and CLI hooks that save 60-90% LLM tokens
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125

//! The core loop, end to end against the real binary: index a project, then
//! answer the questions an agent asks (where is X, who calls it, show me only
//! that function) and intercept the tool calls that would waste tokens.

mod common;

use common::{line_count, Sandbox};
use serde_json::json;

fn stat(stats: &str, key: &str) -> Option<u64> {
    stats
        .lines()
        .find_map(|l| l.trim().strip_prefix(key)?.trim().parse().ok())
}

#[test]
fn index_then_stats_reports_every_file() {
    let sb = Sandbox::indexed("stats");
    let run = sb.tokenix(&["stats"]);
    assert_eq!(run.code, 0, "{}", run.stderr);
    assert_eq!(stat(&run.stdout, "files:"), Some(3), "{}", run.stdout);
    assert!(
        stat(&run.stdout, "chunks:").unwrap_or(0) > 3,
        "{}",
        run.stdout
    );
}

#[test]
fn symbol_graph_answers_where_and_who_calls() {
    let sb = Sandbox::indexed("graph");

    let symbols = sb.tokenix(&["symbols", "apply_tax"]).stdout;
    assert!(
        symbols.contains("src/billing.rs:5-7 [function] apply_tax"),
        "{symbols}"
    );

    let callers = sb.tokenix(&["callers", "apply_tax"]).stdout;
    assert!(callers.contains("invoice_total"), "{callers}");

    let callees = sb.tokenix(&["callees", "main"]).stdout;
    for callee in ["greet", "add", "invoice_total"] {
        assert!(
            callees.contains(callee),
            "main must call {callee}: {callees}"
        );
    }
}

#[test]
fn read_outlines_a_large_file_and_extracts_one_symbol_exactly() {
    let sb = Sandbox::indexed("read");
    let file = sb.path("src/billing.rs");

    let outline = sb.tokenix(&["read", "src/billing.rs"]).stdout;
    assert!(outline.contains("[function] apply_tax"), "{outline}");
    assert!(
        outline.lines().count() < line_count(&file),
        "an outline must be shorter than the file it replaces"
    );

    let body = sb
        .tokenix(&["read", "src/billing.rs", "--symbol", "apply_tax"])
        .stdout;
    assert!(
        body.contains("fn apply_tax(amount: u32) -> u32 {\n    amount + amount / 10\n}"),
        "--symbol must return the exact source: {body}"
    );
    assert!(!body.contains("helper_1"), "and nothing else: {body}");
}

#[test]
fn read_hook_never_replaces_a_read_with_an_outline_or_a_marker() {
    let sb = Sandbox::indexed("hook-read");
    let big = sb.path("src/billing.rs").to_string_lossy().into_owned();
    // An outline hides comments, test bodies and every use of a name, and an
    // A/B with an LLM showed multi-site edits going incomplete because of it.
    // The same holds for the second read of an unchanged file: the agent gets
    // the file, always.
    for attempt in ["first", "second (unchanged)"] {
        let run = sb.hook("Read", json!({ "file_path": big }));
        assert_eq!(
            run.code, 0,
            "{attempt} whole read must pass: {}",
            run.stderr
        );
        assert!(run.stderr.is_empty(), "{attempt}: {}", run.stderr);
    }
    let ranged = sb.hook(
        "Read",
        json!({ "file_path": big, "offset": 1, "limit": 20 }),
    );
    assert_eq!(ranged.code, 0);
}

#[test]
fn grep_hook_never_answers_in_place_of_the_native_grep() {
    let sb = Sandbox::indexed("hook-grep");
    // An identifier used to be answered from the index, hiding every textual
    // match of it; the agent's own grep must always be the one that runs.
    let run = sb.hook("Grep", json!({ "pattern": "apply_tax" }));
    assert_eq!(run.code, 0, "{}", run.stderr);
    assert!(run.stderr.is_empty(), "{}", run.stderr);
}

#[test]
fn hook_fails_open_without_an_index() {
    let sb = Sandbox::new("hook-no-index");
    sb.write("src/billing.rs", &common::billing_rs());
    let big = sb.path("src/billing.rs").to_string_lossy().into_owned();
    let run = sb.hook("Read", json!({ "file_path": big }));
    assert_eq!(run.code, 0, "no index must never block the agent");
}

#[test]
fn pack_stays_within_its_token_budget() {
    let sb = Sandbox::indexed("pack");
    let run = sb.tokenix(&["pack", "--budget", "300", "--format", "json"]);
    assert_eq!(run.code, 0, "{}", run.stderr);
    let pack: serde_json::Value = serde_json::from_str(&run.stdout).expect("pack JSON");
    let tokens = pack["tokens"].as_u64().expect("tokens");
    assert!(tokens > 0 && tokens <= 300, "budget exceeded: {tokens}");
    assert!(!pack["files"].as_array().expect("files").is_empty());
}