tokenix 0.65.4

Semantic search, symbol graphs, secrets scanning, output filters, and CLI hooks that save 60-90% LLM tokens
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
//! End-to-end hook tests: spawn the real `tokenix hook` binary with agent
//! payload shapes on stdin and assert the PreToolUse JSON contract it emits.
//!
//! This is the layer where real-world regressions actually happened (Claude's
//! `{stdout,stderr}` tool_response shape, PowerShell tool-name casing, the
//! hook-post silent no-op) — unit tests on inner functions cannot catch a
//! broken end-to-end contract because the handler exits the process.

use std::io::Write;
use std::process::{Command, Stdio};

/// Machine-wide state (hook log, recall stash) goes here instead of the
/// developer's `~/.tokenix`; shared per test process so dedup can see it.
fn isolated_home() -> std::path::PathBuf {
    let home = std::env::temp_dir().join(format!("tokenix-hook-e2e-home-{}", std::process::id()));
    std::fs::create_dir_all(&home).expect("isolated TOKENIX_HOME");
    home
}

/// Run `tokenix hook` with `payload` on stdin from an isolated temp cwd (the
/// hook writes its event log relative to the repo root it detects, so tests
/// must not run inside this repository).
fn run_hook(payload: &str) -> (String, i32) {
    let dir = std::env::temp_dir().join(format!(
        "tokenix-hook-e2e-{}-{:x}",
        std::process::id(),
        payload.len()
    ));
    std::fs::create_dir_all(&dir).expect("temp cwd");
    let mut child = Command::new(env!("CARGO_BIN_EXE_tokenix"))
        .arg("hook")
        .current_dir(&dir)
        .env("TOKENIX_HOME", isolated_home())
        .stdin(Stdio::piped())
        .stdout(Stdio::piped())
        .stderr(Stdio::piped())
        .spawn()
        .expect("spawn tokenix hook");
    child
        .stdin
        .as_mut()
        .expect("stdin")
        .write_all(payload.as_bytes())
        .expect("write payload");
    let out = child.wait_with_output().expect("hook exit");
    (
        String::from_utf8_lossy(&out.stdout).into_owned(),
        out.status.code().unwrap_or(-1),
    )
}

fn claude_bash_payload(command: &str) -> String {
    format!(
        r#"{{"session_id":"e2e-test","transcript_path":"/tmp/t.jsonl","cwd":"/tmp","hook_event_name":"PreToolUse","tool_name":"Bash","tool_input":{{"command":{cmd},"description":"e2e"}}}}"#,
        cmd = serde_json::to_string(command).unwrap()
    )
}

fn updated_command(stdout: &str) -> Option<String> {
    let v: serde_json::Value = serde_json::from_str(stdout.trim()).ok()?;
    let hso = &v["hookSpecificOutput"];
    assert_eq!(
        hso["hookEventName"], "PreToolUse",
        "rewrite JSON must carry the PreToolUse hookEventName (Claude ignores it otherwise)"
    );
    hso["updatedInput"]["command"].as_str().map(str::to_string)
}

#[test]
fn claude_bash_filtered_command_is_rewritten_to_tokenix_run() {
    let (stdout, code) = run_hook(&claude_bash_payload("terraform plan -out tf.plan"));
    assert_eq!(code, 0);
    let cmd = updated_command(&stdout).expect("expected a rewrite for a filter-matching command");
    assert!(
        cmd.contains(" run ") && cmd.contains("terraform plan -out tf.plan"),
        "must wrap the original command in `tokenix run`: {cmd}"
    );
}

#[test]
fn claude_git_status_is_rewritten_to_short() {
    let (stdout, code) = run_hook(&claude_bash_payload("git status"));
    assert_eq!(code, 0);
    let cmd = updated_command(&stdout).expect("git status must be rewritten");
    assert_eq!(cmd, "git status --short");
}

/// Regression: `git-log.toml` used to make the hook wrap `git log` in
/// `tokenix run '<command>'`, hiding the real `git` invocation behind an
/// opaque wrapper. A downstream guard that must verify a Bash command really
/// is (and stays) `git` — e.g. Claude Code's worktree-isolation check on a
/// subagent's git operations — cannot see through `'tokenix' run '...'` and
/// refuses it outright, even though the command was exactly the transparent,
/// safe `git log` such a guard exists to allow. `git log` (like every other
/// git subcommand) must stay a plain, visible `git` invocation, exactly like
/// `git status` already does.
#[test]
fn claude_git_log_is_never_wrapped_in_tokenix_run() {
    let (stdout, code) = run_hook(&claude_bash_payload("git log -1"));
    assert_eq!(code, 0);
    assert!(
        stdout.trim().is_empty(),
        "git log must stay a plain `git` invocation, not opaque tokenix run: {stdout}"
    );
}

/// The same guarantee must hold when `git log` is the tail of a compound
/// command: a filter match on that one segment used to wrap the *whole*
/// compound command in `tokenix run`, hiding the git call just as much.
#[test]
fn claude_compound_command_with_trailing_git_segment_stays_plain() {
    let (stdout, code) = run_hook(&claude_bash_payload(
        "pwd && git status && git log --oneline -5",
    ));
    assert_eq!(code, 0);
    assert!(
        stdout.trim().is_empty(),
        "a compound command ending in `git log` must not be wrapped either: {stdout}"
    );
}

#[test]
fn tokenix_disabled_prefix_passes_through() {
    let (stdout, code) = run_hook(&claude_bash_payload("TOKENIX_DISABLED=1 terraform plan"));
    assert_eq!(code, 0);
    assert!(
        stdout.trim().is_empty(),
        "bypassed command must not be rewritten, got: {stdout}"
    );
}

#[test]
fn recursive_tokenix_command_passes_through() {
    let (stdout, code) = run_hook(&claude_bash_payload("tokenix run \"git status\""));
    assert_eq!(code, 0);
    assert!(stdout.trim().is_empty(), "no recursion rewrite: {stdout}");
}

#[test]
fn unfiltered_command_passes_through() {
    let (stdout, code) = run_hook(&claude_bash_payload("some-unknown-tool-xyz --flag value"));
    assert_eq!(code, 0);
    assert!(stdout.trim().is_empty(), "no filter → no rewrite: {stdout}");
}

#[test]
fn help_invocation_passes_through() {
    let (stdout, code) = run_hook(&claude_bash_payload("terraform plan --help"));
    assert_eq!(code, 0);
    assert!(
        stdout.trim().is_empty(),
        "help output must never be filtered: {stdout}"
    );
}

#[test]
fn malformed_stdin_fails_open() {
    let (stdout, code) = run_hook("this is not json {");
    assert_eq!(code, 0, "hook must fail open, never block the agent");
    assert!(stdout.trim().is_empty());
}

#[test]
fn empty_tool_name_passes_through() {
    let (stdout, code) =
        run_hook(r#"{"hook_event_name":"PreToolUse","tool_name":"","tool_input":{}}"#);
    assert_eq!(code, 0);
    assert!(stdout.trim().is_empty());
}

/// Cross-call dedup + `tokenix retrieve`: a repeated successful command must
/// collapse to a marker, and the marker's key must return the original bytes.
/// Runs the real binary twice, like the hook does.
#[test]
fn repeated_successful_command_dedupes_and_stays_retrievable() {
    let dir = std::env::temp_dir().join(format!("tokenix-dedup-e2e-{}", std::process::id()));
    std::fs::create_dir_all(&dir).expect("temp cwd");

    // The stash lives in ~/.tokenix and outlives the test run, so the fixture
    // must be unique per process — otherwise a previous run's entry makes the
    // *first* call dedup and the test asserts against the wrong baseline.
    let marker_word = format!("DEDUPFIXTURE{}", std::process::id());
    // ~4 KB: comfortably over the 200-token dedup floor, and well under the
    // ~8 KB Windows command-line limit that a bigger fixture would blow.
    let payload: String = (0..90)
        .map(|i| format!("{marker_word}-{i}-filler-text-for-the-token-floor"))
        .collect::<Vec<_>>()
        .join(" ");
    let command = if cfg!(windows) {
        format!("echo {payload}")
    } else {
        format!("echo '{payload}'")
    };

    let run = |cmd: &str| {
        let out = Command::new(env!("CARGO_BIN_EXE_tokenix"))
            .args(["run", cmd])
            .current_dir(&dir)
            .env("TOKENIX_HOME", isolated_home())
            .output()
            .expect("tokenix run");
        String::from_utf8_lossy(&out.stdout).into_owned()
    };

    let first = run(&command);
    assert!(
        first.contains(&marker_word) && !first.contains("output identical to"),
        "first run must show real output, not a marker: {}",
        &first[..first.len().min(200)]
    );

    let second = run(&command);
    if !second.contains("output identical to") {
        // Environment-dependent (no home dir / unwritable ~/.tokenix): the
        // feature degrades to plain pass-through, which is still correct.
        eprintln!("dedup did not engage in this environment; skipping key check");
        let _ = std::fs::remove_dir_all(&dir);
        return;
    }
    assert!(
        second.len() < first.len() / 2,
        "dedup marker must be far cheaper than the output"
    );

    let key = second
        .split("tokenix retrieve ")
        .nth(1)
        .and_then(|rest| rest.split(|c: char| !c.is_ascii_alphanumeric()).next())
        .expect("marker must carry a retrieve key")
        .to_string();

    let retrieved = Command::new(env!("CARGO_BIN_EXE_tokenix"))
        .args(["retrieve", &key])
        .current_dir(&dir)
        .env("TOKENIX_HOME", isolated_home())
        .output()
        .expect("tokenix retrieve");
    let body = String::from_utf8_lossy(&retrieved.stdout);
    assert!(
        retrieved.status.success() && body.contains(&marker_word),
        "retrieve must return the original bytes for key {key}"
    );

    let _ = std::fs::remove_dir_all(&dir);
}

/// Regression for the cross-command collision: two *different* commands whose
/// output is byte-identical must never dedupe into each other. Before the fix,
/// `find_identical` matched purely on the compressed-output digest, ignoring
/// which command produced it — so the second (different) command's marker
/// claimed its output was "identical to" the *first* command, and because
/// `remember()` is skipped on a dedup hit, the second command's own output was
/// never stashed. `tokenix retrieve` on the advertised key then handed back
/// bytes that belonged to a command the agent never ran.
#[test]
fn different_commands_with_identical_output_never_dedupe_e2e() {
    let dir = std::env::temp_dir().join(format!(
        "tokenix-cross-cmd-dedup-e2e-{}",
        std::process::id()
    ));
    std::fs::create_dir_all(&dir).expect("temp cwd");
    let home = isolated_home();

    // Same fixture text, produced by two genuinely different commands.
    let marker_word = format!("CROSSCMDFIXTURE{}", std::process::id());
    let payload: String = (0..90)
        .map(|i| format!("{marker_word}-{i}-filler-text-for-the-token-floor"))
        .collect::<Vec<_>>()
        .join(" ");
    let (command_a, command_b) = if cfg!(windows) {
        (
            format!("echo {payload}"),
            format!("printf '%s\\n' '{payload}'"),
        )
    } else {
        (
            format!("echo '{payload}'"),
            format!("printf '%s\\n' '{payload}'"),
        )
    };

    let run = |cmd: &str| {
        let out = Command::new(env!("CARGO_BIN_EXE_tokenix"))
            .args(["run", cmd])
            .current_dir(&dir)
            .env("TOKENIX_HOME", &home)
            .output()
            .expect("tokenix run");
        String::from_utf8_lossy(&out.stdout).into_owned()
    };

    let first = run(&command_a);
    assert!(
        first.contains(&marker_word) && !first.contains("output identical to"),
        "first run (command A) must show real output, not a marker: {}",
        &first[..first.len().min(200)]
    );

    // Command B is a different command with the same output text. It must
    // show real output too — never a marker claiming it matches command A.
    let second = run(&command_b);
    assert!(
        !second.contains("output identical to"),
        "a different command must never be deduped against an unrelated \
         command's stash just because their output coincides: {}",
        &second[..second.len().min(300)]
    );
    assert!(
        second.contains(&marker_word),
        "command B's real output must still be shown: {}",
        &second[..second.len().min(200)]
    );

    let _ = std::fs::remove_dir_all(&dir);
}

/// An uncapped content grep must come back with a `head_limit` injected — and
/// it must work from a temp cwd with no index at all, since the stale-index gate
/// exits before the tool handlers.
#[test]
fn uncapped_content_grep_gets_head_limit() {
    let payload = r#"{"hook_event_name":"PreToolUse","tool_name":"Grep","tool_input":{"pattern":"foo","output_mode":"content","-C":3}}"#;
    let (stdout, code) = run_hook(payload);
    assert_eq!(code, 0);
    let v: serde_json::Value =
        serde_json::from_str(stdout.trim()).unwrap_or_else(|_| panic!("expected JSON: {stdout}"));
    let updated = &v["hookSpecificOutput"]["updatedInput"];
    assert_eq!(v["hookSpecificOutput"]["hookEventName"], "PreToolUse");
    assert!(
        updated["head_limit"].is_number(),
        "grep must be capped: {stdout}"
    );
    assert_eq!(updated["pattern"], "foo", "original args preserved");
    assert_eq!(updated["-C"], 3);
}

#[test]
fn bounded_grep_passes_through_untouched() {
    let payload = r#"{"hook_event_name":"PreToolUse","tool_name":"Grep","tool_input":{"pattern":"foo","output_mode":"content","head_limit":20}}"#;
    let (stdout, code) = run_hook(payload);
    assert_eq!(code, 0);
    assert!(
        stdout.trim().is_empty(),
        "agent-bounded grep must not be rewritten: {stdout}"
    );
}

#[test]
fn files_with_matches_grep_passes_through() {
    let payload = r#"{"hook_event_name":"PreToolUse","tool_name":"Grep","tool_input":{"pattern":"foo","output_mode":"files_with_matches"}}"#;
    let (stdout, code) = run_hook(payload);
    assert_eq!(code, 0);
    assert!(
        stdout.trim().is_empty(),
        "cheap output mode must not be rewritten: {stdout}"
    );
}

/// The PowerShell tool routes through `run --shell pwsh` with native-exe
/// quoting — Windows-only behavior (exact tool name "PowerShell"; the
/// lowercase variants route through the bash path).
#[cfg(windows)]
#[test]
fn powershell_tool_gets_pwsh_shell_rewrite() {
    let payload = r#"{"hook_event_name":"PreToolUse","tool_name":"PowerShell","tool_input":{"command":"Get-Content src/main.rs"}}"#;
    let (stdout, code) = run_hook(payload);
    assert_eq!(code, 0);
    let cmd = updated_command(&stdout).expect("PowerShell command must be rewritten");
    assert!(
        cmd.starts_with("& '") && cmd.contains("run --shell pwsh"),
        "must be a native-exe pwsh call: {cmd}"
    );
    assert!(
        cmd.contains("Get-Content src/main.rs"),
        "original command preserved: {cmd}"
    );
}

/// Same guarantee as `claude_git_log_is_never_wrapped_in_tokenix_run`, for the
/// PowerShell tool's own `run --shell pwsh` wrap path.
#[cfg(windows)]
#[test]
fn powershell_git_log_is_never_wrapped_in_tokenix_run() {
    let payload = r#"{"hook_event_name":"PreToolUse","tool_name":"PowerShell","tool_input":{"command":"git log -1"}}"#;
    let (stdout, code) = run_hook(payload);
    assert_eq!(code, 0);
    assert!(
        stdout.trim().is_empty(),
        "git log under PowerShell must stay plain, not wrapped in run --shell pwsh: {stdout}"
    );
}

#[cfg(windows)]
#[test]
fn powershell_disabled_env_passes_through() {
    let payload = r#"{"hook_event_name":"PreToolUse","tool_name":"PowerShell","tool_input":{"command":"$env:TOKENIX_DISABLED='1'; Get-Content big.log"}}"#;
    let (stdout, code) = run_hook(payload);
    assert_eq!(code, 0);
    assert!(
        stdout.trim().is_empty(),
        "bypass must skip rewrite: {stdout}"
    );
}

/// Run `tokenix hook-post` with `payload` on stdin, same isolation shape as
/// `run_hook` — the hook writes its event log relative to the repo root it
/// detects.
fn run_hook_post(payload: &str) -> (String, i32) {
    let dir = std::env::temp_dir().join(format!(
        "tokenix-hook-post-e2e-{}-{:x}",
        std::process::id(),
        payload.len()
    ));
    std::fs::create_dir_all(&dir).expect("temp cwd");
    let mut child = Command::new(env!("CARGO_BIN_EXE_tokenix"))
        .arg("hook-post")
        .current_dir(&dir)
        .env("TOKENIX_HOME", isolated_home())
        .stdin(Stdio::piped())
        .stdout(Stdio::piped())
        .stderr(Stdio::piped())
        .spawn()
        .expect("spawn tokenix hook-post");
    child
        .stdin
        .as_mut()
        .expect("stdin")
        .write_all(payload.as_bytes())
        .expect("write payload");
    let out = child.wait_with_output().expect("hook-post exit");
    (
        String::from_utf8_lossy(&out.stdout).into_owned(),
        out.status.code().unwrap_or(-1),
    )
}

/// Regression for the PostToolUse `updatedToolOutput` fix (issue #73): a
/// non-Bash, non-ListDirectory tool result (e.g. a `Read`) previously exited
/// this hook immediately on the stale belief that Claude Code PostToolUse
/// "cannot replace or shorten a tool result" — silently skipping redaction
/// for exactly the surface the PreToolUse Bash rewrite never sees. A secret
/// embedded in a `Read` result must now be stripped before the model would
/// ever receive it.
#[test]
fn claude_read_result_secret_is_redacted_via_updated_tool_output() {
    let secret = "AKIAIOSFODNN7EXAMPLE"; // gitleaks:allow synthetic test fixture
    let payload = format!(
        r#"{{"hook_event_name":"PostToolUse","tool_name":"Read","tool_input":{{"file_path":"notes.txt"}},"tool_response":{text}}}"#,
        text = serde_json::to_string(&format!("config dump:\naws_key={secret}\n")).unwrap()
    );
    let (stdout, code) = run_hook_post(&payload);
    assert_eq!(code, 0, "hook-post must always exit 0");

    let v: serde_json::Value =
        serde_json::from_str(stdout.trim()).expect("must emit JSON when a secret was redacted");
    assert_eq!(
        v["hookSpecificOutput"]["hookEventName"], "PostToolUse",
        "must carry the PostToolUse hookEventName"
    );
    let updated = v["hookSpecificOutput"]["updatedToolOutput"]
        .as_str()
        .expect("updatedToolOutput must be a string");
    assert!(
        !updated.contains(secret),
        "secret survived into updatedToolOutput: {updated}"
    );
    assert!(
        updated.contains("[REDACTED]"),
        "expected a redaction marker: {updated}"
    );
}