vallum 0.5.1

Security boundary between AI coding agents and your shell — redacts secrets, neutralizes prompt injection, sanitizes untrusted terminal output, audits every command.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115

//! Detection eval harness (dev/CI only; not shipped in the `vallum` binary).
//!
//!   cargo run --example eval            # print the metrics table to stdout
//!   cargo run --example eval -- --write # regenerate evals/report.md
//!   cargo run --example eval -- --check # CI: fail on stale report OR floor breach

use std::path::PathBuf;
use std::process::ExitCode;

use vallum::eval::{self, Report};

// Honest floors — NOT 100%. See docs/superpowers/specs (local) for rationale.
// 2026-07-03: corpus grown to ~85 injection / ~54 benign (curated deepset
// imports + hand-authored zh/DAN/multilingual/mutation rows). Imported
// samples are deliberately hard; measured injection recall is 0.812. Floor set
// just below the measurement per the calibrate-to-measurement policy; precision
// stays 1.000 and benign FP 0.000 (hard invariants). Regenerate with
// `cargo run --example eval -- --write`.
const MIN_INJECTION_RECALL: f64 = 0.81;
const MAX_BENIGN_FP_RATE: f64 = 0.05;
const MIN_SECRET_RECALL: f64 = 1.0;
const MIN_ENTROPY_RECALL: f64 = 0.90;
const MAX_ENTROPY_BENIGN_FP: f64 = 0.05;

fn report_path() -> PathBuf {
    PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("evals/report.md")
}

fn floor_violations(r: &Report) -> Vec<String> {
    let mut v = Vec::new();
    if r.injection.recall < MIN_INJECTION_RECALL {
        v.push(format!(
            "injection recall {:.3} < floor {:.3}",
            r.injection.recall, MIN_INJECTION_RECALL
        ));
    }
    if r.injection.fp_rate > MAX_BENIGN_FP_RATE {
        v.push(format!(
            "benign FP-rate {:.3} > ceiling {:.3}",
            r.injection.fp_rate, MAX_BENIGN_FP_RATE
        ));
    }
    if r.secrets.recall < MIN_SECRET_RECALL {
        v.push(format!(
            "secret recall {:.3} < floor {:.3}",
            r.secrets.recall, MIN_SECRET_RECALL
        ));
    }
    if r.entropy.secret_recall < MIN_ENTROPY_RECALL {
        v.push(format!(
            "entropy recall {:.3} < floor {:.3}",
            r.entropy.secret_recall, MIN_ENTROPY_RECALL
        ));
    }
    if r.entropy.benign_fp_rate > MAX_ENTROPY_BENIGN_FP {
        v.push(format!(
            "entropy benign FP-rate {:.3} > ceiling {:.3}",
            r.entropy.benign_fp_rate, MAX_ENTROPY_BENIGN_FP
        ));
    }
    v
}

fn main() -> ExitCode {
    let report = eval::build_report();
    let rendered = eval::render_report(&report);
    let path = report_path();

    match std::env::args().nth(1).as_deref() {
        None => {
            print!("{rendered}");
            ExitCode::SUCCESS
        }
        Some("--write") => {
            std::fs::write(&path, &rendered)
                .unwrap_or_else(|e| panic!("write {}: {e}", path.display()));
            eprintln!("wrote {}", path.display());
            ExitCode::SUCCESS
        }
        Some("--check") => {
            let mut ok = true;
            match std::fs::read_to_string(&path) {
                Ok(committed) if committed == rendered => {}
                Ok(_) => {
                    eprintln!(
                        "eval: {} is stale — run `cargo run --example eval -- --write`",
                        path.display()
                    );
                    ok = false;
                }
                Err(e) => {
                    eprintln!("eval: cannot read {}: {e}", path.display());
                    ok = false;
                }
            }
            let violations = floor_violations(&report);
            for v in &violations {
                eprintln!("eval: floor breach — {v}");
            }
            if !violations.is_empty() {
                ok = false;
            }
            if ok {
                eprintln!("eval: report fresh, all floors satisfied");
                ExitCode::SUCCESS
            } else {
                ExitCode::FAILURE
            }
        }
        Some(other) => {
            eprintln!("eval: unknown argument {other:?} (use --write or --check)");
            ExitCode::from(2)
        }
    }
}