amont-agent 2.26.0

//! Replaying the transcripts through the current rules.
//!
//! This is the part that decides whether a rule is allowed to block. A rule
//! whose fire rate nobody has looked at is a rule nobody can defend, and the
//! whole shadow-first ladder is meaningless without a number to graduate on.
//!
//! Rates are per 1,000 tool calls in the same bucket, so weeks of very
//! different length compare directly. The denominator is every Bash call seen,
//! including subagents' — the hook fires for those too.

use std::collections::BTreeMap;
use std::ops::Range;

use crate::json;

use crate::civil::{iso, week_start, Day};
use crate::rules::{self, Rule};
use crate::shell;
use crate::transcript::{self, Scan, ScanError, Stats};

pub struct Sample {
    pub day: Day,
    pub cwd: String,
    /// The whole command, redacted. `--format cases` needs it in full so the
    /// case can be replayed through the rules later; the excerpt below is for
    /// a human skimming the terminal.
    ///
    /// Redacted even though this is only ever printed, because the intended
    /// destination is a file that gets committed — and a curated corpus is
    /// exactly the kind of place a token goes unnoticed for a year.
    pub command: String,
    /// Centred on the match, not on the head of the command — a sample printed
    /// from the head of a 2 KB script shows text unrelated to the match, and
    /// then human review reviews the wrong thing.
    pub excerpt: String,
}

#[derive(Default, Clone)]
pub struct Bucket {
    pub calls: u32,
    pub fires: Vec<u32>,
}

/// What a backtest replays.
///
/// A rule and an assertion answer different questions, but both do it with a
/// PURE `examine` over the command string, and that is the only half history
/// can be replayed against — `confirm` and `verify` touch the world, and the
/// world has moved. So the backtester takes either.
#[derive(Clone, Copy)]
pub enum Subject {
    Rule(&'static Rule),
    /// An assertion's rate is measured against every Bash call, while it only
    /// ever RUNS after a successful one. That makes the number a slight
    /// under-estimate of how often it fires in practice — the right direction
    /// for a figure used to decide whether something is too intrusive.
    Assertion(&'static crate::assertions::Assertion),
}

impl Subject {
    pub fn id(&self) -> &'static str {
        match self {
            Subject::Rule(r) => r.id,
            Subject::Assertion(a) => a.id,
        }
    }

    pub fn default_stance(&self) -> crate::rules::Stance {
        match self {
            Subject::Rule(r) => r.default_stance,
            Subject::Assertion(a) => a.default_stance,
        }
    }

    pub fn evidence(&self) -> crate::rules::Evidence {
        match self {
            Subject::Rule(r) => r.evidence,
            Subject::Assertion(a) => a.evidence,
        }
    }

    /// The same dispatch as the hook's: a rule through [`Rule::judge`], and an
    /// assertion — which reads only the legacy lexer — never on a command
    /// that lexer could not read.
    ///
    /// [`Rule::judge`]: crate::rules::Rule::judge
    fn span(&self, input: &crate::rules::Input) -> Option<Range<usize>> {
        match self {
            Subject::Rule(r) => r.judge(input).map(|f| f.span),
            Subject::Assertion(a) => match input.legacy {
                shell::Parsed::Opaque(_) => None,
                parsed => (a.examine)(parsed).map(|c| c.span),
            },
        }
    }
}

pub struct Report {
    pub rules: Vec<Subject>,
    pub weeks: BTreeMap<Day, Bucket>,
    pub totals: Vec<u32>,
    pub samples: Vec<Vec<Sample>>,
    pub stats: Stats,
    pub opaque: u64,
    /// Commands where SOME pipeline could not be read, and the rest was
    /// judged. The measurement of per-pipeline opacity: how much the guard
    /// now sees that whole-command opacity used to throw away.
    pub partly: u64,
}

pub fn run(scan: &Scan, rules: &[Subject], samples_per_rule: usize) -> Result<Report, ScanError> {
    let mut weeks: BTreeMap<Day, Bucket> = BTreeMap::new();
    let mut totals = vec![0u32; rules.len()];
    let mut samples: Vec<Vec<Sample>> = rules.iter().map(|_| Vec::new()).collect();
    let mut opaque = 0u64;
    let mut partly = 0u64;

    let stats = transcript::for_each_call(scan, |call| {
        let bucket = weeks.entry(week_start(call.day)).or_insert_with(|| Bucket {
            calls: 0,
            fires: vec![0; rules.len()],
        });
        bucket.calls += 1;

        if call.command.is_empty() {
            return;
        }
        let parsed = shell::lex(call.command);
        // Counted, then still judged: whether a rule may run on a command the
        // legacy lexer could not read is `Rule::judge`'s call, as it is in the
        // hook. Legacy rules and assertions skip it exactly as before.
        if matches!(parsed, shell::Parsed::Opaque(_)) {
            opaque += 1;
        } else if !parsed.fully_read() {
            // Counted, and then judged: the readable pipelines still run.
            partly += 1;
        }
        // A transcript does not record which shell ran the command.
        let input = crate::rules::Input::new(call.command, crate::rules::Dialect::Unknown, &parsed);
        for (i, rule) in rules.iter().enumerate() {
            // `examine` only. `confirm` touches the world, and the world has
            // moved since these commands ran — replaying it would produce a
            // number that describes today rather than then.
            if let Some(span) = rule.span(&input) {
                bucket.fires[i] += 1;
                totals[i] += 1;
                if samples[i].len() < samples_per_rule {
                    samples[i].push(Sample {
                        day: call.day,
                        cwd: short_cwd(call.cwd),
                        command: crate::journal::redact(call.command),
                        excerpt: excerpt(call.command, span.start, span.end),
                    });
                }
            }
        }
    })?;

    Ok(Report {
        rules: rules.to_vec(),
        weeks,
        totals,
        samples,
        stats,
        opaque,
        partly,
    })
}

/// A window of the command showing the match.
///
/// When the matched span is longer than the window — a `git commit` carrying
/// two `-m` bodies before its pipe, say — the ELLIPSIS GOES IN THE MIDDLE. The
/// two ends of a span are the two things a reviewer needs to see together: for
/// `pipe-to-tail` they are the mutating verb and the sink it pipes into, and an
/// excerpt that shows only the head proves nothing about why the rule fired.
/// The first version of this printed the head and produced samples that looked
/// like false positives while being correct.
pub fn excerpt(src: &str, from: usize, to: usize) -> String {
    // Derived from the terminal, not invented here: `term::term_width`
    // already owns the `$COLUMNS`-with-a-sane-floor answer, and a second copy
    // would be a second place for the fallback to drift. Samples are printed
    // under a six-space indent, hence the margin.
    let width = crate::term::term_width().saturating_sub(8).max(40);
    let from = floor_char(src, from.min(src.len()));
    let to = ceil_char(src, to.min(src.len()).max(from));
    let mid = one_line(src[from..to].trim());
    let mut out = if mid.chars().count() > width {
        let all: Vec<char> = mid.chars().collect();
        let half = width / 2;
        let head: String = all[..half].iter().collect();
        let tail: String = all[all.len() - half..].iter().collect();
        format!("{head} […] {tail}")
    } else {
        mid
    };
    if from > 0 {
        out = format!("… {out}");
    }
    if to < src.len() {
        out = format!("{out} …");
    }
    out
}

fn one_line(s: &str) -> String {
    s.split_whitespace().collect::<Vec<_>>().join(" ")
}

fn floor_char(s: &str, mut i: usize) -> usize {
    while i > 0 && !s.is_char_boundary(i) {
        i -= 1;
    }
    i
}

fn ceil_char(s: &str, mut i: usize) -> usize {
    while i < s.len() && !s.is_char_boundary(i) {
        i += 1;
    }
    i
}

fn short_cwd(cwd: &str) -> String {
    match std::env::var("HOME") {
        Ok(h) if !h.is_empty() && cwd.starts_with(&h) => format!("~{}", &cwd[h.len()..]),
        _ => cwd.to_string(),
    }
}

fn rate(fires: u32, calls: u32) -> f64 {
    if calls == 0 {
        0.0
    } else {
        1000.0 * f64::from(fires) / f64::from(calls)
    }
}

impl Report {
    /// The per-1000 rate for one rule across every bucket, oldest first.
    pub fn series(&self, i: usize) -> Vec<f64> {
        self.weeks
            .values()
            .map(|b| rate(b.fires[i], b.calls))
            .collect()
    }

    pub fn render(&self, roots: &transcript::Roots) -> String {
        let mut s = String::new();
        s.push_str(&format!(
            "transcripts: {} ({} files, {:.0} MB, {} calls, {} duplicates dropped)\n",
            roots
                .dirs
                .iter()
                .map(|d| short_cwd(&d.display().to_string()))
                .collect::<Vec<_>>()
                .join(", "),
            self.stats.files,
            self.stats.bytes as f64 / 1_048_576.0,
            self.stats.calls,
            self.stats.duplicates,
        ));
        s.push_str(
            "UTC weeks. Sessions run on the web are not on this machine and cannot be counted.\n",
        );
        // What the guard could not read. `opaque` was counted and never
        // printed; `partly` is the measurement of per-pipeline opacity.
        s.push_str(&format!(
            "unreadable: {} command(s) could not be read at all; {} more had a pipeline hidden.\n\n",
            self.opaque, self.partly,
        ));

        // Wide enough for the longest id plus a separating space. A fixed
        // width let `gh-pr-merge-auto` run into its neighbour's header.
        let width = self.rules.iter().map(|r| r.id().len()).max().unwrap_or(8) + 2;
        s.push_str(&format!("{:<14}{:>8}", "week starting", "calls"));
        for r in &self.rules {
            s.push_str(&format!("{:>width$}", r.id()));
        }
        s.push('\n');
        for (week, b) in &self.weeks {
            s.push_str(&format!("{:<14}{:>8}", iso(*week), b.calls));
            for i in 0..self.rules.len() {
                s.push_str(&format!("{:>width$.1}", rate(b.fires[i], b.calls)));
            }
            s.push('\n');
        }
        let calls: u32 = self.weeks.values().map(|b| b.calls).sum();
        s.push_str(&format!("{:<14}{:>8}", "total", calls));
        for t in &self.totals {
            s.push_str(&format!("{t:>width$}"));
        }
        s.push_str("\n\n");

        s.push_str(&format!(
            "{:<18}{:>8}{:>11}{:>13}  {}\n",
            "rule", "total", "p95/1000", "trend", "ships as"
        ));
        for (i, r) in self.rules.iter().enumerate() {
            let series = self.series(i);
            s.push_str(&format!(
                "{:<18}{:>8}{:>11.1}{:>13}  {}\n",
                r.id(),
                self.totals[i],
                p95(&series),
                match r.evidence().trend {
                    rules::Trend::Flat(w) => format!("flat {w}w"),
                    rules::Trend::Improving => "improving".to_string(),
                    rules::Trend::Rare => "rare".to_string(),
                    rules::Trend::Routine => "routine".to_string(),
                },
                r.default_stance().as_str(),
            ));
        }

        for (i, r) in self.rules.iter().enumerate() {
            if self.samples[i].is_empty() {
                continue;
            }
            s.push_str(&format!("\n{} — samples\n", r.id()));
            for sample in &self.samples[i] {
                s.push_str(&format!(
                    "  {}  {}\n      {}\n",
                    iso(sample.day),
                    sample.cwd,
                    sample.excerpt
                ));
            }
        }
        s
    }

    pub fn to_json(&self, roots: &transcript::Roots) -> String {
        let rules: Vec<String> = self
            .rules
            .iter()
            .enumerate()
            .map(|(i, r)| {
                let weeks: Vec<String> = self
                    .weeks
                    .iter()
                    .map(|(w, b)| {
                        json::object(&[
                            json::string_field("week", &iso(*w)),
                            json::int_field("calls", i64::from(b.calls)),
                            json::int_field("fires", i64::from(b.fires[i])),
                            json::string_field(
                                "per_1000",
                                &format!("{:.1}", rate(b.fires[i], b.calls)),
                            ),
                        ])
                    })
                    .collect();
                json::object(&[
                    json::string_field("id", r.id()),
                    json::int_field("total", i64::from(self.totals[i])),
                    json::string_field("stance", r.default_stance().as_str()),
                    json::string_field("p95_per_1000", &format!("{:.1}", p95(&self.series(i)))),
                    format!("\"weeks\":{}", json::array(&weeks)),
                ])
            })
            .collect();
        json::object(&[
            json::string_array_field(
                "transcripts",
                &roots
                    .dirs
                    .iter()
                    .map(|d| d.display().to_string())
                    .collect::<Vec<_>>(),
            ),
            json::int_field("files", self.stats.files as i64),
            json::int_field("calls", self.stats.calls as i64),
            json::int_field("duplicates", self.stats.duplicates as i64),
            json::int_field("opaque", self.opaque as i64),
            json::int_field("partly", self.partly as i64),
            format!("\"rules\":{}", json::array(&rules)),
            json::string_field(
                "note",
                "UTC weeks; local CLI sessions only, web sessions are not on this machine",
            ),
        ])
    }
}

/// The 95th percentile of the weekly rates — the number a graduation is judged
/// on, because a rule is intrusive at its worst week, not its average one.
pub fn p95(series: &[f64]) -> f64 {
    if series.is_empty() {
        return 0.0;
    }
    let mut v = series.to_vec();
    v.sort_by(|a, b| a.partial_cmp(b).unwrap());
    let idx = (((v.len() - 1) as f64) * 0.95).round() as usize;
    v[idx]
}