powercut 0.1.0

//! The CI ratchet: `assert_no_worse_than`.
//!
//! `assert_conforms()` is the wrong assertion for a durability harness.
//! Engines legitimately differ — a store may be `NotMeasured` on one platform,
//! `Lost` under a pragma its own docs warn you about, `Survived` everywhere
//! else — and a suite that demands uniformity gets switched off within a
//! month. What CI actually wants is *"nothing started differing in a NEW way
//! today"*.
//!
//! Three rules make the ratchet honest, and the third is the one people forget:
//!
//! 1. a verdict that got **worse** fails;
//! 2. a **new kind** of violation fails, even at the same verdict;
//! 3. a case that used to be in the baseline and is now **absent** fails, and
//!    a case that becomes [`Verdict::NotMeasured`] when it previously was not
//!    fails. Coverage is not allowed to quietly evaporate — that is how a gate
//!    dies without anybody noticing.

use std::collections::{BTreeMap, BTreeSet};
use std::fmt::Write as _;
use std::path::Path;

use crate::report::{Report, Verdict, ViolationKind};
use crate::{Error, Result};

/// The environment variable that rewrites a baseline instead of asserting
/// against it: `POWERCUT_UPDATE_BASELINE=1 cargo test`.
///
/// **It is ignored when `CI` is set.** Update mode turns every case green by
/// definition — it records whatever happened today as the new truth — so a
/// leaked `POWERCUT_UPDATE_BASELINE` in a CI environment would silently
/// convert the ratchet into a rubber stamp, and nobody would see a red again.
/// That is the precise failure this crate exists to prevent, so the ratchet
/// refuses: under `CI` it warns on stderr and performs the normal comparison
/// instead. Unset `CI` if you genuinely need to record a baseline there.
pub const UPDATE_ENV: &str = "POWERCUT_UPDATE_BASELINE";

/// The de-facto "this is an automated build" variable, set by GitHub Actions,
/// GitLab CI, CircleCI, Travis, Jenkins and friends. See [`UPDATE_ENV`].
const CI_ENV: &str = "CI";

/// One recorded case.
///
/// ```
/// # use powercut::{BaselineEntry, Verdict};
/// let e = BaselineEntry::new(Verdict::Lost, ["lost-data"]);
/// assert_eq!(e.verdict, Verdict::Lost);
/// ```
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct BaselineEntry {
    /// The verdict this case is allowed to have.
    pub verdict: Verdict,
    /// The violation kinds this case is allowed to produce.
    pub kinds: BTreeSet<ViolationKind>,
}

impl BaselineEntry {
    /// Build an entry from a verdict and a set of violation-kind labels.
    ///
    /// Unknown labels are ignored rather than rejected, so a baseline written
    /// by a newer `powercut` still parses in an older one.
    ///
    /// ```
    /// # use powercut::{BaselineEntry, Verdict, ViolationKind};
    /// let e = BaselineEntry::new(Verdict::Lost, ["lost-data", "divergent"]);
    /// assert!(e.kinds.contains(&ViolationKind::Divergent));
    /// ```
    #[must_use]
    pub fn new<'a>(verdict: Verdict, kinds: impl IntoIterator<Item = &'a str>) -> Self {
        Self {
            verdict,
            kinds: kinds.into_iter().filter_map(ViolationKind::parse).collect(),
        }
    }
}

/// A recorded set of expected outcomes, one per case label.
///
/// The on-disk format is three tab-separated columns and a header line. It is
/// meant to be read, reviewed and argued with in a pull request — a baseline
/// nobody can read is a baseline nobody notices changing.
///
/// ```text
/// # powercut baseline v1 — <case> <TAB> <verdict> <TAB> <violation kinds, comma-separated>
/// sqlite/synchronous=OFF<TAB>lost<TAB>lost-data
/// sqlite/durable-ack<TAB>survived
/// ```
///
/// (`<TAB>` above is a literal tab character.)
///
/// ```
/// # use powercut::{Baseline, Verdict};
/// let text = "# powercut baseline v1\nsqlite/off\tlost\tlost-data\n";
/// let baseline = Baseline::parse(text);
/// assert_eq!(baseline.get("sqlite/off").unwrap().verdict, Verdict::Lost);
/// ```
#[derive(Clone, Debug, Default)]
pub struct Baseline {
    entries: BTreeMap<String, BaselineEntry>,
}

impl Baseline {
    /// An empty baseline.
    ///
    /// ```
    /// # use powercut::Baseline;
    /// assert!(Baseline::new().is_empty());
    /// ```
    #[must_use]
    pub fn new() -> Self {
        Self::default()
    }

    /// `true` when nothing is recorded.
    ///
    /// ```
    /// # use powercut::Baseline;
    /// assert!(Baseline::new().is_empty());
    /// ```
    #[must_use]
    pub fn is_empty(&self) -> bool {
        self.entries.is_empty()
    }

    /// Look up one case.
    ///
    /// ```
    /// # use powercut::Baseline;
    /// assert!(Baseline::new().get("nope").is_none());
    /// ```
    #[must_use]
    pub fn get(&self, label: &str) -> Option<&BaselineEntry> {
        self.entries.get(label)
    }

    /// Parse the on-disk format. Blank lines and `#` comments are ignored;
    /// unparseable verdicts are skipped rather than panicking.
    ///
    /// ```
    /// # use powercut::{Baseline, Verdict};
    /// let b = Baseline::parse("redb/default\tsurvived\n");
    /// assert_eq!(b.get("redb/default").unwrap().verdict, Verdict::Survived);
    /// ```
    #[must_use]
    pub fn parse(text: &str) -> Self {
        let mut entries = BTreeMap::new();
        for line in text.lines() {
            let line = line.trim_end();
            if line.is_empty() || line.starts_with('#') {
                continue;
            }
            let mut cols = line.split('\t');
            let (Some(label), Some(verdict)) = (cols.next(), cols.next()) else {
                continue;
            };
            let Some(verdict) = Verdict::parse(verdict.trim()) else {
                continue;
            };
            let kinds = cols.next().unwrap_or("");
            entries.insert(
                label.trim().to_string(),
                BaselineEntry::new(verdict, kinds.split(',').map(str::trim)),
            );
        }
        Self { entries }
    }

    /// Render the on-disk format for a set of reports.
    ///
    /// ```
    /// # use powercut::Baseline;
    /// assert!(Baseline::render(&[]).starts_with("# powercut baseline v1"));
    /// ```
    #[must_use]
    pub fn render(reports: &[Report]) -> String {
        let mut out = String::from(
            "# powercut baseline v1 — <case>\\t<verdict>\\t<violation kinds, comma-separated>\n\
             # Regenerate with: POWERCUT_UPDATE_BASELINE=1 cargo test\n",
        );
        let mut rows: Vec<_> = reports
            .iter()
            .map(|r| {
                let kinds: BTreeSet<&str> = r
                    .finding()
                    .violations()
                    .iter()
                    .map(|v| v.kind.as_str())
                    .collect();
                (
                    r.label().to_string(),
                    r.verdict(),
                    kinds.into_iter().collect::<Vec<_>>().join(","),
                )
            })
            .collect();
        rows.sort();
        for (label, verdict, kinds) in rows {
            let _ = writeln!(out, "{label}\t{verdict}\t{kinds}");
        }
        out
    }

    /// Read a baseline from disk.
    ///
    /// # Errors
    ///
    /// Fails if the file exists but cannot be read. A **missing** file is not
    /// an error here — [`assert_no_worse_than`] turns that into its own,
    /// clearly worded failure with the exact content to record.
    pub fn load(path: impl AsRef<Path>) -> Result<Option<Self>> {
        let path = path.as_ref();
        match std::fs::read_to_string(path) {
            Ok(text) => Ok(Some(Self::parse(&text))),
            Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
            Err(e) => Err(Error::io(Some(path.to_path_buf()), e)),
        }
    }
}

/// Compare `reports` against the baseline recorded at `path` and return every
/// way today is worse than the recorded day.
///
/// An empty vector means the ratchet held. Use [`assert_no_worse_than`] to
/// turn a non-empty one into a good panic.
///
/// # Errors
///
/// Fails if the baseline file exists but cannot be read, or if
/// `POWERCUT_UPDATE_BASELINE` is set and the file cannot be written.
///
/// ```
/// # use powercut::{compare_to_baseline, ScratchDir};
/// let dir = ScratchDir::new("doc-ratchet").unwrap();
/// let path = dir.path().join("baseline.tsv");
/// // No baseline recorded yet, and no reports: nothing to compare.
/// let regressions = compare_to_baseline(&[], &path).unwrap();
/// assert!(regressions.is_empty());
/// ```
pub fn compare_to_baseline(reports: &[Report], path: impl AsRef<Path>) -> Result<Vec<String>> {
    let path = path.as_ref();

    if std::env::var_os(UPDATE_ENV).is_some() {
        // Update mode greens every case by construction. Honouring it in CI
        // would retire the gate without anybody noticing, so refuse and fall
        // through to the real comparison — loudly, because a silent refusal
        // would be its own version of the same sin.
        if std::env::var_os(CI_ENV).is_some() {
            eprintln!(
                "powercut: {UPDATE_ENV} is set, but so is {CI_ENV}. Recording a baseline \
                 accepts whatever happened today as correct, so honouring it here would \
                 turn this ratchet into a rubber stamp. IGNORING the update request and \
                 asserting against `{}` as recorded. Record baselines locally, read the \
                 diff, and commit it.",
                path.display()
            );
        } else {
            if let Some(parent) = path.parent() {
                if !parent.as_os_str().is_empty() {
                    std::fs::create_dir_all(parent)
                        .map_err(|e| Error::io(Some(parent.to_path_buf()), e))?;
                }
            }
            std::fs::write(path, Baseline::render(reports))
                .map_err(|e| Error::io(Some(path.to_path_buf()), e))?;
            return Ok(Vec::new());
        }
    }

    let Some(baseline) = Baseline::load(path)? else {
        if reports.is_empty() {
            return Ok(Vec::new());
        }
        return Ok(vec![format!(
            "no baseline at `{}`. A durability suite with no recorded baseline cannot tell \
             `unchanged` from `never ran`. Record one with `{UPDATE_ENV}=1 cargo test`, read \
             it, and commit it. It should contain:\n\n{}",
            path.display(),
            Baseline::render(reports)
        )]);
    };

    let mut regressions = Vec::new();
    let mut seen = BTreeSet::new();

    for report in reports {
        seen.insert(report.label().to_string());
        let Some(entry) = baseline.get(report.label()) else {
            regressions.push(format!(
                "`{}` is not in the baseline. New cases must be recorded deliberately, \
                 not adopted silently.",
                report.label()
            ));
            continue;
        };
        let verdict = report.verdict();
        if verdict.severity() > entry.verdict.severity() {
            regressions.push(format!(
                "`{}` regressed: baseline {}, now {}.\n{}",
                report.label(),
                entry.verdict,
                verdict,
                indent(&report.to_text())
            ));
            continue;
        }
        if verdict == Verdict::NotMeasured && entry.verdict != Verdict::NotMeasured {
            regressions.push(format!(
                "`{}` was {} in the baseline and is NOT MEASURED now. Losing the ability to \
                 measure is a regression even though the verdict looks milder — this is \
                 exactly how a gate dies without anybody noticing.\n{}",
                report.label(),
                entry.verdict,
                indent(&report.to_text())
            ));
            continue;
        }
        let new_kinds: Vec<&str> = report
            .finding()
            .violations()
            .iter()
            .map(|v| v.kind)
            .filter(|k| !entry.kinds.contains(k))
            .map(ViolationKind::as_str)
            .collect();
        if !new_kinds.is_empty() {
            regressions.push(format!(
                "`{}` started failing in a NEW way ({}), even though the verdict is unchanged \
                 at {}.\n{}",
                report.label(),
                new_kinds.join(", "),
                verdict,
                indent(&report.to_text())
            ));
        }
    }

    for label in baseline.entries.keys() {
        if !seen.contains(label) {
            regressions.push(format!(
                "`{label}` is in the baseline but produced no report. A case that stops running \
                 must never read as a pass — either restore it or remove it from the baseline \
                 in a commit somebody reviews."
            ));
        }
    }

    Ok(regressions)
}

/// The CI assertion: today is no worse than the recorded baseline.
///
/// # Panics
///
/// Panics, with the full self-adjudicating report for every regressed case, if
/// any of the three ratchet rules fires — or if the baseline is missing, in
/// which case the panic message contains the exact file to commit.
///
/// ```no_run
/// # use powercut::assert_no_worse_than;
/// # let reports: Vec<powercut::Report> = Vec::new();
/// assert_no_worse_than(&reports, "tests/durability-baseline.tsv");
/// ```
pub fn assert_no_worse_than(reports: &[Report], path: impl AsRef<Path>) {
    let path = path.as_ref();
    match compare_to_baseline(reports, path) {
        Ok(regressions) if regressions.is_empty() => {}
        Ok(regressions) => panic!(
            "\npowercut: {} regression(s) against `{}`\n\n{}\n",
            regressions.len(),
            path.display(),
            regressions.join("\n\n")
        ),
        Err(e) => panic!(
            "powercut: could not evaluate the baseline `{}`: {e}",
            path.display()
        ),
    }
}

fn indent(text: &str) -> String {
    text.lines()
        .map(|l| format!("    {l}"))
        .collect::<Vec<_>>()
        .join("\n")
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn round_trips() {
        let text = "# c\nalpha\tlost\tlost-data,divergent\nbeta\tsurvived\t\n";
        let b = Baseline::parse(text);
        assert_eq!(b.get("alpha").unwrap().kinds.len(), 2);
        assert_eq!(b.get("beta").unwrap().verdict, Verdict::Survived);
        assert!(b.get("beta").unwrap().kinds.is_empty());
    }

    #[test]
    fn a_baseline_case_that_stops_reporting_is_a_regression() {
        let dir = crate::ScratchDir::new("ratchet-missing").unwrap();
        let path = dir.path().join("b.tsv");
        std::fs::write(&path, "gone\tsurvived\t\n").unwrap();
        let regressions = compare_to_baseline(&[], &path).unwrap();
        assert_eq!(regressions.len(), 1);
        assert!(regressions[0].contains("must never read as a pass"));
    }
}