tak_cli/

config.rs

//! `tak.toml` — declared benchmarks.
//!
//! Named after the tool rather than after its current contents. It already
//! holds more than a command list in spirit, and gates, runner classes and
//! competitor definitions all belong here too; `bench.toml` would be misnamed
//! the moment the first of those lands.
//!
//! The point of declaring benchmarks is that CI and a laptop run the same
//! thing. A command line in a workflow file drifts from the one people use
//! locally, and the numbers stop being comparable without anyone noticing.

use anyhow::{Context, Result, bail};
use serde::Deserialize;
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};

pub const FILE_NAME: &str = "tak.toml";

/// Defaults chosen to match `tak run`'s, so moving a command into `tak.toml`
/// does not silently change what it measures.
pub const DEFAULT_RUNS: u32 = 20;
pub const DEFAULT_WARMUP: u32 = 3;

#[derive(Debug, Deserialize)]
pub struct Config {
    /// Benchmarks by name. A BTreeMap so runs are ordered and reproducible
    /// rather than following the file's incidental key order.
    #[serde(default)]
    pub bench: BTreeMap<String, Bench>,
    /// Project-level environment settings. See `settings.toml` for what these
    /// mean; this type only says where they can be written.
    #[serde(default)]
    pub env: Option<EnvSection>,
    /// Regression-gate settings.
    #[serde(default)]
    pub gate: Option<GateSection>,
    /// Report-rendering settings.
    #[serde(default)]
    pub report: Option<ReportSection>,
    #[serde(default)]
    pub runner: Option<RunnerSection>,
}

/// The `[gate]` table in `tak.toml`.
#[derive(Debug, Deserialize, Default, Clone, PartialEq)]
pub struct GateSection {
    pub pct: Option<f64>,
}

/// The `[report]` table in `tak.toml`.
#[derive(Debug, Deserialize, Default, Clone, PartialEq)]
pub struct ReportSection {
    pub credit: Option<bool>,
}

/// The `[runner]` table in `tak.toml`.
#[derive(Debug, Deserialize, Default, Clone, PartialEq)]
pub struct RunnerSection {
    pub class: Option<String>,
}

/// Every `tak.toml` table that feeds a setting.
#[derive(Debug, Deserialize, Default)]
pub struct SettingsSections {
    #[serde(default)]
    pub env: Option<EnvSection>,
    #[serde(default)]
    pub gate: Option<GateSection>,
    #[serde(default)]
    pub report: Option<ReportSection>,
    /// Runner identity.
    #[serde(default)]
    pub runner: Option<RunnerSection>,
}

/// The `[env]` table in `tak.toml`.
///
/// Both fields are `Option` so an absent key defers to the environment and the
/// declared default, while `deny = []` is an explicit empty list. Making them
/// plain `Vec` would erase that distinction and turn "I did not mention this"
/// into "I want nothing scrubbed".
#[derive(Debug, Deserialize, Default, Clone, PartialEq, Eq)]
pub struct EnvSection {
    pub allow: Option<Vec<String>>,
    pub deny: Option<Vec<String>>,
}

#[derive(Debug, Deserialize)]
pub struct Bench {
    cmd: Cmd,
    pub runs: Option<u32>,
    pub warmup: Option<u32>,
}

/// A command, written either as a list or as a plain string.
#[derive(Debug, Deserialize)]
#[serde(untagged)]
enum Cmd {
    Argv(Vec<String>),
    Line(String),
}

impl Bench {
    /// The command as argv.
    ///
    /// A string is split on whitespace and nothing else. There is deliberately
    /// no shell: spawning one adds its own startup cost and variance to every
    /// sample, which for commands in the 10ms range is a large fraction of the
    /// measurement. Anything needing a pipe or a glob should be a list whose
    /// first element is the interpreter.
    pub fn argv(&self) -> Result<Vec<String>> {
        let v = match &self.cmd {
            Cmd::Argv(v) => v.clone(),
            Cmd::Line(s) => s.split_whitespace().map(str::to_string).collect(),
        };
        if v.is_empty() {
            bail!("empty command");
        }
        Ok(v)
    }

    pub fn runs(&self) -> u32 {
        self.runs.unwrap_or(DEFAULT_RUNS)
    }

    pub fn warmup(&self) -> u32 {
        self.warmup.unwrap_or(DEFAULT_WARMUP)
    }
}

impl Config {
    pub fn parse(text: &str) -> Result<Self> {
        let cfg: Config = toml::from_str(text).context("could not parse tak.toml")?;
        // Every declared benchmark is validated up front rather than failing
        // partway through a run that has already spent minutes measuring.
        for (name, b) in &cfg.bench {
            b.argv()
                .with_context(|| format!("benchmark `{name}` has no command"))?;
        }
        Ok(cfg)
    }

    /// Read the setting tables, without validating benchmarks.
    ///
    /// Settings and benchmarks live in the same file but are needed at
    /// different times. Resolving settings through [`Config::parse`] would make
    /// a broken `[bench.x]` abort `tak run -- somecmd`, which does not read
    /// benchmarks at all, and `tak settings`, which reads none of them either.
    ///
    /// Unknown keys are ignored, so `[bench]` is not even looked at here. A TOML
    /// *syntax* error still fails: the file may carry `[env]` settings that
    /// change what gets scrubbed from a subject's environment, and quietly
    /// falling back to defaults would apply a weaker filter than the project
    /// asked for without saying so.
    pub fn find_settings(start: &Path) -> Result<SettingsSections> {
        for dir in start.ancestors() {
            let path = dir.join(FILE_NAME);
            if path.is_file() {
                let text = std::fs::read_to_string(&path)
                    .with_context(|| format!("could not read {}", path.display()))?;
                let parsed: SettingsSections = toml::from_str(&text)
                    .with_context(|| format!("could not parse {}", path.display()))?;
                return Ok(parsed);
            }
        }
        Ok(SettingsSections::default())
    }

    /// Find and load `tak.toml`, searching upward from `start`.
    ///
    /// Walking up means `tak run` behaves the same from a subdirectory as from
    /// the repository root, which is where people actually are.
    pub fn find(start: &Path) -> Result<Option<(PathBuf, Self)>> {
        for dir in start.ancestors() {
            let path = dir.join(FILE_NAME);
            if path.is_file() {
                let text = std::fs::read_to_string(&path)
                    .with_context(|| format!("could not read {}", path.display()))?;
                let cfg = Self::parse(&text).with_context(|| format!("in {}", path.display()))?;
                return Ok(Some((path, cfg)));
            }
        }
        Ok(None)
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn a_command_may_be_a_list_or_a_string() {
        let c = Config::parse(
            r#"
            [bench.a]
            cmd = ["mycli", "--version"]
            [bench.b]
            cmd = "mycli --help"
            "#,
        )
        .unwrap();
        assert_eq!(c.bench["a"].argv().unwrap(), ["mycli", "--version"]);
        assert_eq!(c.bench["b"].argv().unwrap(), ["mycli", "--help"]);
    }

    /// A string is split on whitespace and nothing else — no shell means no
    /// quoting, and pretending otherwise would measure the wrong thing.
    #[test]
    fn a_string_command_gets_no_shell_semantics() {
        let c = Config::parse(
            r#"[bench.a]
cmd = "mycli 'two words'""#,
        )
        .unwrap();
        assert_eq!(c.bench["a"].argv().unwrap(), ["mycli", "'two", "words'"]);
    }

    #[test]
    fn defaults_match_the_cli() {
        let c = Config::parse("[bench.a]\ncmd = \"x\"").unwrap();
        assert_eq!(c.bench["a"].runs(), DEFAULT_RUNS);
        assert_eq!(c.bench["a"].warmup(), DEFAULT_WARMUP);
    }

    #[test]
    fn per_benchmark_overrides_win() {
        let c = Config::parse("[bench.a]\ncmd = \"x\"\nruns = 5\nwarmup = 1").unwrap();
        assert_eq!(c.bench["a"].runs(), 5);
        assert_eq!(c.bench["a"].warmup(), 1);
    }

    /// Validation happens at load, not partway through a run that has already
    /// spent minutes measuring.
    #[test]
    fn an_empty_command_is_rejected_at_parse_time() {
        let err = Config::parse("[bench.a]\ncmd = []").unwrap_err();
        assert!(format!("{err:#}").contains('a'), "{err:#}");
    }

    #[test]
    fn benchmarks_run_in_a_stable_order() {
        let c = Config::parse("[bench.zebra]\ncmd = \"z\"\n[bench.alpha]\ncmd = \"a\"").unwrap();
        assert_eq!(c.bench.keys().collect::<Vec<_>>(), ["alpha", "zebra"]);
    }

    #[test]
    fn an_empty_file_declares_nothing() {
        assert!(Config::parse("").unwrap().bench.is_empty());
    }

    #[test]
    fn find_walks_up_from_a_subdirectory() {
        let root = std::env::temp_dir().join(format!("tak-cfg-{}", std::process::id()));
        let nested = root.join("a").join("b");
        std::fs::create_dir_all(&nested).unwrap();
        std::fs::write(root.join(FILE_NAME), "[bench.x]\ncmd = \"true\"").unwrap();

        let (path, cfg) = Config::find(&nested).unwrap().expect("should find it");
        assert_eq!(path, root.join(FILE_NAME));
        assert!(cfg.bench.contains_key("x"));

        std::fs::remove_dir_all(&root).ok();
    }
}