thirdshift 0.34.0

Turn a GitHub issue into a ready-for-review pull request, unattended
//! Private Eval tasks: manifests, suite selection and time-safe repositories.

use std::path::PathBuf;

use anyhow::{Context, Result, bail};

pub(crate) mod approval;
mod arms;
pub(crate) use arms::Runner;
pub(crate) mod bug_origin;
pub mod check_runner;
pub(crate) mod experiment;
mod format;
mod import;
mod process_tasks;
mod repository;
mod suite;

pub enum Action {
    List,
    ImportReplays,
    ProposeProcess {
        repository: PathBuf,
        limit: u32,
    },
    Approve {
        suite: String,
        task: String,
    },
    Touched(Vec<String>),
    Prepare {
        suite: String,
        task: String,
        destination: PathBuf,
    },
    Verify {
        repository: PathBuf,
        base: String,
    },
    PrepareArms {
        repository: PathBuf,
        arms: Box<[arms::Requested; 2]>,
        record: PathBuf,
    },
    PrepareSelectedArms {
        repository: PathBuf,
        options: Box<experiment::Options>,
        record: PathBuf,
    },
    CheckRunner(check_runner::Request),
}

pub fn parse(args: &[String]) -> Result<Action> {
    match args {
        [] => Ok(Action::List),
        [flag] if flag == "--json" => Ok(Action::List),
        [action, flag] if action == "list" && flag == "--json" => Ok(Action::List),
        [action, suite, task] if action == "approve" => {
            format::name(suite)?;
            format::name(task)?;
            Ok(Action::Approve {
                suite: suite.clone(),
                task: task.clone(),
            })
        }
        [action, suite, task, flag] if action == "approve" && flag == "--json" => {
            parse(&[action.clone(), suite.clone(), task.clone()])
        }
        [action] if action == "list" => Ok(Action::List),
        [action] if action == "import-replays" => Ok(Action::ImportReplays),
        [action, repository] if action == "propose-process" => Ok(Action::ProposeProcess {
            repository: repository.into(),
            limit: 100,
        }),
        [action, repository, flag, limit] if action == "propose-process" && flag == "--limit" => {
            let limit: u32 = limit.parse()?;
            anyhow::ensure!((1..=100).contains(&limit), "proposal limit must be 1..100");
            Ok(Action::ProposeProcess {
                repository: repository.into(),
                limit,
            })
        }
        [action, paths @ ..] if action == "touched" => {
            for path in paths {
                format::relative_path(path)?;
            }
            Ok(Action::Touched(paths.to_vec()))
        }
        [action, suite, task, destination] if action == "prepare" => {
            format::name(suite)?;
            format::name(task)?;
            Ok(Action::Prepare {
                suite: suite.clone(),
                task: task.clone(),
                destination: destination.into(),
            })
        }
        [action, runner, options @ ..] if action == "check-runner" => {
            check_runner_args(runner, options)
        }
        [action, repository, base] if action == "verify" => {
            format::commit_id(base)?;
            Ok(Action::Verify {
                repository: repository.into(),
                base: base.clone(),
            })
        }
        [action, repository, record, options @ ..]
            if action == "prepare-arms"
                && options.iter().any(|flag| {
                    matches!(flag.as_str(), "--baseline-runner" | "--candidate-runner")
                }) =>
        {
            Ok(Action::PrepareSelectedArms {
                repository: repository.into(),
                record: record.into(),
                options: Box::new(experiment::parse(options)?),
            })
        }
        [
            action,
            repository,
            baseline,
            baseline_harness,
            candidate,
            candidate_harness,
            record,
            options @ ..,
        ] if action == "prepare-arms" => {
            let mut arms = [
                arms::Requested::new(baseline, baseline_harness)?,
                arms::Requested::new(candidate, candidate_harness)?,
            ];
            arms::Requested::options(&mut arms, options)?;
            Ok(Action::PrepareArms {
                repository: repository.into(),
                arms: Box::new(arms),
                record: record.into(),
            })
        }
        _ => bail!(
            "usage: thirdshift evals list | approve <suite> <task> [--json] | import-replays | propose-process <repository> [--limit <1..100>] | touched [<path>...] | prepare <suite> <task> <destination> | verify <repository> <base> | check-runner <name-or-file.toml> [--suite <suite> --task <task>] [--json] | prepare-arms <repository> <baseline-commit> <baseline-harness> <candidate-commit> <candidate-harness> <record.json>"
        ),
    }
}

fn check_runner_args(runner: &str, options: &[String]) -> Result<Action> {
    let (mut suite, mut task, mut json) = (None, None, false);
    let mut options = options.iter();
    while let Some(flag) = options.next() {
        let slot = match flag.as_str() {
            "--json" if !json => {
                json = true;
                continue;
            }
            "--suite" if suite.is_none() => &mut suite,
            "--task" if task.is_none() => &mut task,
            _ => bail!(
                "usage: thirdshift evals check-runner <name-or-file.toml> [--suite <suite> --task <task>] [--json]"
            ),
        };
        let value = options
            .next()
            .with_context(|| format!("{flag} requires a value"))?;
        format::name(value)?;
        *slot = Some(value.clone());
    }
    let task = match (suite, task) {
        (Some(suite), Some(task)) => Some((suite, task)),
        (None, None) => None,
        _ => bail!("a dry run needs both --suite and --task"),
    };
    Ok(Action::CheckRunner(check_runner::Request {
        runner: runner.into(),
        task,
        json,
    }))
}

pub fn execute(action: Action) -> Result<String> {
    let root = crate::config::home()?.join(".thirdshift/evals/suites");
    match action {
        Action::ProposeProcess { repository, limit } => Ok(serde_json::to_string_pretty(
            &process_tasks::propose(&repository, limit)?,
        )?),
        Action::Approve { suite, task } => Ok(serde_json::to_string_pretty(&approval::approve(
            &suite, &task,
        )?)?),
        Action::List => Ok(serde_json::to_string_pretty(&suite::list(&root)?)?),
        Action::ImportReplays => Ok(serde_json::to_string(&import::replays(
            &crate::config::home()?,
        )?)?),
        Action::Touched(paths) => Ok(serde_json::to_string_pretty(&suite::touched(
            &root, &paths,
        )?)?),
        Action::Prepare {
            suite,
            task,
            destination,
        } => {
            let suite = root.join(suite);
            suite::Manifest::load(&suite)?;
            let directory = suite.join("tasks").join(task);
            let task = format::Task::load(&directory)?;
            Ok(serde_json::to_string_pretty(&repository::prepare(
                &directory,
                &task,
                &destination,
            )?)?)
        }
        // The CLI prints a failed report before exiting 1; callers of this
        // function get the same report as the error instead.
        Action::CheckRunner(request) => {
            let (report, ok) = check_runner::execute(&request)?;
            anyhow::ensure!(ok, "{report}");
            Ok(report)
        }
        Action::Verify { repository, base } => {
            repository::verify(&repository, &base)?;
            Ok(r#"{"safe":true}"#.into())
        }
        Action::PrepareArms {
            repository,
            arms,
            record,
        } => Ok(serde_json::to_string_pretty(&arms::prepare(
            &repository,
            *arms,
            &record,
        )?)?),
        Action::PrepareSelectedArms {
            repository,
            options,
            record,
        } => Ok(serde_json::to_string_pretty(&experiment::prepare_arms(
            &repository,
            *options,
            &record,
        )?)?),
    }
}