thirdshift 0.26.0

Turn a GitHub issue into a ready-for-review pull request, unattended
//! Prepare both Arms before exposing their fixed binaries to the Trial runner.

use std::fs;
use std::io::Write;
use std::path::{Path, PathBuf};
use std::process::{Command, Output};

use anyhow::{Context, Result, bail};
use serde::{Deserialize, Serialize};

use super::format;
use crate::harness::Harness;
use crate::process::{self, Control, Interruption};

mod frozen;
pub(crate) use frozen::Process;

pub(crate) struct Requested {
    commit: String,
    harness: Harness,
    process: Option<String>,
    model: Option<String>,
    effort: Option<String>,
}

impl Requested {
    pub fn new(commit: &str, harness: &str) -> Result<Self> {
        format::commit_id(commit)?;
        Ok(Self {
            commit: commit.into(),
            harness: Harness::named(harness).context("unknown Arm Harness")?,
            process: None,
            model: None,
            effort: None,
        })
    }

    pub fn with_process(
        mut self,
        process: Option<&str>,
        model: Option<&str>,
        effort: Option<&str>,
    ) -> Self {
        self.process = process.map(String::from);
        self.model = model.map(String::from);
        self.effort = effort.map(String::from);
        self
    }

    pub fn options(arms: &mut [Self; 2], options: &[String]) -> Result<()> {
        let mut seen = std::collections::BTreeSet::new();
        let mut options = options.iter();
        while let Some(flag) = options.next() {
            anyhow::ensure!(seen.insert(flag), "repeated {flag}");
            let value = options
                .next()
                .with_context(|| format!("{flag} requires a value"))?;
            let (index, field) = if let Some(field) = flag.strip_prefix("--baseline-") {
                (0, field)
            } else if let Some(field) = flag.strip_prefix("--candidate-") {
                (1, field)
            } else {
                bail!("unknown Arm flag: {flag}");
            };
            match field {
                "process" => arms[index].process = Some(value.clone()),
                "model" => arms[index].model = Some(value.clone()),
                "effort" => arms[index].effort = Some(value.clone()),
                _ => bail!("unknown Arm flag: {flag}"),
            }
        }
        if arms.iter().any(|arm| arm.process.is_some()) {
            for arm in arms {
                arm.process.get_or_insert_with(|| "ticket-run".into());
            }
        }
        Ok(())
    }
}

#[derive(Clone, Deserialize, Serialize)]
pub(crate) struct FixedHarness {
    pub name: String,
    pub path: PathBuf,
    pub version: String,
}

#[derive(Clone, Deserialize, Serialize)]
pub(crate) struct Arm {
    pub commit: String,
    pub binary: PathBuf,
    pub harness: FixedHarness,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub process: Option<Process>,
}

#[derive(Clone, Deserialize, Serialize)]
pub(crate) struct Prepared {
    format: u32,
    pub baseline: Arm,
    pub candidate: Arm,
}

pub(super) fn prepare(repository: &Path, arms: [Requested; 2], record: &Path) -> Result<Prepared> {
    if record.try_exists()? {
        bail!("Arm record already exists: {}", record.display());
    }
    let [baseline, candidate] = arms;
    // Resolve all Harnesses at the start, before either potentially long build.
    // The same Harness is resolved only once even when both Arms use it.
    let parent = record
        .parent()
        .filter(|p| !p.as_os_str().is_empty())
        .unwrap_or(Path::new("."));
    let frozen = tempfile::Builder::new()
        .prefix(".arm-inputs-")
        .tempdir_in(parent)?;
    let frozen_root = frozen.path().canonicalize()?;
    let mut pinned = std::collections::BTreeMap::new();
    let baseline_harness = frozen::pin(baseline.harness, &mut pinned)?;
    let candidate_harness = frozen::pin(candidate.harness, &mut pinned)?;
    let mut baseline_process =
        Process::freeze(&baseline, &frozen_root.join("baseline"), &mut pinned, None)?;
    let mut candidate_process = Process::freeze(
        &candidate,
        &frozen_root.join("candidate"),
        &mut pinned,
        None,
    )?;
    let builds = crate::config::home()?.join(".thirdshift/evals/builds");
    fs::create_dir_all(&builds)?;
    let builds = builds.canonicalize()?;
    let baseline_binary = build(repository, &builds, &baseline.commit)?;
    let candidate_binary = build(repository, &builds, &candidate.commit)?;
    Process::from_build(
        &baseline,
        &baseline_binary,
        &mut baseline_process,
        &mut pinned,
    )?;
    Process::from_build(
        &candidate,
        &candidate_binary,
        &mut candidate_process,
        &mut pinned,
    )?;
    let prepared = Prepared {
        format: 1,
        baseline: Arm {
            binary: baseline_binary,
            commit: baseline.commit,
            harness: baseline_harness,
            process: baseline_process,
        },
        candidate: Arm {
            binary: candidate_binary,
            commit: candidate.commit,
            harness: candidate_harness,
            process: candidate_process,
        },
    };
    let json = serde_json::to_string_pretty(&prepared)?;
    let parent = record
        .parent()
        .filter(|p| !p.as_os_str().is_empty())
        .unwrap_or(Path::new("."));
    let mut temporary = tempfile::NamedTempFile::new_in(parent)?;
    temporary.write_all(json.as_bytes())?;
    temporary
        .persist_noclobber(record)
        .with_context(|| format!("can't publish Arm record {}", record.display()))?;
    if prepared.baseline.process.is_some() || prepared.candidate.process.is_some() {
        let _ = frozen.keep();
    }
    Ok(prepared)
}

pub(super) fn fixed_harness(harness: Harness) -> Result<FixedHarness> {
    let name = harness.name();
    let path = process::find_executable(name)
        .with_context(|| format!("Arm Harness {name} is not on PATH"))?
        .canonicalize()?;
    let output = checked(
        Command::new(&path)
            .arg("--version")
            .envs(harness.adapter().environment().iter().copied()),
        "Arm Harness version",
    )?;
    let stdout = String::from_utf8(output.stdout)?;
    let stderr = String::from_utf8(output.stderr)?;
    let version = if stdout.trim().is_empty() {
        stderr.trim()
    } else {
        stdout.trim()
    }
    .to_owned();
    if version.is_empty() {
        bail!("Arm Harness {name} reported an empty version");
    }
    Ok(FixedHarness {
        name: name.into(),
        path,
        version,
    })
}

pub(super) fn build(repository: &Path, builds: &Path, commit: &str) -> Result<PathBuf> {
    let git = crate::git::Git::new(repository);
    if git.run(&["--no-replace-objects", "cat-file", "-t", commit])? != "commit" {
        bail!("Arm {commit} is not a commit");
    }
    let lock = fs::File::create(builds.join(format!("{commit}.lock")))?;
    lock.lock()?;
    let cached = builds.join(commit);
    let binary = cached.join("thirdshift");
    if process::is_executable(&binary) {
        return Ok(binary);
    }
    if cached.try_exists()? {
        bail!("invalid Arm build cache entry: {}", cached.display());
    }
    let temporary = tempfile::Builder::new()
        .prefix(".arm-build-")
        .tempdir_in(builds)?;
    let worktree = temporary.path().join("worktree");
    let worktree_path = worktree
        .to_str()
        .context("Arm worktree path must be UTF-8")?;
    git.run(&[
        "--no-replace-objects",
        "-c",
        "core.hooksPath=/dev/null",
        "worktree",
        "add",
        "--detach",
        worktree_path,
        commit,
    ])?;
    let target = temporary.path().join("target");
    let result = checked(
        Command::new("cargo")
            .args(["build", "--release", "--locked"])
            .current_dir(&worktree)
            .env("GIT_NO_REPLACE_OBJECTS", "1")
            .env("CARGO_TARGET_DIR", &target),
        "Arm build",
    );
    // Cleanup must still finish when ordinary work was interrupted.
    let cleanup = git
        .completion()
        .run(&["worktree", "remove", "--force", worktree_path]);
    result.with_context(|| format!("could not build Arm {commit}"))?;
    cleanup?;
    let built = target.join("release/thirdshift");
    if !process::is_executable(&built) {
        bail!(
            "Arm build {commit} produced no executable at {}",
            built.display()
        );
    }
    let published = temporary.path().join("published");
    fs::create_dir(&published)?;
    fs::copy(built, published.join("thirdshift"))?;
    fs::rename(published, cached)?;
    Ok(binary)
}

fn checked(command: &mut Command, name: &str) -> Result<Output> {
    let output = process::output(
        command,
        None,
        Control {
            name,
            interruption: Interruption::Ordinary,
            stop: &|child| process::stop(child, &[libc::SIGTERM, libc::SIGKILL]),
        },
    )?;
    if !output.status.success() {
        bail!(
            "{name} failed ({}): {}\n{}",
            output.status,
            String::from_utf8_lossy(&output.stderr).trim(),
            String::from_utf8_lossy(&output.stdout).trim()
        );
    }
    Ok(output)
}