thirdshift 0.35.1

Turn a GitHub issue into a ready-for-review pull request, unattended
//! A Custom runner's declared inputs, frozen independently of a thirdshift build.

use std::collections::{BTreeMap, BTreeSet};
use std::fs;
use std::path::{Path, PathBuf};

use anyhow::{Context, Result, ensure};
use serde::{Deserialize, Serialize};

use super::FixedHarness;
use super::frozen::{digest, tree_hashes};
use crate::git::Git;
use crate::harness::Harness;
use crate::proxy::{self, Provider};

#[derive(Clone, Deserialize, Serialize)]
#[serde(deny_unknown_fields)]
pub(crate) struct Manifest {
    pub command: Vec<String>,
    #[serde(default)]
    pub harnesses: BTreeMap<String, Vec<String>>,
    #[serde(default)]
    pub providers: BTreeSet<Provider>,
    #[serde(default)]
    pub files: BTreeMap<String, Target>,
    #[serde(default)]
    pub roles: BTreeMap<String, Role>,
    #[serde(default)]
    pub git_exclude: Vec<String>,
}

#[derive(Clone, Deserialize, Serialize)]
#[serde(untagged)]
pub(crate) enum Target {
    Bundle(String),
    Home { home: String },
}

impl Target {
    fn path(&self) -> Result<PathBuf> {
        let (root, relative) = match self {
            Self::Bundle(relative) => ("bundle", relative),
            Self::Home { home } => ("home", home),
        };
        crate::evals::format::relative_path(relative)?;
        Ok(Path::new(root).join(relative))
    }
}

#[derive(Clone, Deserialize, Serialize)]
#[serde(deny_unknown_fields)]
pub(crate) struct Role {
    pub model: String,
    pub effort: Option<String>,
}

#[derive(Clone, Deserialize, Serialize)]
pub(crate) struct Provenance {
    pub repository: PathBuf,
    pub origin: Option<String>,
    pub path: PathBuf,
    pub commit: String,
    pub dirty: bool,
}

#[derive(Clone, Deserialize, Serialize)]
pub(crate) struct Runner {
    pub name: String,
    pub directory: PathBuf,
    pub manifest: Manifest,
    pub files: BTreeMap<String, String>,
    pub provenance: BTreeMap<String, Provenance>,
    #[serde(default)]
    pub harnesses: BTreeMap<String, RunnerHarness>,
}

#[derive(Clone, Deserialize, Serialize)]
pub(crate) struct RunnerHarness {
    pub binary: FixedHarness,
    pub hosts: BTreeSet<String>,
    pub families: BTreeSet<String>,
}

/// The part of a Custom runner's inputs a static problem belongs to.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub(crate) enum Check {
    Manifest,
    Files,
    Harnesses,
    Logins,
    Providers,
    Roles,
}

impl Check {
    pub const ALL: [Self; 6] = [
        Self::Manifest,
        Self::Files,
        Self::Harnesses,
        Self::Logins,
        Self::Providers,
        Self::Roles,
    ];

    pub fn name(self) -> &'static str {
        match self {
            Self::Manifest => "manifest",
            Self::Files => "files",
            Self::Harnesses => "harnesses",
            Self::Logins => "logins",
            Self::Providers => "providers",
            Self::Roles => "roles",
        }
    }
}

/// Where `freeze` sends each problem: the first one ends an ordinary freeze,
/// while `evals check-runner` collects them all.
pub(crate) type ProblemSink<'a> = dyn FnMut(Check, anyhow::Error) -> Result<()> + 'a;

impl Runner {
    pub fn freeze(name: &str, directory: &Path) -> Result<Self> {
        Self::freeze_reporting(name, directory, &mut |_, error| Err(error))?
            .context("Custom runner manifest could not be read")
    }

    /// Resolve a runner name or `.toml` path to its manifest file.
    pub fn manifest_path(name: &str) -> Result<PathBuf> {
        if name.ends_with(".toml") {
            PathBuf::from(name)
        } else {
            crate::evals::format::name(name)?;
            crate::config::home()?
                .join(".thirdshift/runners")
                .join(format!("{name}.toml"))
        }
        .canonicalize()
        .with_context(|| format!("cannot resolve Custom runner {name}"))
    }

    /// Freeze every input that passes, reporting each one that does not.
    /// Returns `None` only when no manifest could be read; the Runner it does
    /// return is complete only when nothing was reported.
    pub fn freeze_reporting(
        name: &str,
        directory: &Path,
        report: &mut ProblemSink<'_>,
    ) -> Result<Option<Self>> {
        let loaded = (|| -> Result<_> {
            let path = Self::manifest_path(name)?;
            let bytes = fs::read(&path)?;
            let manifest: Manifest = toml::from_str(std::str::from_utf8(&bytes)?)?;
            ensure!(
                !manifest.command.is_empty() && !manifest.command[0].is_empty(),
                "Custom runner command must contain a program"
            );
            ensure!(
                manifest.command.iter().all(|arg| !arg.contains('\0')),
                "Custom runner command contains NUL"
            );
            ensure!(
                manifest
                    .git_exclude
                    .iter()
                    .all(|pattern| !pattern.contains(['\n', '\r', '\0'])),
                "git_exclude patterns must each be one line"
            );
            Ok((path, bytes, manifest))
        })();
        let (path, bytes, manifest) = match loaded {
            Ok(loaded) => loaded,
            Err(error) => {
                report(Check::Manifest, error)?;
                return Ok(None);
            }
        };
        fs::create_dir_all(directory.join("bundle"))?;
        fs::create_dir_all(directory.join("home"))?;
        fs::write(directory.join("manifest.toml"), &bytes)?;
        let mut provenance = BTreeMap::new();
        let mut harnesses = BTreeMap::new();
        for (name, models) in &manifest.harnesses {
            match pin(name, models) {
                Ok(pinned) => {
                    harnesses.insert(name.clone(), pinned);
                }
                Err(error) => report(Check::Harnesses, error)?,
            }
        }
        if let Some(origin) = source_provenance(&path)? {
            provenance.insert("manifest.toml".into(), origin);
        }
        let parent = path.parent().context("runner manifest has no parent")?;
        for (source, target) in &manifest.files {
            let copied = (|| -> Result<_> {
                let destination = directory.join(target.path()?);
                let source_path = parent
                    .join(source)
                    .canonicalize()
                    .with_context(|| format!("cannot find Custom runner file {source}"))?;
                copy(&source_path, &destination, 0)?;
                source_provenance(&source_path)
            })();
            match copied {
                Ok(Some(origin)) => {
                    provenance.insert(source.clone(), origin);
                }
                Ok(None) => {}
                Err(error) => report(Check::Files, error.context(format!("file {source}")))?,
            }
        }
        Ok(Some(Self {
            name: name.into(),
            directory: directory.to_path_buf(),
            manifest,
            files: tree_hashes(directory)?,
            provenance,
            harnesses,
        }))
    }

    pub fn source_digest(name: &str) -> Result<String> {
        let temporary = tempfile::tempdir()?;
        Self::freeze(name, temporary.path())?.digest()
    }

    pub fn digest(&self) -> Result<String> {
        Ok(digest(&serde_json::to_vec(&self.files)?))
    }

    pub fn verify(&self) -> Result<()> {
        ensure!(
            tree_hashes(&self.directory)? == self.files,
            "frozen Custom runner file changed"
        );
        let frozen: Manifest =
            toml::from_str(&fs::read_to_string(self.directory.join("manifest.toml"))?)?;
        ensure!(
            serde_json::to_vec(&frozen)? == serde_json::to_vec(&self.manifest)?,
            "frozen Custom runner manifest changed"
        );
        Ok(())
    }

    /// Install checked bytes into a fresh attempt, preserving executable modes.
    pub fn install(&self, attempt: &Path) -> Result<PathBuf> {
        self.verify()?;
        let runner_dir = attempt.join("runner");
        fs::create_dir_all(&runner_dir)?;
        fs::create_dir_all(attempt.join("home"))?;
        for (name, expected) in &self.files {
            let target = if let Some(relative) = name.strip_prefix("bundle/") {
                runner_dir.join(relative)
            } else if name.starts_with("home/") {
                attempt.join(name)
            } else {
                continue;
            };
            if name.ends_with('/') {
                fs::create_dir_all(target)?;
                continue;
            }
            let source = self.directory.join(name);
            let bytes = fs::read(&source)?;
            ensure!(
                digest(&bytes) == *expected,
                "frozen Custom runner file changed: {name}"
            );
            fs::create_dir_all(target.parent().context("runner file has no parent")?)?;
            fs::write(&target, bytes)?;
            fs::set_permissions(target, fs::metadata(source)?.permissions())?;
        }
        Ok(runner_dir)
    }
}

/// Pin one declared Harness and the provider hosts its Models reach.
fn pin(name: &str, models: &[String]) -> Result<RunnerHarness> {
    let harness =
        Harness::named(name).with_context(|| format!("unknown Custom runner Harness {name}"))?;
    ensure!(
        !matches!(harness, Harness::OpenCode | Harness::Kimi) || !models.is_empty(),
        "Custom runner {name} requires explicit Models for provider host resolution"
    );
    let binary = super::fixed_harness(harness)?;
    let mut hosts = BTreeSet::new();
    let mut families = BTreeSet::new();
    let models: Vec<Option<&str>> = if models.is_empty() {
        vec![None]
    } else {
        models.iter().map(|model| Some(model.as_str())).collect()
    };
    for model in models {
        let allowed = proxy::allowed_hosts(harness, model)?;
        families.insert(proxy::harness_family(harness, model, &allowed));
        hosts.extend(allowed);
    }
    Ok(RunnerHarness {
        binary,
        hosts,
        families,
    })
}

fn source_provenance(path: &Path) -> Result<Option<Provenance>> {
    let directory = if path.is_dir() {
        path
    } else {
        path.parent().context("source has no parent")?
    };
    let git = Git::new(directory);
    let Ok(root) = git.run(&["rev-parse", "--show-toplevel"]) else {
        return Ok(None);
    };
    let repository = PathBuf::from(root).canonicalize()?;
    Ok(Some(Provenance {
        path: path.strip_prefix(&repository)?.to_path_buf(),
        commit: git.run(&["rev-parse", "HEAD"])?,
        origin: git.run(&["config", "--get", "remote.origin.url"]).ok(),
        dirty: !git
            .run(&["status", "--porcelain", "--untracked-files=all"])?
            .is_empty(),
        repository,
    }))
}

fn copy(source: &Path, destination: &Path, depth: usize) -> Result<()> {
    ensure!(depth <= 64, "Custom runner directory nested too deeply");
    let kind = fs::symlink_metadata(source)?.file_type();
    ensure!(
        kind.is_dir() || kind.is_file(),
        "Custom runner inputs must be regular files or directories: {}",
        source.display()
    );
    ensure!(
        !destination.exists(),
        "overlapping Custom runner file targets: {}",
        destination.display()
    );
    if kind.is_dir() {
        fs::create_dir_all(destination)?;
        for entry in fs::read_dir(source)? {
            let entry = entry?;
            copy(
                &entry.path(),
                &destination.join(entry.file_name()),
                depth + 1,
            )?;
        }
    } else {
        fs::create_dir_all(destination.parent().context("runner file has no parent")?)?;
        fs::copy(source, destination)?;
    }
    Ok(())
}