procyon 0.1.2

//! Running the Developer Parity tasks.
//!
//! The question this measures is not "is Procyon better than the tool next to it". It is how far a
//! developer gets on a real Stellar task, and how much of that distance the harness is responsible
//! for. So a task is defined by an **acceptance script**, not by a rubric: it exits zero or it does
//! not, and nothing about the model's own account of its work is allowed to decide that. A harness
//! grading itself is the failure mode this design exists to avoid.
//!
//! Each run happens in a throwaway copy of the task's seed workspace, so a task cannot be passed by
//! artifacts another task left behind, and a failed run leaves the seed untouched.
//!
//! What is measured here is only what a machine can settle: whether acceptance passed, how long it
//! took, how many tool calls it cost, how many of those failed, and how many named a tool that does
//! not exist. That last one is the only part of "hallucination rate" that is not a judgement call,
//! and it is reported under its own name rather than dressed up as the whole thing. The rest of the
//! metrics in the spec — final quality, whether the workflow was the right one — need a reviewer,
//! and this prints the evidence for one rather than pretending to be one.

use std::path::{Path, PathBuf};
use std::time::{Duration, Instant};

use color_eyre::eyre::{bail, Context, Result};

/// One task: a prompt, a seed workspace, and the script that decides whether it worked.
#[derive(Debug, Clone, PartialEq)]
pub struct Task {
    pub name: String,
    /// What the developer would have typed.
    pub prompt: String,
    /// Exits zero if the work is acceptable. Run from the root of the run's workspace copy.
    pub verify: PathBuf,
    /// Copied to make the run's workspace. Absent means the run starts from an empty directory,
    /// which is the newcomer's actual starting point.
    pub seed: Option<PathBuf>,
    pub timeout: Duration,
}

/// Default wall-clock allowance. Long enough for a build on a cold cargo cache, since a task that
/// times out during `cargo build` measures the machine rather than the harness.
const DEFAULT_TIMEOUT: Duration = Duration::from_secs(900);

/// Reads the tasks under a directory, sorted by name so a report is comparable between runs.
pub fn load_tasks(dir: &Path) -> Result<Vec<Task>> {
    if !dir.exists() {
        bail!("No task directory at {}", dir.display());
    }

    let mut tasks = Vec::new();
    for entry in std::fs::read_dir(dir)
        .with_context(|| format!("reading {}", dir.display()))?
        .flatten()
    {
        let path = entry.path();
        if path.join("task.toml").exists() {
            tasks.push(load_task(&path)?);
        }
    }
    tasks.sort_by(|a, b| a.name.cmp(&b.name));

    if tasks.is_empty() {
        bail!("No tasks found under {}", dir.display());
    }
    Ok(tasks)
}

fn load_task(dir: &Path) -> Result<Task> {
    let manifest = dir.join("task.toml");
    let text = std::fs::read_to_string(&manifest)
        .with_context(|| format!("reading {}", manifest.display()))?;
    let parsed: toml::Value = text
        .parse()
        .with_context(|| format!("parsing {}", manifest.display()))?;

    let string = |key: &str| parsed.get(key).and_then(|v| v.as_str()).map(str::to_string);

    let name = string("name").unwrap_or_else(|| {
        dir.file_name()
            .map(|n| n.to_string_lossy().to_string())
            .unwrap_or_default()
    });
    let Some(prompt) = string("prompt") else {
        bail!("{} has no 'prompt'", manifest.display());
    };

    let verify = dir.join(string("verify").unwrap_or_else(|| "verify.sh".to_string()));
    if !verify.exists() {
        // Refused rather than defaulted to "passes": a task with no acceptance script cannot be
        // failed, and a suite of those reports a perfect score for doing nothing.
        bail!(
            "{} names no acceptance script that exists ({})",
            name,
            verify.display()
        );
    }

    let seed = dir.join(string("seed").unwrap_or_else(|| "workspace".to_string()));

    Ok(Task {
        name,
        prompt,
        verify,
        seed: seed.is_dir().then_some(seed),
        timeout: parsed
            .get("timeout_secs")
            .and_then(|v| v.as_integer())
            .filter(|secs| *secs > 0)
            .map(|secs| Duration::from_secs(secs as u64))
            .unwrap_or(DEFAULT_TIMEOUT),
    })
}

/// What one attempt at one task cost and whether it worked.
#[derive(Debug, Clone, PartialEq)]
pub struct Outcome {
    pub task: String,
    /// The acceptance script's verdict, and the only thing that decides pass or fail.
    pub accepted: bool,
    pub seconds: f64,
    pub tool_calls: usize,
    pub failed_calls: usize,
    pub unsupported_calls: usize,
    /// Why the attempt could not be scored at all — a timeout, a provider that refused. Distinct
    /// from `accepted: false`, which means it ran and the work was not good enough.
    pub error: Option<String>,
}

impl Outcome {
    /// The single line a terminal reader wants.
    pub fn summary(&self) -> String {
        let verdict = match (&self.error, self.accepted) {
            (Some(error), _) => format!("error  {}", error),
            (None, true) => "pass".to_string(),
            (None, false) => "fail".to_string(),
        };
        format!(
            "{:<28} {:<8} {:>6.1}s  {} calls, {} failed, {} unsupported",
            self.task,
            verdict,
            self.seconds,
            self.tool_calls,
            self.failed_calls,
            self.unsupported_calls
        )
    }
}

/// TOML, for the same reason the knowledge snapshot is: a report is only useful next to the last
/// one, and diffing is how that comparison actually gets done.
pub fn report_toml(outcomes: &[Outcome]) -> String {
    let accepted = outcomes.iter().filter(|o| o.accepted).count();
    let mut out = format!(
        "procyon = \"{}\"\ntasks = {}\naccepted = {}\n",
        env!("CARGO_PKG_VERSION"),
        outcomes.len(),
        accepted
    );

    for outcome in outcomes {
        out.push_str("\n[[task]]\n");
        out.push_str(&format!("name = \"{}\"\n", outcome.task));
        out.push_str(&format!("accepted = {}\n", outcome.accepted));
        out.push_str(&format!("seconds = {:.1}\n", outcome.seconds));
        out.push_str(&format!("tool_calls = {}\n", outcome.tool_calls));
        out.push_str(&format!("failed_calls = {}\n", outcome.failed_calls));
        out.push_str(&format!(
            "unsupported_calls = {}\n",
            outcome.unsupported_calls
        ));
        if let Some(error) = &outcome.error {
            out.push_str(&format!("error = {}\n", toml_string(error)));
        }
    }
    out
}

/// Escapes a string for TOML. Error text is arbitrary — a provider message, a path — and an
/// unescaped quote in it would make the report unparseable, which is the one thing it must not be.
fn toml_string(value: &str) -> String {
    let escaped = value
        .replace('\\', "\\\\")
        .replace('"', "\\\"")
        .replace('\n', " ");
    format!("\"{}\"", escaped)
}

/// Runs one task and scores it.
///
/// The workspace is a copy and the process's current directory is moved into it, because that is
/// what the tools resolve paths against. Restored afterwards even when the run fails, or the next
/// task would start somewhere nobody chose.
pub async fn run_task(cfg: &crate::config::AppConfig, task: &Task) -> Outcome {
    let mut outcome = Outcome {
        task: task.name.clone(),
        accepted: false,
        seconds: 0.0,
        tool_calls: 0,
        failed_calls: 0,
        unsupported_calls: 0,
        error: None,
    };

    let workspace = match tempfile::tempdir() {
        Ok(dir) => dir,
        Err(e) => {
            outcome.error = Some(format!("could not make a workspace: {}", e));
            return outcome;
        }
    };
    if let Some(seed) = &task.seed {
        if let Err(e) = copy_dir(seed, workspace.path()) {
            outcome.error = Some(format!("could not seed the workspace: {}", e));
            return outcome;
        }
    }

    let restore = std::env::current_dir().ok();
    if let Err(e) = std::env::set_current_dir(workspace.path()) {
        outcome.error = Some(format!("could not enter the workspace: {}", e));
        return outcome;
    }

    let started = Instant::now();
    // `--allow-changes`, because a task nobody may write for cannot be completed. The mainnet gate
    // and the secret-key refusal are untouched by that: they are `Deny`, not `Confirm`, and a
    // benchmark that could sign on mainnet would be a benchmark nobody should run.
    let attempt = tokio::time::timeout(
        task.timeout,
        crate::runtime::run_once(cfg, &task.prompt, true),
    )
    .await;
    outcome.seconds = started.elapsed().as_secs_f64();

    match attempt {
        Ok(Ok(run)) => {
            outcome.tool_calls = run.tool_calls();
            outcome.failed_calls = run.failed_calls();
            outcome.unsupported_calls = run.unsupported_calls();
            match accept(&task.verify) {
                Ok(accepted) => outcome.accepted = accepted,
                Err(e) => outcome.error = Some(format!("acceptance script: {}", e)),
            }
        }
        Ok(Err(e)) => outcome.error = Some(e.to_string()),
        Err(_) => {
            outcome.error = Some(format!("timed out after {}s", task.timeout.as_secs()));
            // Scored anyway: a task the harness could not finish inside the allowance is a result,
            // and whether the work happens to pass regardless is worth knowing.
            if let Ok(accepted) = accept(&task.verify) {
                outcome.accepted = accepted;
            }
        }
    }

    if let Some(restore) = restore {
        let _ = std::env::set_current_dir(restore);
    }
    outcome
}

/// Runs the acceptance script in the current directory.
///
/// Its output is captured rather than inherited, and forwarded to stderr — a `cargo test` run
/// writes to stdout, and stdout is where the report goes, so inheriting it made
/// `--bench > report.toml` produce a file that was part report and part compiler log. Forwarded
/// rather than dropped, because when a task fails this is the only explanation there is.
fn accept(verify: &Path) -> Result<bool> {
    let output = std::process::Command::new("sh")
        .arg(verify)
        .output()
        .with_context(|| format!("running {}", verify.display()))?;

    for stream in [&output.stdout, &output.stderr] {
        if !stream.is_empty() {
            eprint!("{}", String::from_utf8_lossy(stream));
        }
    }
    Ok(output.status.success())
}

/// Copies a seed workspace. Shallow-recursive and deliberately dumb: a seed is a handful of source
/// files, and anything that needs more than this belongs in a fixture the task fetches itself.
fn copy_dir(from: &Path, to: &Path) -> std::io::Result<()> {
    for entry in std::fs::read_dir(from)?.flatten() {
        let source = entry.path();
        let target = to.join(entry.file_name());
        if source.is_dir() {
            std::fs::create_dir_all(&target)?;
            copy_dir(&source, &target)?;
        } else {
            std::fs::copy(&source, &target)?;
        }
    }
    Ok(())
}

#[cfg(test)]
mod tests {
    use super::*;

    fn task_dir(manifest: &str, with_verify: bool) -> tempfile::TempDir {
        let dir = tempfile::tempdir().unwrap();
        std::fs::write(dir.path().join("task.toml"), manifest).unwrap();
        if with_verify {
            std::fs::write(dir.path().join("verify.sh"), "exit 0\n").unwrap();
        }
        dir
    }

    #[test]
    fn a_task_is_read_from_its_manifest() {
        let dir = task_dir(
            "name = \"deploy-counter\"\nprompt = \"Deploy the counter.\"\ntimeout_secs = 60\n",
            true,
        );
        let task = load_task(dir.path()).unwrap();

        assert_eq!(task.name, "deploy-counter");
        assert_eq!(task.prompt, "Deploy the counter.");
        assert_eq!(task.timeout, Duration::from_secs(60));
        assert!(task.seed.is_none(), "no workspace/ means an empty start");
    }

    // A task with no acceptance script cannot be failed, and a suite of those would report a
    // perfect score for doing nothing.
    #[test]
    fn a_task_without_an_acceptance_script_is_refused() {
        let dir = task_dir("prompt = \"Do something.\"\n", false);
        let err = load_task(dir.path()).unwrap_err().to_string();
        assert!(err.contains("acceptance script"), "got {}", err);
    }

    #[test]
    fn a_task_without_a_prompt_is_refused() {
        let dir = task_dir("name = \"x\"\n", true);
        assert!(load_task(dir.path())
            .unwrap_err()
            .to_string()
            .contains("prompt"));
    }

    #[test]
    fn a_zero_or_missing_timeout_falls_back_to_the_default() {
        for manifest in [
            "prompt = \"p\"\n",
            "prompt = \"p\"\ntimeout_secs = 0\n",
            "prompt = \"p\"\ntimeout_secs = -5\n",
        ] {
            let dir = task_dir(manifest, true);
            assert_eq!(load_task(dir.path()).unwrap().timeout, DEFAULT_TIMEOUT);
        }
    }

    #[test]
    fn a_seed_workspace_is_picked_up_when_it_exists() {
        let dir = task_dir("prompt = \"p\"\n", true);
        std::fs::create_dir(dir.path().join("workspace")).unwrap();
        assert!(load_task(dir.path()).unwrap().seed.is_some());
    }

    #[test]
    fn tasks_load_in_a_stable_order() {
        let root = tempfile::tempdir().unwrap();
        for name in ["zeta", "alpha", "mid"] {
            let dir = root.path().join(name);
            std::fs::create_dir(&dir).unwrap();
            std::fs::write(
                dir.join("task.toml"),
                format!("name = \"{}\"\nprompt = \"p\"\n", name),
            )
            .unwrap();
            std::fs::write(dir.join("verify.sh"), "exit 0\n").unwrap();
        }

        let names: Vec<String> = load_tasks(root.path())
            .unwrap()
            .into_iter()
            .map(|t| t.name)
            .collect();
        assert_eq!(names, ["alpha", "mid", "zeta"]);
    }

    #[test]
    fn an_empty_task_directory_is_an_error_rather_than_an_empty_pass() {
        let root = tempfile::tempdir().unwrap();
        assert!(load_tasks(root.path()).is_err());
    }

    fn outcome(accepted: bool, error: Option<&str>) -> Outcome {
        Outcome {
            task: "deploy-counter".to_string(),
            accepted,
            seconds: 12.34,
            tool_calls: 7,
            failed_calls: 1,
            unsupported_calls: 2,
            error: error.map(str::to_string),
        }
    }

    // "Ran and the work was not good enough" and "could not be scored" are different results, and
    // a report that renders them the same is a report that hides broken runs.
    #[test]
    fn a_failure_and_an_error_read_differently() {
        assert!(outcome(false, None).summary().contains("fail"));
        assert!(outcome(false, Some("timed out after 900s"))
            .summary()
            .contains("error"));
        assert!(outcome(true, None).summary().contains("pass"));
    }

    #[test]
    fn the_report_carries_every_metric_and_parses() {
        let report = report_toml(&[outcome(true, None), outcome(false, Some("timed out"))]);
        let parsed: toml::Value = report.parse().expect("a report must parse");

        assert_eq!(parsed["tasks"].as_integer(), Some(2));
        assert_eq!(parsed["accepted"].as_integer(), Some(1));
        assert_eq!(parsed["task"][0]["tool_calls"].as_integer(), Some(7));
        assert_eq!(parsed["task"][0]["unsupported_calls"].as_integer(), Some(2));
        assert_eq!(parsed["task"][1]["error"].as_str(), Some("timed out"));
    }

    // Error text is arbitrary — a provider message, a path with a quote in it — and the one thing
    // the report must never be is unparseable.
    #[test]
    fn an_error_containing_quotes_does_not_break_the_report() {
        let report = report_toml(&[outcome(false, Some("no identity named \"alice\"\nretry"))]);
        let parsed: toml::Value = report.parse().expect("a report must parse");
        assert_eq!(
            parsed["task"][0]["error"].as_str(),
            Some("no identity named \"alice\" retry")
        );
    }

    #[test]
    fn a_seed_is_copied_including_its_subdirectories() {
        let from = tempfile::tempdir().unwrap();
        let to = tempfile::tempdir().unwrap();
        std::fs::create_dir_all(from.path().join("contracts/counter/src")).unwrap();
        std::fs::write(from.path().join("contracts/counter/src/lib.rs"), "// seed").unwrap();
        std::fs::write(from.path().join("Cargo.toml"), "[workspace]").unwrap();

        copy_dir(from.path(), to.path()).unwrap();

        assert_eq!(
            std::fs::read_to_string(to.path().join("contracts/counter/src/lib.rs")).unwrap(),
            "// seed"
        );
        assert!(to.path().join("Cargo.toml").exists());
    }

    #[test]
    fn acceptance_is_the_scripts_verdict_and_nothing_else() {
        let dir = tempfile::tempdir().unwrap();
        let pass = dir.path().join("pass.sh");
        let fail = dir.path().join("fail.sh");
        std::fs::write(&pass, "exit 0\n").unwrap();
        std::fs::write(&fail, "echo nope >&2\nexit 1\n").unwrap();

        assert!(accept(&pass).unwrap());
        assert!(!accept(&fail).unwrap());
    }
}