thirdshift 0.37.0

Turn a GitHub issue into a ready-for-review pull request, unattended
use std::fs;
use std::path::Path;

use serde_json::json;

use super::{Recorder, arms, collect, web_search};
use crate::harness::Harness;
use crate::proxy::{Accounting, HarnessInvocation};
use crate::trial::{Outcome, Output, VoidCause};

fn attempt() -> tempfile::TempDir {
    let root = tempfile::tempdir().unwrap();
    for directory in ["session", "output", "repository"] {
        fs::create_dir_all(root.path().join(directory)).unwrap();
    }
    fs::write(
        root.path().join("trace-context.json"),
        serde_json::to_vec(&crate::logs::trace::Context::of_session(root.path())).unwrap(),
    )
    .unwrap();
    root
}

fn runner(root: &Path, manifest: serde_json::Value) -> arms::Runner {
    serde_json::from_value(json!({
        "name": "fixture", "directory": root, "manifest": manifest,
        "files": {}, "provenance": {}
    }))
    .unwrap()
}

fn delivered() -> Output {
    let mut output = Output::failed(1_000, "Custom runner failed");
    output.outcome = Outcome::Delivered;
    output.cause = None;
    output
}

fn sdk_session(root: &Path) {
    let output = root.join("output");
    fs::write(output.join("review.jsonl"), "review\n").unwrap();
    fs::write(
        output.join("sessions.json"),
        serde_json::to_vec(&json!([{
            "role":"review", "transcript_file":"review.jsonl",
            "started_at":"2026-11-14T15:00:00Z", "ended_at":"2026-11-14T15:00:02Z",
            "models":{"sdk-model":{"uncached_input_tokens":1,"cache_read_tokens":0,
                "cache_write_tokens":0,"output_tokens":1}}
        }]))
        .unwrap(),
    )
    .unwrap();
}

#[test]
fn timed_out_runner_after_provider_traffic_stays_failed() {
    let root = attempt();
    let mut recorder = Recorder::start(root.path()).unwrap();
    let runner = runner(
        root.path(),
        json!({"command": ["runner"], "providers": ["openai"]}),
    );
    let traffic = Accounting {
        provider_connections: 1,
        harness_invocations: vec![HarnessInvocation {
            harness: Harness::Codex,
            provider_connections: 1,
        }],
    };
    let mut output = Output::failed(30_000, "Trial timed out");
    collect(
        &mut recorder,
        &runner,
        &traffic,
        &mut output,
        false,
        &root.path().join("signal"),
    )
    .unwrap();
    assert!(output.outcome == Outcome::Failed, "{:?}", output.cause);
    assert_eq!(output.void_cause, None);
    assert_eq!(output.cause.as_deref(), Some("Trial timed out"));
}

#[test]
fn undeclared_runner_sending_provider_traffic_without_sessions_is_an_accounting_gap() {
    let root = attempt();
    let mut recorder = Recorder::start(root.path()).unwrap();
    let runner = runner(root.path(), json!({"command": ["runner"]}));
    let traffic = Accounting {
        provider_connections: 1,
        harness_invocations: Vec::new(),
    };
    let mut output = delivered();
    collect(
        &mut recorder,
        &runner,
        &traffic,
        &mut output,
        true,
        &root.path().join("signal"),
    )
    .unwrap();
    assert!(output.outcome == Outcome::Void, "{:?}", output.cause);
    assert_eq!(output.void_cause, Some(VoidCause::AccountingGap));
}

#[test]
fn harness_run_across_the_proxy_without_a_harness_session_is_an_accounting_gap() {
    let root = attempt();
    sdk_session(root.path());
    let mut recorder = Recorder::start(root.path()).unwrap();
    let runner = runner(root.path(), json!({"command": ["runner"]}));
    let traffic = Accounting {
        provider_connections: 2,
        harness_invocations: vec![HarnessInvocation {
            harness: Harness::Claude,
            provider_connections: 1,
        }],
    };
    let mut output = delivered();
    collect(
        &mut recorder,
        &runner,
        &traffic,
        &mut output,
        true,
        &root.path().join("signal"),
    )
    .unwrap();
    assert!(output.outcome == Outcome::Void, "{:?}", output.cause);
    assert_eq!(output.void_cause, Some(VoidCause::AccountingGap));
    assert!(
        output.cause.as_deref().unwrap().contains("claude"),
        "{:?}",
        output.cause
    );
}

#[test]
fn web_search_scan_skips_events_from_other_arm_builds() {
    let attempt = tempfile::tempdir().unwrap();
    fs::create_dir(attempt.path().join("session")).unwrap();
    let events = serde_json::to_string(&json!([
        {"type": "event_from_a_newer_arm_build"},
        {"type": "tool_call_started", "id": "s", "parent_id": null,
         "tool": "WebSearch", "input": {"query": "fix"}}
    ]))
    .unwrap();
    let line = json!({"resourceLogs":[{"scopeLogs":[{"logRecords":[{"attributes":[
        {"key":"thirdshift.events","value":{"stringValue": events}}]}]}]}]});
    fs::write(
        attempt.path().join("session/trial.logs.jsonl"),
        format!("{line}\n"),
    )
    .unwrap();
    assert!(web_search(attempt.path()).unwrap());
}

#[test]
fn custom_runner_results_require_a_confined_bounded_regular_file() {
    use std::os::unix::fs::symlink;
    use std::os::unix::process::ExitStatusExt;

    let process = std::process::Output {
        status: std::process::ExitStatus::from_raw(0),
        stdout: Vec::new(),
        stderr: Vec::new(),
    };
    for case in [
        "regular",
        "at-limit",
        "too-long",
        "file-symlink",
        "directory-symlink",
        "directory",
        "missing",
    ] {
        let root = attempt();
        let directory = root.path().join("output");
        let path = directory.join("result.json");
        let mut bytes = br#"{"version":1,"outcome":"delivered"}"#.to_vec();
        if matches!(case, "at-limit" | "too-long") {
            let limit = usize::try_from(super::RESULT_LIMIT).unwrap();
            bytes.resize(limit + usize::from(case == "too-long"), b' ');
        }
        fs::write(&path, bytes).unwrap();
        match case {
            "file-symlink" => {
                fs::rename(&path, directory.join("original.json")).unwrap();
                symlink("original.json", &path).unwrap();
            }
            "directory-symlink" => {
                let moved = root.path().join("original-output");
                fs::rename(&directory, &moved).unwrap();
                symlink(moved, &directory).unwrap();
            }
            "directory" => {
                fs::remove_file(&path).unwrap();
                fs::create_dir(&path).unwrap();
            }
            "missing" => fs::remove_file(&path).unwrap(),
            _ => {}
        }
        let (output, reported) = super::result(&process, root.path(), 1);
        let accepted = matches!(case, "regular" | "at-limit");
        assert_eq!(reported, accepted, "{case}: {:?}", output.cause);
        assert!(
            output.outcome
                == if accepted {
                    Outcome::Delivered
                } else {
                    Outcome::Failed
                },
            "{case}: {:?}",
            output.cause
        );
        if case == "too-long" {
            assert!(output.cause.as_deref().unwrap().contains("longer than"));
        }
    }
}