use std::fs;
use std::path::{Path, PathBuf};
use std::time::{Duration, Instant};
use anyhow::{Context, Result};
use serde::{Deserialize, Serialize};
use super::evidence::Started;
use super::grading::process::{self, GraderRun, Source};
use super::trial_results::files::{Files, Native, directory, metadata};
use super::{Arm, Task, TrialResult, store, trial_results, trial_sessions};
use crate::evals::{format, repository};
use crate::sandbox::code_grader::{Check, CheckRun};
use crate::{interrupt, progress};
#[derive(Serialize, Deserialize)]
pub(crate) struct Pair {
pub suite: String,
pub task: String,
pub trial: usize,
pub baseline: Side,
pub candidate: Side,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub panel: Option<super::pairwise::PanelCalls>,
}
impl Pair {
pub(crate) fn key(&self) -> String {
trial_results::pair_key(&self.suite, &self.task, self.trial)
}
}
#[derive(Default, Serialize, Deserialize)]
pub(crate) struct Side {
#[serde(default)]
pub diff: Vec<FileDiff>,
pub test_files: Vec<String>,
pub fails_on_base: Option<bool>,
pub passes_on_other: Option<bool>,
pub disagreement: bool,
pub base: Vec<Check>,
pub cross: Vec<Check>,
pub error: Option<String>,
}
#[derive(Serialize, Deserialize)]
pub(crate) struct FileDiff {
pub path: String,
pub test: bool,
pub patch: String,
}
pub(super) fn run(experiment: &store::Stored) -> Result<()> {
let bar = &experiment.description.bar;
let timeout = Duration::from_secs(bar.timeout_seconds);
let pairs = experiment.directory.join("pairs");
let mut caps = None;
for task in &experiment.tasks {
let Some(new_tests) = task
.manifest
.process_grading
.as_ref()
.and_then(|grading| grading.new_tests.as_ref())
else {
continue;
};
for index in 0..bar.trials_per_task[&task.suite] {
let path = pairs.join(format!(
"{}.json",
trial_results::pair_key(&task.suite, &task.name, index)
));
let identities = [Arm::Baseline, Arm::Candidate].map(|arm| trial_results::Identity {
suite: &task.suite,
task: &task.name,
arm,
trial: index,
});
if path.exists()
|| identities
.iter()
.any(|identity| !trial_results::finished(&experiment.directory, identity))
{
continue;
}
interrupt::check()?;
progress::step(format!(
"Pair evidence {}/{} #{}: base-tests and cross-tests",
task.suite,
task.name,
index + 1
));
let caps = match &mut caps {
Some(caps) => caps,
None => caps.insert(crate::config::UserConfig::load()?.trial_caps.with_env()?),
};
let [baseline, candidate] = identities.map(|identity| {
last_commit(&experiment.directory, &identity).map_err(|error| format!("{error:#}"))
});
let evidence = Evidence {
task,
new_tests,
timeout,
caps,
};
let pair = Pair {
suite: task.suite.clone(),
task: task.name.clone(),
trial: index,
baseline: evidence.side(resolved(&baseline), resolved(&candidate))?,
candidate: evidence.side(resolved(&candidate), resolved(&baseline))?,
panel: None,
};
fs::create_dir_all(&pairs)?;
store::write(&path, &pair)?;
}
}
Ok(())
}
pub(super) fn read(files: &impl Files, root: &Path) -> Result<Vec<Pair>> {
let mut pairs: Vec<Pair> = Vec::new();
for entry in directory(files, &root.join("pairs"))? {
if entry.kind.is_file() && entry.path.extension().is_some_and(|ext| ext == "json") {
pairs.push(serde_json::from_slice(&files.read(&entry.path)?)?);
}
}
pairs.sort_by(|a, b| (&a.suite, &a.task, a.trial).cmp(&(&b.suite, &b.task, b.trial)));
Ok(pairs)
}
pub(super) fn lines(pairs: &[Pair]) -> String {
let mut text = String::new();
for pair in pairs {
for (arm, other, side) in [
("Baseline", "Candidate", &pair.baseline),
("Candidate", "Baseline", &pair.candidate),
] {
let mut findings = Vec::new();
if let Some(error) = &side.error {
findings.push(format!("evidence unavailable: {error}"));
}
match side.fails_on_base {
Some(false) => findings.push("pass on the base".to_owned()),
None if !side.base.is_empty() => {
findings.push("did not finish on the base".to_owned());
}
_ => {}
}
match side.passes_on_other {
Some(false) => {
findings.push(format!("fail on the {other}'s commit (disagreement)"));
}
None if !side.cross.is_empty() => {
findings.push(format!("did not finish on the {other}'s commit"));
}
_ => {}
}
for finding in findings {
text.push_str(&format!(
"\nPair {}/{} #{} · {arm} tests {finding}",
pair.suite,
pair.task,
pair.trial + 1
));
}
}
}
text
}
pub(super) struct Commit {
pub repository: PathBuf,
pub commit: String,
}
pub(super) fn trial_result(
experiment: &Path,
identity: &trial_results::Identity<'_>,
) -> Result<TrialResult> {
Ok(serde_json::from_slice(&fs::read(
experiment
.join("results")
.join(format!("{}.json", identity.key())),
)?)?)
}
pub(super) fn last_commit(
experiment: &Path,
identity: &trial_results::Identity<'_>,
) -> Result<Commit> {
let key = identity.key();
let result = trial_result(experiment, identity)?;
let last = result
.attempts
.len()
.checked_sub(1)
.context("Trial has no attempts")?;
for attempt in trial_sessions::trial(&Native, &experiment.join("trials").join(&key))? {
let started = metadata::<Started>(&Native, &attempt.path.join("started.json"))?;
if started.is_some_and(|started| started.index == last) {
let repository = attempt.path.join("repository");
let commit = process::head(&repository, Instant::now() + Duration::from_secs(60))
.context("Trial repository is no longer retained")?;
return Ok(Commit { repository, commit });
}
}
anyhow::bail!("Trial's last attempt is no longer retained")
}
fn diff(
own: &Commit,
base: &str,
new_tests: &format::NewTests,
deadline: Instant,
) -> Result<Vec<FileDiff>> {
let mut files = Vec::new();
for path in process::touched(&own.repository, base, &own.commit, deadline)? {
let path = path.as_str();
let patch = repository::bytes_until(
&own.repository,
&[
"diff",
"--no-color",
"--no-ext-diff",
"--no-textconv",
"--no-renames",
base,
&own.commit,
"--",
path,
],
deadline,
)?;
files.push(FileDiff {
path: path.to_owned(),
test: new_tests.matches(path) || crate::evals::process_tasks::is_test_path(path),
patch: String::from_utf8_lossy(&patch).into_owned(),
});
}
files.sort_by(|a, b| (a.test, &a.path).cmp(&(b.test, &b.path)));
Ok(files)
}
fn resolved(commit: &Result<Commit, String>) -> Result<&Commit, String> {
commit.as_ref().map_err(Clone::clone)
}
struct Evidence<'a> {
task: &'a Task,
new_tests: &'a format::NewTests,
timeout: Duration,
caps: &'a crate::scope::Caps,
}
fn outcome(ran: &[CheckRun]) -> Option<bool> {
if ran.iter().any(|ran| ran.exited && !ran.check.passed) {
Some(false)
} else {
ran.iter().all(|ran| ran.exited).then_some(true)
}
}
impl Evidence<'_> {
fn side(&self, own: Result<&Commit, String>, other: Result<&Commit, String>) -> Result<Side> {
let mut side = Side::default();
let result = (|| -> Result<()> {
let own = own.map_err(anyhow::Error::msg)?;
let manifest = &self.task.manifest;
let deadline = Instant::now() + self.timeout;
side.diff = diff(own, &manifest.base, self.new_tests, deadline)?;
side.test_files =
process::changed(&own.repository, &manifest.base, &own.commit, deadline)?
.into_iter()
.filter(|path| self.new_tests.matches(path))
.collect();
if side.test_files.is_empty() {
return Ok(());
}
let commands: Vec<_> = side
.test_files
.iter()
.map(|path| self.new_tests.command_for(path))
.collect();
let tests = || Source {
repository: &own.repository,
commit: &own.commit,
};
let source = self.task.directory.join(&manifest.repository);
let base = process::overlaid(
Source {
repository: &source,
commit: &manifest.base,
},
tests(),
&side.test_files,
&commands,
GraderRun {
name: "base-test",
deadline,
caps: self.caps,
},
)?;
side.fails_on_base = outcome(&base).map(|passed| !passed);
side.base = base.into_iter().map(|ran| ran.check).collect();
let other = other.map_err(|error| anyhow::anyhow!("other Arm: {error}"))?;
let cross = process::overlaid(
Source {
repository: &other.repository,
commit: &other.commit,
},
tests(),
&side.test_files,
&commands,
GraderRun {
name: "cross-test",
deadline: Instant::now() + self.timeout,
caps: self.caps,
},
)?;
side.passes_on_other = outcome(&cross);
side.disagreement = side.passes_on_other == Some(false);
side.cross = cross.into_iter().map(|ran| ran.check).collect();
Ok(())
})();
if let Err(error) = result {
interrupt::check()?;
side.error = Some(format!("{error:#}"));
}
Ok(side)
}
}