use std::path::PathBuf;
use anyhow::Result;
use serde::Serialize;
use super::arms::{Check, Runner};
use super::experiment::dry_run::{self, DryRun};
pub struct Request {
pub runner: String,
pub task: Option<(String, String)>,
pub json: bool,
}
#[derive(Serialize)]
pub(crate) struct Report {
pub version: u32,
pub runner: String,
pub manifest: Option<PathBuf>,
pub digest: Option<String>,
pub ok: bool,
pub problems: Vec<Problem>,
pub trial: Option<DryRun>,
}
#[derive(Serialize)]
pub(crate) struct Problem {
pub check: Check,
pub message: String,
}
pub(crate) fn check_names() -> [&'static str; 6] {
Check::ALL.map(Check::name)
}
pub fn execute(request: &Request) -> Result<(String, bool)> {
let report = check(request)?;
let text = if request.json {
serde_json::to_string_pretty(&report)?
} else {
render(&report)
};
Ok((text, report.ok))
}
fn check(request: &Request) -> Result<Report> {
let task = request
.task
.as_ref()
.map(|(suite, task)| dry_run::task(suite, task))
.transpose()?;
let frozen = tempfile::Builder::new()
.prefix("thirdshift-check-runner-")
.tempdir()?;
let mut problems = Vec::new();
let runner = Runner::freeze_reporting(&request.runner, frozen.path(), &mut |check, error| {
problems.push(Problem {
check,
message: format!("{error:#}"),
});
Ok(())
})?;
let digest = match &runner {
Some(runner) if problems.is_empty() => Some(runner.digest()?),
_ => None,
};
if let Some(runner) = &runner {
let missing = crate::sandbox::custom::missing_logins(runner, &crate::config::home()?)?;
for (check, messages) in [
(Check::Logins, missing.harnesses),
(Check::Providers, missing.providers),
] {
problems.extend(
messages
.into_iter()
.map(|message| Problem { check, message }),
);
}
let prices = crate::trial::accounting::Prices::default();
for (name, role) in &runner.manifest.roles {
if !prices.models.contains_key(&role.model) {
problems.push(Problem {
check: Check::Roles,
message: format!(
"role {name} names Model {}, which the Trial price table ({}) does not price",
role.model, prices.date
),
});
}
}
}
let trial = match (runner, task) {
(Some(runner), Some(task)) if problems.is_empty() => Some(dry_run::run(&runner, &task)?),
_ => None,
};
Ok(Report {
version: 1,
runner: request.runner.clone(),
manifest: Runner::manifest_path(&request.runner).ok(),
digest,
ok: problems.is_empty() && trial.as_ref().is_none_or(|trial| trial.passed),
problems,
trial,
})
}
fn render(report: &Report) -> String {
let mut lines = vec![match &report.manifest {
Some(path) => format!("Custom runner {} ({})", report.runner, path.display()),
None => format!("Custom runner {}", report.runner),
}];
if let Some(digest) = &report.digest {
lines.push(format!("Digest: {digest}"));
}
if report.problems.is_empty() {
lines.push("Static checks: passed".into());
} else {
lines.push("Static checks: failed".into());
for problem in &report.problems {
lines.push(format!(" {}: {}", problem.check.name(), problem.message));
}
}
if let Some(trial) = &report.trial {
lines.push(format!(
"Dry run {}/{}: {}{}; Score {}",
trial.suite,
trial.task,
trial.outcome,
trial
.void_cause
.map(|cause| format!(" ({cause})"))
.unwrap_or_default(),
if trial.passed { "passed" } else { "failed" }
));
if let Some(cause) = &trial.cause {
lines.push(format!(" Cause: {cause}"));
}
for check in &trial.checks {
lines.push(format!(
" {} {}",
if check.passed { "passed" } else { "failed" },
check.reason
));
}
for session in &trial.sessions {
let models = session
.models
.keys()
.cloned()
.collect::<Vec<_>>()
.join(", ");
lines.push(match (&session.cost_usd, &session.gap) {
(Some(cost), _) => format!(" Session {}: ${cost:.6} ({models})", session.role),
(None, Some(gap)) => format!(" Session {}: unpriced: {gap}", session.role),
(None, None) => format!(" Session {}: unpriced", session.role),
});
}
lines.push(match (trial.sessions.is_empty(), trial.cost_usd) {
(true, _) => " Sessions: none recorded".into(),
(false, Some(cost)) => format!(" Cost: ${cost:.6}"),
(false, None) => " Cost: incomplete".into(),
});
}
lines.push(if report.ok {
"Custom runner check passed.".into()
} else {
"Custom runner check failed.".into()
});
lines.join("\n")
}