use std::path::{Path, PathBuf};
use anyhow::{Result, ensure};
use chrono::{DateTime, FixedOffset};
use serde::{Deserialize, Serialize};
use super::Work;
use super::effects::{OnMachine, Outside};
use crate::harness::Choice;
use crate::issue::IssueUrl;
#[derive(Clone, Serialize, Deserialize)]
pub(super) struct CommandTree {
#[serde(flatten, deserialize_with = "read_command")]
pub command: CommandRecord,
#[serde(default)]
pub dispatched: Vec<CommandRecord>,
}
fn read_command<'de, D: serde::Deserializer<'de>>(reader: D) -> Result<CommandRecord, D::Error> {
let value = serde_json::Value::deserialize(reader)?;
serde_json::from_value(value).map_err(serde::de::Error::custom)
}
impl CommandTree {
fn active_dispatch_index(&self) -> Option<usize> {
self.dispatched
.iter()
.rposition(|dispatched| dispatched.outcome.is_none())
}
pub fn active(&mut self) -> &mut CommandRecord {
match self.active_dispatch_index() {
Some(index) => &mut self.dispatched[index],
None => &mut self.command,
}
}
pub fn active_ref(&self) -> &CommandRecord {
self.active_dispatch_ref().unwrap_or(&self.command)
}
pub fn active_dispatch_ref(&self) -> Option<&CommandRecord> {
let index = self.active_dispatch_index()?;
Some(&self.dispatched[index])
}
pub fn active_dispatch(&mut self) -> Option<&mut CommandRecord> {
let index = self.active_dispatch_index()?;
Some(&mut self.dispatched[index])
}
}
pub(crate) fn experiment_rechecked(log: &Path, id: &str, outcome: &str) -> Result<()> {
let path = log.with_extension("command.json");
let bytes = match std::fs::read(&path) {
Ok(bytes) => bytes,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()),
Err(error) => return Err(error.into()),
};
let mut tree: CommandTree = serde_json::from_slice(&bytes)?;
ensure!(
tree.command.kind == "Experiment"
&& tree
.command
.experiment
.as_ref()
.is_some_and(|experiment| experiment.id == id)
&& tree.command.outcome.is_some(),
"Command record does not belong to the completed Experiment"
);
tree.command.outcome = Some(outcome.into());
OnMachine.replace_record(&path, &serde_json::to_vec_pretty(&tree)?)
}
#[derive(Clone, Serialize, Deserialize)]
pub(super) struct CommandRecord {
pub version: u32,
pub kind: String,
pub repository: String,
#[serde(default)]
pub name: String,
pub issue: Option<Issue>,
pub hostname: String,
pub process_id: u32,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub thirdshift_version: Option<String>,
#[serde(skip)]
pub version_recovered: bool,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub started_by: Option<super::Actor>,
pub process_start: String,
pub harness: String,
pub model: Option<String>,
pub effort: Option<String>,
pub started_at: String,
#[serde(default)]
pub ended_at: Option<String>,
#[serde(skip)]
pub end_time_estimated: bool,
pub stages: Vec<Stage>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub quota_wait_until: Option<String>,
pub session_log: Option<PathBuf>,
pub outcome: Option<String>,
#[serde(default)]
pub run_metrics: Option<super::run_metrics::RunMetrics>,
pub cause: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub stopped_at: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub failure_category: Option<FailureCategory>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub waiting_for: Vec<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub limit: Option<crate::session::limits::Limit>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub resume_command: Option<String>,
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub handed_back_for_pickup: bool,
pub issue_branch: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub base_branch: Option<String>,
pub pull_request: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub pull_request_title: Option<String>,
#[serde(default)]
pub parent_record: Option<PathBuf>,
#[serde(default)]
pub parent_dispatch: Option<String>,
#[serde(default)]
pub tickets: Vec<TicketRecord>,
#[serde(default)]
pub security_findings_waiting: bool,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub security_findings: Vec<crate::security::RecordedFinding>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub security_fix_findings: Vec<crate::security::RecordedFinding>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub experiment: Option<crate::evals::experiment::Description>,
#[serde(flatten)]
pub process: crate::processes::Identity,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub(crate) enum FailureCategory {
Held,
Waiting,
Crashed,
StageFailed,
}
impl FailureCategory {
pub(crate) const ALL: [Self; 4] = [Self::Held, Self::Waiting, Self::Crashed, Self::StageFailed];
pub(crate) fn name(self) -> &'static str {
match self {
Self::Held => "held",
Self::Waiting => "waiting",
Self::Crashed => "crashed",
Self::StageFailed => "stage_failed",
}
}
}
#[derive(Clone, Serialize, Deserialize)]
pub(crate) struct Issue {
pub number: u64,
pub url: String,
pub title: Option<String>,
}
impl From<&IssueUrl> for Issue {
fn from(issue: &IssueUrl) -> Self {
Self {
number: issue.number,
url: issue.url.clone(),
title: None,
}
}
}
#[derive(Clone, Serialize, Deserialize)]
pub(crate) struct TicketRecord {
pub issue: Issue,
pub state: TicketState,
pub blockers: Vec<Issue>,
}
#[derive(Clone, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub(crate) enum TicketState {
Running,
Merged,
Failed,
Waiting,
#[serde(rename = "not started")]
NotStarted,
}
#[derive(Clone, Serialize, Deserialize)]
pub(super) struct Stage {
pub name: String,
pub began_at: String,
pub session_log: Option<PathBuf>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub harness: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub model: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub effort: Option<String>,
#[serde(default, skip_serializing_if = "is_zero")]
pub suspended_seconds: u64,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub suspensions: Vec<Suspension>,
}
#[derive(Clone, Serialize, Deserialize)]
pub(super) struct Suspension {
pub resumed_at: DateTime<FixedOffset>,
pub seconds: u64,
}
impl Stage {
pub fn suspended_between(
&self,
since: DateTime<FixedOffset>,
until: DateTime<FixedOffset>,
) -> u64 {
if self.suspensions.is_empty() {
return DateTime::parse_from_rfc3339(&self.began_at)
.ok()
.filter(|began| since <= *began)
.map_or(0, |_| self.suspended_seconds);
}
self.suspensions
.iter()
.filter_map(|suspension| {
let duration =
chrono::Duration::try_seconds(i64::try_from(suspension.seconds).ok()?)?;
let began = suspension.resumed_at.checked_sub_signed(duration)?;
Some(
(suspension.resumed_at.min(until) - began.max(since))
.num_seconds()
.max(0) as u64,
)
})
.fold(0, u64::saturating_add)
.min(self.suspended_seconds)
}
}
fn is_zero(seconds: &u64) -> bool {
*seconds == 0
}
pub(super) struct StageTime<'a> {
pub stage: &'a Stage,
pub began: Option<DateTime<FixedOffset>>,
pub ended: Option<DateTime<FixedOffset>>,
}
impl StageTime<'_> {
pub fn seconds(&self) -> Option<f64> {
let (began, ended) = self.began.zip(self.ended)?;
let milliseconds = ended.signed_duration_since(began).num_milliseconds();
(milliseconds >= 0).then_some(milliseconds as f64 / 1000.0)
}
}
macro_rules! stage_names {
($($(#[$meta:meta])* $stage:ident => $name:literal,)*) => {
#[derive(Clone, Copy)]
pub enum StageName {
$($(#[$meta])* $stage,)*
}
impl StageName {
pub fn name(self) -> &'static str {
match self {
$(Self::$stage => $name,)*
}
}
pub(super) fn order(name: &str) -> usize {
match name {
$($name => Self::$stage as usize,)*
_ => usize::MAX,
}
}
}
};
}
stage_names! {
ProcessStep => "Process Step",
Implement => "implement",
CodeReview => "Code review",
Resume => "Resume",
SecurityReview => "Security review",
PullRequest => "pull request",
CiWatch => "CI watch",
Repair => "Repair",
SelfMerge => "Self-merge",
Cleanup => "cleanup",
ArchitectureReview => "Architecture review",
Tickets => "Tickets",
SpecReview => "Spec review",
SecurityAudit => "Security audit",
SecurityReproduction => "Security reproduction",
SecurityFixPublishing => "Security fix publishing",
SecurityCloseCheck => "Security close check",
WaitingForCommands => "Waiting for Commands",
ArmBuilds => "Arm builds",
EvalTasks => "Eval tasks",
Trials => "Trials",
Sampling => "sampling",
Grading => "grading",
Triage => "Triage",
ReleaseSummary => "Release summary",
DispatchedRun => "dispatched Run",
DispatchedSpecRun => "dispatched Spec run",
}
impl StageName {
pub(super) fn dispatched(work: Work<'_>) -> Self {
match work {
Work::Starting { .. } => Self::ProcessStep,
Work::SpecRun(_) => Self::DispatchedSpecRun,
Work::Run(_)
| Work::BaseFix(_)
| Work::PickupRun(_)
| Work::ArchitectRun(_)
| Work::SecurityRun(_)
| Work::ScoringRun(_)
| Work::ReviewRun(_)
| Work::FixRun(_)
| Work::ReleaseRun(_)
| Work::TriageRun { .. }
| Work::Experiment(_)
| Work::Process { .. } => Self::DispatchedRun,
}
}
pub(super) fn is_dispatch(name: &str) -> bool {
[Self::DispatchedRun, Self::DispatchedSpecRun]
.into_iter()
.any(|stage| stage.name() == name)
}
pub(super) fn is_session(name: &str) -> bool {
[
Self::Implement,
Self::CodeReview,
Self::Resume,
Self::Repair,
Self::SpecReview,
Self::SecurityReview,
Self::ArchitectureReview,
Self::Triage,
Self::ReleaseSummary,
Self::SecurityAudit,
Self::SecurityReproduction,
Self::SecurityFixPublishing,
]
.into_iter()
.any(|stage| stage.name() == name)
}
pub(super) fn known(name: &str) -> bool {
Self::order(name) != usize::MAX
}
}
impl CommandRecord {
pub(super) fn process_label(&self) -> Option<&str> {
(self.kind == "Process").then(|| self.process.name.as_deref().unwrap_or(self.name.as_str()))
}
pub(super) fn process_heading(&self) -> Option<String> {
let name = self.process_label()?;
Some(match self.issue.as_ref() {
Some(issue) => format!("Process {name} · {}#{}", self.repository, issue.number),
None => format!("Process {name} on {}", self.repository),
})
}
pub fn stage_times(
&self,
live_until: Option<DateTime<FixedOffset>>,
) -> impl Iterator<Item = StageTime<'_>> {
self.stages.iter().enumerate().map(move |(index, stage)| {
let ended = if let Some(next) = self.stages.get(index + 1) {
DateTime::parse_from_rfc3339(&next.began_at).ok()
} else if self.outcome.is_some() {
self.ended_at
.as_deref()
.and_then(|end| DateTime::parse_from_rfc3339(end).ok())
} else {
live_until
};
StageTime {
stage,
began: DateTime::parse_from_rfc3339(&stage.began_at).ok(),
ended,
}
})
}
pub fn session_references(&self) -> impl Iterator<Item = &PathBuf> {
self.stages
.iter()
.filter_map(|stage| stage.session_log.as_ref())
.chain(self.session_log.as_ref())
}
pub fn finish(
&mut self,
account: &crate::run_ending::Account<'_>,
outside: &mut impl super::effects::Outside,
ranges: &super::usual_range::UsualRanges,
unaddressed_findings: Option<crate::pull_request::UnaddressedFindings>,
) {
self.ended_at = Some(outside.now().to_rfc3339());
self.outcome = Some(account.outcome.into());
self.cause = account.pre_audit_failure.map(str::to_owned).or_else(|| {
account
.ended
.as_ref()
.err()
.map(|cause| cause.full().to_owned())
});
let failure = account
.ended
.as_ref()
.err()
.filter(|_| account.outcome != "paused");
self.stopped_at = failure.and_then(|_| {
account
.recovery
.and_then(|recovery| recovery.stage().recorded())
.map(|stage| stage.name().to_owned())
.or_else(|| self.stopped_stage().map(str::to_owned))
});
self.failure_category = failure.map(|cause| cause.category(account.interrupted));
if let Some(pr_url) = account.pr_url {
self.pull_request = Some(pr_url.into());
}
self.security_findings_waiting = account.security_findings_waiting;
self.security_findings = account.security_findings.unwrap_or_default().to_vec();
if matches!(
self.kind.as_str(),
"Run"
| "Spec run"
| "Base fix"
| "Review run"
| "Triage run"
| "Fix run"
| "Release run"
) {
self.run_metrics = Some(super::run_metrics::RunMetrics::of(
self,
ranges,
outside,
unaddressed_findings,
));
}
}
pub fn live(&self, outside: &mut impl super::effects::Outside) -> bool {
self.outcome.is_none()
&& !self.process_start.is_empty()
&& outside.process_start(self.process_id).as_deref() == Some(&self.process_start)
}
pub fn new(work: Work<'_>, stamp: &str, harness: &Choice, now: DateTime<FixedOffset>) -> Self {
let issue = work.issue_url().map(|issue| Issue {
number: issue.number,
url: issue.url.clone(),
title: match work {
Work::PickupRun(listed) => Some(listed.title.clone()),
Work::Starting { .. }
| Work::Run(_)
| Work::BaseFix(_)
| Work::SpecRun(_)
| Work::ArchitectRun(_)
| Work::SecurityRun(_)
| Work::ScoringRun(_)
| Work::ReviewRun(_)
| Work::FixRun(_)
| Work::ReleaseRun(_)
| Work::TriageRun { .. }
| Work::Experiment(_)
| Work::Process { .. } => None,
},
});
let name = match work {
Work::Starting { command, issue, .. } => match issue {
Some(issue) => format!("{}#{}", issue.repo_slug(), issue.number),
None => format!("{}-{stamp}", command.folder()),
},
Work::ArchitectRun(_) => format!("architect-{stamp}"),
Work::SecurityRun(_) => format!("secure-{stamp}"),
Work::ScoringRun(_) => format!("scoring-{stamp}"),
Work::ReviewRun(_) => format!("review-{stamp}"),
Work::FixRun(_) => format!("fix-{stamp}"),
Work::ReleaseRun(_) => format!("release-{stamp}"),
Work::TriageRun { .. } => format!("triage-{stamp}"),
Work::PickupRun(_) => format!("pickup-{stamp}"),
Work::Experiment(_) => format!("experiment-{stamp}"),
Work::Process { name, .. } => format!("process-{name}-{stamp}"),
Work::Run(issue) | Work::BaseFix(issue) | Work::SpecRun(issue) => {
format!("{}#{}", issue.repo_slug(), issue.number)
}
};
Self {
version: 2,
kind: if matches!(work, Work::BaseFix(_)) {
"Base fix".into()
} else {
work.kind().to_string()
},
repository: work.repo().slug(),
name,
issue,
hostname: String::new(),
process_id: std::process::id(),
thirdshift_version: Some(env!("CARGO_PKG_VERSION").into()),
version_recovered: false,
started_by: super::Actor::starting(),
process_start: String::new(),
harness: harness.harness.name().into(),
model: harness.model.clone(),
effort: harness.effort.clone(),
started_at: now.to_rfc3339(),
ended_at: None,
end_time_estimated: false,
stages: Vec::new(),
quota_wait_until: None,
session_log: None,
outcome: None,
run_metrics: None,
cause: None,
stopped_at: None,
failure_category: None,
waiting_for: Vec::new(),
limit: None,
resume_command: None,
handed_back_for_pickup: false,
issue_branch: None,
base_branch: None,
pull_request: None,
pull_request_title: None,
parent_record: None,
parent_dispatch: None,
tickets: Vec::new(),
security_findings_waiting: false,
security_findings: Vec::new(),
security_fix_findings: Vec::new(),
experiment: None,
process: work.process(),
}
}
pub fn stopped_stage(&self) -> Option<&str> {
let names = || self.stages.iter().rev().map(|stage| stage.name.as_str());
names()
.find(|name| *name != StageName::Cleanup.name())
.or_else(|| names().next())
}
pub fn current_stage(&self) -> Option<&Stage> {
self.stages
.iter()
.rev()
.find(|stage| StageName::known(&stage.name))
}
}