use std::path::PathBuf;
use crate::harness::Harness;
use crate::harness::interpretation::SessionFailure;
const STOPPED: &str = "Stopped at: ";
const PROGRESS: &str = "Progress: ";
const PRESERVED: &str = "Preserved: ";
const BEFORE_RETRYING: &str = "Before retrying: ";
const RECOVERY: &str = "Recovery: ";
const SECURITY_REVIEW: &str = "Security review will use ";
const LAUNCH: &str = "Run from the Launch directory ";
const CONTINUING: &str = "Continuing reuses ";
const RETRY: &str = "Retry starts work ";
const RETAINED_WORK: &str = "Retained work: ";
const LOCAL_WORK: &str = "work may exist only locally on ";
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Stage {
Implementation,
Tickets,
SpecReview,
SecurityReview,
Push,
PullRequest,
Repair,
Ci,
SelfMerge,
BaseFix,
}
impl Stage {
pub fn recorded(self) -> Option<crate::logs::Stage> {
use crate::logs::Stage as Recorded;
match self {
Self::Implementation => Some(Recorded::Implement),
Self::Tickets => Some(Recorded::Tickets),
Self::SpecReview => Some(Recorded::SpecReview),
Self::SecurityReview => Some(Recorded::SecurityReview),
Self::Push | Self::PullRequest => Some(Recorded::PullRequest),
Self::Repair => Some(Recorded::Repair),
Self::Ci => Some(Recorded::CiWatch),
Self::SelfMerge => Some(Recorded::SelfMerge),
Self::BaseFix => None,
}
}
fn name(self) -> &'static str {
match self {
Self::Implementation => "Implementation",
Self::Tickets => "Ticket execution",
Self::SpecReview => "Spec review",
Self::SecurityReview => "Security review",
Self::Push => "Push",
Self::PullRequest => "Pull request readiness",
Self::Repair => "Repair",
Self::Ci => "CI watching",
Self::SelfMerge => "Self-merge",
Self::BaseFix => "Base fix",
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Preservation {
Remote,
Local,
None,
}
pub fn preservation_of(
result: Result<crate::worktree::Preserved, anyhow::Error>,
report: impl FnOnce(String),
) -> Preservation {
match result {
Ok(crate::worktree::Preserved::Pushed) => Preservation::Remote,
Ok(crate::worktree::Preserved::NoChanges) => Preservation::None,
Err(problem) => {
report(format!(
"could not push the failed ticket run's work, so it may exist only locally: {problem:#}"
));
Preservation::Local
}
}
}
#[derive(Debug, PartialEq, Eq)]
pub struct Continuation {
pub command: String,
pub launch: PathBuf,
pub base: String,
pub branch: String,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct TicketProgress {
pub landed: usize,
pub total: usize,
}
#[derive(Debug, PartialEq, Eq)]
pub struct Recovery {
stage: Stage,
preservation: Preservation,
worktree: Option<PathBuf>,
tickets: Option<(TicketProgress, bool)>,
continuation: Option<Continuation>,
security_review_harness: Option<Harness>,
local_child_work: Vec<String>,
spec_pr_changed: bool,
fix_run: bool,
}
impl Recovery {
pub fn new(stage: Stage, preservation: Preservation) -> Self {
Self {
stage,
preservation,
worktree: None,
tickets: None,
continuation: None,
security_review_harness: None,
local_child_work: Vec::new(),
spec_pr_changed: false,
fix_run: false,
}
}
pub fn stage(&self) -> Stage {
self.stage
}
pub fn record_worktree(&mut self, path: PathBuf) {
self.worktree = Some(path);
}
pub fn add_tickets(&mut self, progress: TicketProgress, all_accounted: bool) {
let reviewed = !matches!(self.stage, Stage::Tickets | Stage::SpecReview) && all_accounted;
self.tickets = Some((progress, reviewed));
}
pub fn add_command_context(&mut self, context: Continuation, on_origin: bool) {
if self.preservation == Preservation::None && on_origin {
self.preservation = Preservation::Remote;
}
self.continuation = Some(context);
}
pub(crate) fn for_fix_run(&mut self) {
self.fix_run = true;
}
pub fn add_child_work(&mut self, work: impl IntoIterator<Item = String>) {
self.local_child_work.extend(work);
}
pub fn add_spec_pr_changed(&mut self) {
self.spec_pr_changed = true;
}
pub fn add_security_review_harness(&mut self, harness: Harness) {
self.security_review_harness = Some(harness);
}
pub fn lines(&self) -> Vec<String> {
let mut lines = vec![format!("{STOPPED}{}", self.stage.name())];
if let Some((TicketProgress { landed, total }, reviewed)) = self.tickets {
let review = if reviewed { "complete" } else { "incomplete" };
lines.push(format!(
"{PROGRESS}{landed}/{total} Tickets landed; final review {review}"
));
}
let Some(context) = &self.continuation else {
return lines;
};
lines.push(match self.preservation {
Preservation::Remote => format!("{PRESERVED}remotely on {}", context.branch),
Preservation::Local => format!(
"{PRESERVED}{LOCAL_WORK}{} at {}; push was not confirmed",
context.branch,
self.worktree
.as_deref()
.unwrap_or(&context.launch)
.display()
),
Preservation::None => format!("{PRESERVED}no work preserved"),
});
lines.extend(
self.local_child_work
.iter()
.map(|line| format!("{RETAINED_WORK}{line}")),
);
if self.preservation == Preservation::Local || !self.local_child_work.is_empty() {
lines.push(format!("{BEFORE_RETRYING}inspect and publish the retained work, reconcile with origin without forcing, then free the Issue branch and Worktree acquisition path. Do not discard local work."));
}
if self.spec_pr_changed {
lines.push(format!("{BEFORE_RETRYING}the Spec PR changed under the Spec run, so no Recovery command resumes {} into {}. Decide the Spec's Base branch and Spec branch before starting it again.", context.branch, context.base));
return lines;
}
lines.push(format!("{RECOVERY}{}", context.command));
if self.fix_run {
lines.push(format!("{LAUNCH}{} or another clone passing Origin match. The Fix run uses the pull request's Base branch {} and continues its Repair loop.", context.launch.display(), context.base));
return lines;
}
if let Some(harness) = self.security_review_harness {
lines.push(format!(
"{SECURITY_REVIEW}{}, chosen by security.harness in the User config.",
harness.name()
));
}
lines.push(format!(
"{LAUNCH}{} with Base branch {} checked out (the command cannot encode the Base branch). Retry when the failure's cause is resolved.",
context.launch.display(), context.base
));
if self.preservation != Preservation::None {
lines.push(format!("{CONTINUING}the preserved branch and PR and skips completed Tickets. This starts a new Continuation, not the interrupted agent conversation; review and merge/CI repairs may still be needed."));
} else {
lines.push(format!(
"{RETRY}on the original Issue URL; there is no preserved work to continue."
));
}
lines
}
pub fn local_work(line: &str) -> Option<&str> {
let line = line.strip_prefix(RETAINED_WORK).unwrap_or(line);
line.strip_prefix(PRESERVED)
.is_some_and(|value| value.starts_with(LOCAL_WORK))
.then_some(line)
}
pub fn comment(&self, cause: &str, pr_url: Option<&str>, machine: Option<&str>) -> String {
let mut facts = vec![format!("{STOPPED}{}", self.stage.name())];
if let Some((TicketProgress { landed, total }, reviewed)) = self.tickets {
let review = if reviewed { "complete" } else { "incomplete" };
facts.push(format!(
"{PROGRESS}{landed}/{total} Tickets landed; final review {review}"
));
}
let branch = self.continuation.as_ref().map(|context| &context.branch);
let on = machine.unwrap_or("the machine that ran it");
let mut preserved = match (self.preservation, branch) {
(Preservation::Remote, Some(branch)) => format!("Issue branch `{branch}` on origin"),
(Preservation::Local, Some(branch)) => format!(
"Issue branch `{branch}` may exist only locally on {on}; its push was not confirmed"
),
(Preservation::None, Some(_)) | (_, None) => "no work".to_string(),
};
if let Some(pr_url) = pr_url {
preserved += &format!(", with pull request {pr_url}");
}
if !self.local_child_work.is_empty() {
preserved += &format!("; some child work may exist only locally on {on}");
}
facts.push(format!("{PRESERVED}{preserved}"));
let retry = if self.spec_pr_changed {
"The Spec PR changed under the Spec run: decide the Spec's Base branch and Spec branch \
before putting `ready-for-agent` back to retry."
} else {
RETRY_BY_LABEL
};
render(cause, facts, machine, retry)
}
pub fn is_line(line: &str) -> bool {
[
STOPPED,
PROGRESS,
PRESERVED,
BEFORE_RETRYING,
RECOVERY,
SECURITY_REVIEW,
LAUNCH,
CONTINUING,
RETRY,
RETAINED_WORK,
]
.iter()
.any(|prefix| line.starts_with(prefix))
}
}
const RETRY_BY_LABEL: &str = "Put `ready-for-agent` back to retry.";
const BEFORE_WORK: &str = "Start, before any work began";
pub fn comment(
recovery: Option<&Recovery>,
branch: Option<&str>,
cause: &str,
pr_url: Option<&str>,
machine: Option<&str>,
) -> String {
if let Some(recovery) = recovery {
return recovery.comment(cause, pr_url, machine);
}
let preserved = match (branch, pr_url) {
(Some(branch), Some(pr_url)) => {
format!("Issue branch `{branch}`, with pull request {pr_url}")
}
(Some(branch), None) => format!("Issue branch `{branch}`, if it reached origin"),
(None, Some(pr_url)) => format!("pull request {pr_url}"),
(None, None) => "no work".to_string(),
};
let facts = vec![
format!("{STOPPED}{BEFORE_WORK}"),
format!("{PRESERVED}{preserved}"),
];
render(cause, facts, machine, RETRY_BY_LABEL)
}
pub fn cause(error: &anyhow::Error) -> String {
error
.chain()
.map(|layer| {
if let Some(failure) = layer.downcast_ref::<SessionFailure>() {
failure.without_diagnostic()
} else if let Some(not_done) = layer.downcast_ref::<crate::spec_run::TicketsNotDone>() {
not_done.in_public()
} else {
layer.to_string()
}
})
.collect::<Vec<_>>()
.join(": ")
}
fn render(cause: &str, facts: Vec<String>, machine: Option<&str>, retry: &str) -> String {
let cause = without_local_paths(&cause.split_whitespace().collect::<Vec<_>>().join(" "));
let mut text = format!(
"{}\n\nthirdshift's Run on this issue failed, so it waits for the Day shift.\n\n\
- Cause: {cause}\n",
crate::github::GENERATED
);
for fact in facts {
text += &format!("- {fact}\n");
}
text += &format!("- Machine: {}\n\n{retry}\n", machine.unwrap_or("unknown"));
text
}
fn without_local_paths(text: &str) -> String {
text.split(' ')
.map(|word| {
let Some(start) = local_path_start(word) else {
return word.to_string();
};
let rest = &word[start..];
let path = rest.trim_end_matches([')', ']', '\'', '"', '`', ',', '.', ';', ':']);
format!("{}<local path>{}", &word[..start], &rest[path.len()..])
})
.collect::<Vec<_>>()
.join(" ")
}
fn local_path_start(word: &str) -> Option<usize> {
word.char_indices().find_map(|(at, _)| {
let rest = &word[at..];
if !(rest.starts_with('/') || rest.starts_with("~/")) {
return None;
}
let before = word[..at].chars().next_back();
let opens = match before {
None => true,
Some('(' | '[' | '\'' | '"' | '`' | '=') => true,
Some(':') => !rest.starts_with("//"),
Some(_) => false,
};
opens.then_some(at)
})
}
#[cfg(test)]
mod tests {
use super::*;
fn context() -> Continuation {
Continuation {
command: "thirdshift https://github.com/acme/widgets/issues/21".into(),
launch: "/launch".into(),
base: "main".into(),
branch: "issue-21".into(),
}
}
fn preserved(recovery: &Recovery) -> String {
recovery
.lines()
.into_iter()
.find(|line| line.starts_with(PRESERVED))
.expect("no Preserved line")
}
fn has(recovery: &Recovery, prefix: &str) -> bool {
recovery.lines().iter().any(|line| line.starts_with(prefix))
}
#[test]
fn a_continuation_with_nothing_pushed_reads_preserved_remotely() {
let mut recovery = Recovery::new(Stage::Implementation, Preservation::None);
recovery.add_command_context(context(), true);
assert_eq!(preserved(&recovery), "Preserved: remotely on issue-21");
assert!(has(&recovery, CONTINUING));
assert!(!has(&recovery, RETRY));
}
#[test]
fn a_spec_pr_changed_under_the_spec_run_leaves_out_every_command_line() {
let mut recovery = Recovery::new(Stage::Tickets, Preservation::Remote);
recovery.add_command_context(context(), true);
recovery.add_spec_pr_changed();
assert_eq!(preserved(&recovery), "Preserved: remotely on issue-21");
assert!(has(&recovery, BEFORE_RETRYING));
for command in [RECOVERY, LAUNCH, CONTINUING, RETRY] {
assert!(!has(&recovery, command), "{command}");
}
}
#[test]
fn a_fresh_run_with_nothing_pushed_reads_no_work_preserved() {
let mut recovery = Recovery::new(Stage::Implementation, Preservation::None);
recovery.add_command_context(context(), false);
assert_eq!(preserved(&recovery), "Preserved: no work preserved");
assert!(has(&recovery, RETRY));
assert!(!has(&recovery, CONTINUING));
}
#[test]
fn the_final_review_is_complete_only_past_the_spec_review_with_every_ticket_accounted() {
let progress = TicketProgress {
landed: 3,
total: 3,
};
for (stage, all_accounted, review) in [
(Stage::Tickets, true, "incomplete"),
(Stage::SpecReview, true, "incomplete"),
(Stage::Push, false, "incomplete"),
(Stage::Push, true, "complete"),
(Stage::Ci, true, "complete"),
] {
let mut recovery = Recovery::new(stage, Preservation::Remote);
recovery.add_tickets(progress, all_accounted);
assert_eq!(
recovery.lines()[1],
format!("Progress: 3/3 Tickets landed; final review {review}"),
"{stage:?}, all accounted: {all_accounted}"
);
}
}
#[test]
fn retained_work_from_two_layers_is_kept_in_order() {
let mut recovery = Recovery::new(Stage::Ci, Preservation::Remote);
recovery.add_command_context(context(), false);
recovery.add_child_work(["Ticket #22 kept at /kept/22".to_string()]);
recovery.add_child_work(["Base fix #30 kept at /kept/30".to_string()]);
let lines = recovery.lines();
let retained: Vec<_> = lines
.iter()
.filter(|line| line.starts_with(RETAINED_WORK))
.collect();
assert_eq!(
retained,
[
"Retained work: Ticket #22 kept at /kept/22",
"Retained work: Base fix #30 kept at /kept/30"
]
);
assert!(has(&recovery, BEFORE_RETRYING));
}
#[test]
fn local_preservation_names_the_recorded_worktree_or_else_the_launch_directory() {
let mut recovery = Recovery::new(Stage::Push, Preservation::Local);
recovery.add_command_context(context(), false);
assert_eq!(
preserved(&recovery),
"Preserved: work may exist only locally on issue-21 at /launch; push was not confirmed"
);
recovery.record_worktree("/worktrees/issue-21".into());
assert_eq!(
preserved(&recovery),
"Preserved: work may exist only locally on issue-21 at /worktrees/issue-21; push was not confirmed"
);
assert!(has(&recovery, BEFORE_RETRYING));
}
#[test]
fn the_comment_names_the_cause_stop_preserved_branch_and_pull_request_machine_and_retry() {
let mut recovery = Recovery::new(Stage::Ci, Preservation::Remote);
recovery.add_command_context(context(), false);
assert_eq!(
recovery.comment(
"CI failed",
Some("https://github.com/acme/widgets/pull/3"),
Some("night-host")
),
"This comment was generated by AI.\n\n\
thirdshift's Run on this issue failed, so it waits for the Day shift.\n\n\
- Cause: CI failed\n\
- Stopped at: CI watching\n\
- Preserved: Issue branch `issue-21` on origin, with pull request \
https://github.com/acme/widgets/pull/3\n\
- Machine: night-host\n\n\
Put `ready-for-agent` back to retry.\n"
);
}
#[test]
fn the_comment_leaves_out_local_paths_and_the_continue_command() {
let mut recovery = Recovery::new(Stage::Push, Preservation::Local);
recovery.add_command_context(context(), false);
recovery.record_worktree("/worktrees/issue-21".into());
recovery.add_child_work(["Ticket #22 kept at /kept/22".to_string()]);
let comment = recovery.comment(
"could not push (/worktrees/issue-21): see ~/.thirdshift/logs/a.jsonl,\nthen retry",
None,
Some("night-host"),
);
for local in [
"/worktrees",
"/kept",
"~/",
"/launch",
"thirdshift https://",
] {
assert!(!comment.contains(local), "{local}: {comment}");
}
assert!(
comment
.contains("- Cause: could not push (<local path>): see <local path>, then retry\n"),
"{comment}"
);
assert!(
comment.contains(
"- Preserved: Issue branch `issue-21` may exist only locally on night-host; its \
push was not confirmed; some child work may exist only locally on night-host\n"
),
"{comment}"
);
}
#[test]
fn a_comment_without_a_recovery_names_the_branch_and_pull_request_it_has() {
let comment = comment(
None,
Some("issue-21"),
"could not create the worktree",
Some("https://github.com/acme/widgets/pull/3"),
None,
);
assert!(
comment.contains(
"- Preserved: Issue branch `issue-21`, with pull request \
https://github.com/acme/widgets/pull/3\n- Machine: unknown\n"
),
"{comment}"
);
assert!(
comment.contains("- Stopped at: Start, before any work began\n"),
"{comment}"
);
}
#[test]
fn a_comment_without_a_recovery_still_says_where_the_run_stopped() {
let comment = comment(
None,
Some("issue-21"),
"could not create the worktree",
None,
None,
);
assert!(comment.contains("- Stopped at: "), "{comment}");
}
#[test]
fn a_comment_after_the_spec_pr_changed_says_to_decide_before_retrying() {
let mut recovery = Recovery::new(Stage::Tickets, Preservation::Remote);
recovery.add_command_context(context(), false);
recovery.add_spec_pr_changed();
let comment = recovery.comment("Spec PR closed", None, Some("night-host"));
assert!(comment.contains("Spec PR changed"), "{comment}");
}
#[test]
fn the_comment_scrubs_a_path_joined_by_an_equals_sign() {
let comment = comment(
None,
None,
"git --git-dir=/home/u/repo/.git failed",
None,
None,
);
assert!(!comment.contains("/home/u"), "{comment}");
}
#[test]
fn the_comment_scrubs_a_path_after_a_colon_but_keeps_urls() {
let comment = comment(
None,
None,
"path:/home/u/x, see https://github.com/acme/widgets/pull/3",
None,
None,
);
assert!(
comment.contains(
"- Cause: path:<local path>, see https://github.com/acme/widgets/pull/3\n"
),
"{comment}"
);
}
}