use std::fmt::Write;
use std::path::Path;
use std::sync::Arc;
use std::time::Duration;
use tracing::{debug, error, info, warn};
use chrono::Duration as ChronoDuration;
use futures_util::FutureExt;
use futures_util::future::join_all;
use crate::agent::{RETRY_EXHAUSTION_MARKER, run_agent};
use crate::board::{BOARD, BoardStore, PipelineCheck, Ticket, TicketComment, TicketPhase};
use crate::git_commands::{
has_unstaged_changes, list_new_or_untracked_files, parse_new_files_from_porcelain,
run_git_add_all, run_git_diff_stats, run_git_head, run_git_status, run_git_write_tree,
};
use crate::jobs::ResumableStage;
use crate::message_router;
use crate::prompt::{load_prompt, load_prompt_sections, substitute};
use crate::role::{DIAGNOSTICS_ROLE, SANITATION_ROLE, SYSTEM_ROLE};
use crate::session::{manager_agent_id, ticket_agent_id};
use crate::ticket_buffer;
use crate::tools::shell::{ShellMode, ShellTool};
use crate::turso::TxGuard;
use crate::util::panic_message;
use crate::workspace::spawn_workspace_discovery;
use crate::{Agent, DiagnosticsCommands, Role, Workspace, WorkspaceStatus};
pub(crate) const DEFAULT_PARALLEL_AGENT_COUNT: usize = 3;
const QA_PARALLEL_AGENT_COUNT: usize = 1;
pub(crate) const ANALYST_PASS_THRESHOLD: u8 = 7;
const REVIEW_QA_THRESHOLD: u8 = 9;
const PHASE_GATE_BAIL_REASON: &str = "ticket not in expected phase";
#[inline]
fn board() -> &'static BoardStore {
crate::board::store()
}
async fn clear_assigned_to_no_cancel(ticket_id: &str, context: &str) {
if let Err(e) = board().set_assigned_to_no_cancel(ticket_id, None).await {
warn!(
ticket = %ticket_id,
error = %e,
"Failed to clear assigned_to: {context}",
);
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum CircuitBreakerKind {
Sanitation,
Diagnostics,
}
impl CircuitBreakerKind {
const fn max_count(self) -> usize {
match self {
Self::Sanitation => 3,
Self::Diagnostics => 4,
}
}
fn should_trip(self, comments: &[TicketComment]) -> Option<(usize, usize)> {
let max_count = self.max_count();
let count = match self {
Self::Sanitation => count_matching_comments(
comments,
SANITATION_ROLE,
&load_prompt("pipeline/sanitation_failed.md"),
),
Self::Diagnostics => count_matching_comments(
comments,
DIAGNOSTICS_ROLE,
&load_prompt("pipeline/diagnostics_failed.md"),
),
};
if count <= max_count {
None
} else {
Some((count, max_count))
}
}
fn trip_message(self, count: usize, max_count: usize) -> String {
match self {
Self::Sanitation => format!(
"❌ Sanitation circuit breaker tripped after {count} cumulative failures. \
(max: {max_count})",
),
Self::Diagnostics => {
format!("❌ Circuit breaker: {count} prior diagnostic failures. Failing ticket.")
}
}
}
}
fn count_matching_comments(comments: &[TicketComment], role: &str, marker: &str) -> usize {
comments
.iter()
.filter(|c| c.role == role && c.content.contains(marker))
.count()
}
#[must_use]
async fn is_ticket_in_phase(ticket_id: &str, expected_phase: TicketPhase) -> bool {
match board().get_ticket_phase(ticket_id).await {
Ok(Some(phase)) => {
let ok = phase == expected_phase;
if !ok {
debug!(
ticket = %ticket_id,
expected_phase = %expected_phase,
actual = %phase,
"Ticket moved externally — bailing out",
);
}
ok
}
Ok(None) => {
debug!(ticket = %ticket_id, "Ticket not found — row missing (violates architecture invariant)");
false
}
Err(e) => {
warn!(ticket = %ticket_id, error = %e, "Failed to check ticket phase");
false
}
}
}
#[must_use]
async fn complete_job_and_bail_if_phase_moved(
ticket_id: &str,
expected: TicketPhase,
job_id: &str,
) -> bool {
if !is_ticket_in_phase(ticket_id, expected).await {
complete_ticket_stage_job(job_id).await;
return true;
}
false
}
#[must_use]
async fn phase_changed_and_clear_assignment(ticket_id: &str, expected: TicketPhase) -> bool {
if !is_ticket_in_phase(ticket_id, expected).await {
let label = format!("ticket left {expected:?}");
clear_assigned_to_no_cancel(ticket_id, &label).await;
return true;
}
false
}
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
enum NotifyPolicy {
Notify,
Buffer,
}
#[derive(Debug)]
struct TransitionCtx<'a> {
ticket: &'a Ticket,
source: TicketPhase,
target: TicketPhase,
notify: NotifyPolicy,
log_label: &'a str,
breaker_trip: bool,
}
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
enum FinalizeOutcome {
Applied,
Moved,
Failed,
}
#[must_use]
async fn with_comment_and_transition<F>(
ctx: TransitionCtx<'_>,
write_comments: F,
) -> FinalizeOutcome
where
F: AsyncFnOnce(&TxGuard<'_>) -> anyhow::Result<()>,
{
let pipeline_reservation = (ctx.target == TicketPhase::ReadyForDevelopment).then_some(true);
let outcome = match crate::turso::with_tx_outcome(
&board().conn,
&ctx.ticket.id,
ctx.log_label,
async move |tx| {
write_comments(tx).await?;
BoardStore::transition_to_tx(
tx,
&ctx.ticket.id,
Some(ctx.source),
ctx.target,
pipeline_reservation,
)
.await
},
)
.await
{
Ok(true) => FinalizeOutcome::Applied,
Ok(false) => {
debug!(
ticket = %ctx.ticket.id,
"{}: ticket moved externally while in {} — finalization skipped (nothing written)",
ctx.log_label, ctx.source,
);
FinalizeOutcome::Moved
}
Err(e) => {
warn!(
ticket = %ctx.ticket.id,
error = %e,
"{}: transition to {} failed — ticket stuck in {}",
ctx.log_label, ctx.target, ctx.source,
);
clear_assigned_to_no_cancel(&ctx.ticket.id, ctx.log_label).await;
FinalizeOutcome::Failed
}
};
if matches!(outcome, FinalizeOutcome::Applied) {
match ctx.notify {
NotifyPolicy::Notify => {
notify_ticket(ctx.ticket, ctx.source, ctx.target, ctx.breaker_trip).await;
}
NotifyPolicy::Buffer => {
ticket_buffer::push(
&ctx.ticket.workspace_name,
&ctx.ticket.id,
ctx.source,
ctx.target,
ticket_buffer::TransitionOrigin::Pipeline,
);
}
}
}
outcome
}
#[must_use]
async fn comment_and_transition(ctx: TransitionCtx<'_>, role: &str, text: &str) -> FinalizeOutcome {
let ticket = ctx.ticket;
with_comment_and_transition(ctx, async |tx| {
BoardStore::add_comment_tx(tx, &ticket.id, role, text).await?;
Ok(())
})
.await
}
async fn comment_and_transition_or_bail(
ctx: TransitionCtx<'_>,
role: &str,
text: &str,
message: &str,
) {
let ticket_id = &ctx.ticket.id;
let target = ctx.target;
if !matches!(
comment_and_transition(ctx, role, text).await,
FinalizeOutcome::Applied
) {
return;
}
info!(ticket = %ticket_id, target = %target, "{message}");
}
#[must_use]
async fn resolve_ticket_workspace(ticket: &Ticket, log_label: &str) -> Option<crate::Workspace> {
match crate::workspace::get_by_name(&ticket.workspace_name).await {
Ok(Some(ws)) => Some(ws),
Ok(None) => {
warn!(
ticket = %ticket.id,
workspace_name = %ticket.workspace_name,
"Workspace not found for ticket — {log_label}",
);
None
}
Err(e) => {
warn!(
ticket = %ticket.id,
workspace_name = %ticket.workspace_name,
error = %e,
"Failed to look up workspace for ticket — {log_label}",
);
None
}
}
}
fn paused_workspace_sentence() -> &'static str {
"new analysis and development claims are blocked until the workspace is resumed"
}
pub(crate) async fn pause_workspace_on_failure(ticket: &Ticket, reason: &str) -> String {
if crate::shutdown::aborting() {
return String::new();
}
let Some(ws) = resolve_ticket_workspace(ticket, "auto-pause skipped").await else {
return String::new();
};
if ws.paused {
return String::new();
}
match crate::workspace::store().set_paused(&ws.name, true).await {
Ok(()) => {
info!(
ticket = %ticket.id,
workspace = %ws.name,
reason,
"Workspace auto-paused after failure"
);
format!(
"\n\n⚠️ Workspace paused: {reason} — {}.",
paused_workspace_sentence()
)
}
Err(e) => {
warn!(
ticket = %ticket.id,
workspace = %ws.name,
reason,
error = %e,
"Failed to auto-pause workspace after technical failure",
);
String::new()
}
}
}
async fn last_comment_as_failure_details(ticket_id: &str) -> String {
match board().get_comments(ticket_id).await {
Ok(comments) => comments.last().map_or_else(
|| "(unknown failure reason)".to_string(),
|c| c.content.clone(),
),
Err(_) => "(unknown failure reason)".to_string(),
}
}
async fn notify_ticket(
ticket: &Ticket,
source: TicketPhase,
target_phase: TicketPhase,
breaker_trip: bool,
) {
let Some(ws) = resolve_ticket_workspace(ticket, "skipping notification").await else {
error!(
ticket = %ticket.id,
workspace_name = %ticket.workspace_name,
"Workspace resolution failed — notification skipped"
);
return;
};
let transition_log = format!(
"[{}] {}: {} → {} ({})",
ticket.reporter,
ticket.id,
source.as_ref(),
target_phase.as_ref(),
ticket_buffer::TransitionOrigin::Pipeline
);
let drained = crate::ticket_buffer::drain(&ws.name);
let mut message = substitute(
&load_prompt("pipeline/notification.md"),
&[
("{{ticket_id}}", &ticket.id),
("{{ticket_title}}", &ticket.title),
("{{ticket_phase}}", target_phase.as_ref()),
("{{transition_log}}", &transition_log),
("{{ticket_updates}}", &drained),
],
);
let engineer_bounce =
source == TicketPhase::InDevelopment && target_phase == TicketPhase::ReadyForDevelopment;
if target_phase == TicketPhase::Failed || engineer_bounce {
let failure_details = last_comment_as_failure_details(&ticket.id).await;
let workspace_status = if breaker_trip {
"Beware that all the other tickets have been moved back from Ready for Dev \
to Planning."
.to_string()
} else if ws.paused {
format!("The workspace is paused — {}.", paused_workspace_sentence())
} else {
"The workspace was not paused — remaining queued tickets may still be claimed."
.to_string()
};
let template = if engineer_bounce {
"pipeline/engineer_bounce_notification.md"
} else {
"pipeline/failure_notification.md"
};
let warning = substitute(
&load_prompt(template),
&[
("{{failure_details}}", &failure_details),
("{{workspace_status}}", &workspace_status),
],
);
message.push_str("\n\n");
message.push_str(&warning);
}
let agent_id = manager_agent_id(&ws.name);
message_router::route(
&agent_id,
message_router::AgentJob {
content: message,
workspace_name: ws.name,
user_name: String::new(),
channel: String::new(),
kind: message_router::JobKind::TicketNotify,
role: crate::Role::Manager,
reply_target: None,
pending_job_id: None,
},
);
}
#[expect(clippy::too_many_lines)]
pub async fn run_management() {
let resumable = match crate::jobs::recover_from_restart().await {
Ok(r) => r,
Err(e) => {
error!(error = %e, "Boot recovery scan failed — proceeding with plain reset");
if let Some(board) = BOARD.get() {
let _ = board.reset_inflight_tickets(&[]).await;
}
Vec::new()
}
};
if let Err(e) = crate::workspace::store()
.reclassify_analyzing_to_pending()
.await
{
warn!(error = %e, "Boot recovery: failed to reclassify stranded analyzing workspaces");
}
for stage in resumable {
let (job_id, workspace_name) = match &stage {
ResumableStage::TicketStage {
job_id,
workspace_name,
..
}
| ResumableStage::Research {
job_id,
workspace_name,
..
}
| ResumableStage::Analyze {
job_id,
workspace_name,
..
}
| ResumableStage::ResearchCleanup {
job_id,
workspace_name,
} => (job_id, workspace_name),
};
let Ok(Some(workspace)) = crate::workspace::store().get_by_name(workspace_name).await
else {
warn!(
job = %job_id,
workspace = %workspace_name,
"Resume workspace unresolvable — deleting job row",
);
if matches!(&stage, ResumableStage::ResearchCleanup { .. }) {
crate::research_cleanup::release_run_folder(job_id).await;
}
let _ = crate::jobs::terminalize_job(&crate::session::store().conn, job_id).await;
continue;
};
match stage {
ResumableStage::Research {
job_id,
capped: false,
..
} => {
info!(job = %job_id, "Resuming research run at boot");
let ws = workspace.clone();
tokio::spawn(async move {
crate::tools::research::resume_research_run(&job_id, &ws).await;
});
}
ResumableStage::Research {
job_id,
capped: true,
..
} => {
info!(
job = %job_id,
"Delivering research partial report (boot re-dispatch cap exceeded)",
);
let ws = workspace.clone();
tokio::spawn(async move {
crate::tools::research::research_capped_partial_report(&job_id, &ws).await;
});
}
ResumableStage::Analyze {
job_id,
capped: false,
..
} => {
info!(job = %job_id, "Resuming analyze round at boot");
let ws = workspace.clone();
tokio::spawn(async move {
crate::tools::analyze::resume_analyze_round(&job_id, &ws).await;
});
}
ResumableStage::Analyze {
job_id,
capped: true,
..
} => {
info!(
job = %job_id,
"Delivering analyze failure envelope (boot re-dispatch cap exceeded)",
);
let ws = workspace.clone();
tokio::spawn(async move {
crate::tools::analyze::analyze_capped_envelope(&job_id, &ws).await;
});
}
ResumableStage::ResearchCleanup { job_id, .. } => {
info!(job = %job_id, "Resuming research cleanup agent at boot");
let ws = workspace.clone();
tokio::spawn(async move {
crate::research_cleanup::resume_research_cleanup(&job_id, &ws).await;
});
}
ResumableStage::TicketStage {
job_id,
ticket_id,
stage,
..
} => {
if let Ok(Some(ticket)) = crate::board::store().get_ticket(&ticket_id).await {
info!(
job = %job_id,
ticket = %ticket_id,
stage = %stage,
"Resuming ticket stage round at boot",
);
tokio::spawn(resume_ticket_stage_round(stage, job_id, ticket, workspace));
} else {
warn!(
job = %job_id,
ticket = %ticket_id,
"Resume ticket not found — deleting job row",
);
let _ = crate::jobs::complete_ticket_stage_job(
&crate::session::store().conn,
&job_id,
)
.await;
}
}
}
}
let interval = Duration::from_secs(1);
loop {
if !crate::shutdown::sleep_or_shutdown_or_drain(interval).await {
break;
}
poll_round().await;
}
}
fn spawn_dispatch(phase: PollPhase, ticket: Ticket, ws: Workspace) {
let phase_info = phase.info();
let expected_phase = phase_info.expected_phase;
let kind = phase_info.circuit_breaker_kind;
let log_label = phase_info.log_label;
info!(
ticket = %ticket.id,
title = %ticket.title,
workspace = %ws.name,
"Dispatching {} ticket",
phase_info.log_label,
);
crate::registry::AGENT_REGISTRY.cancel_by_ticket_id(&ticket.id);
let ticket = Arc::new(ticket);
let ticket_for_failure = Arc::clone(&ticket);
tokio::spawn(async move {
if !is_ticket_in_phase(&ticket.id, expected_phase).await {
return;
}
if try_trip_circuit_breaker(&ticket, expected_phase, kind, log_label).await {
drain_ready_for_development_siblings(&ticket).await;
return;
}
let result = std::panic::AssertUnwindSafe(async move {
match phase {
PollPhase::BacklogAnalysis => dispatch_backlog_analysts(ticket, ws).await,
PollPhase::EngineerDevelopment => dispatch_engineer(ticket, ws).await,
PollPhase::SanitationCheck => dispatch_sanitation(ticket, ws).await,
PollPhase::DiagnosticsCheck => dispatch_diagnostics(ticket, ws).await,
PollPhase::VerifierCheck(vi) => dispatch_verifiers(ticket, ws, vi).await,
}
})
.catch_unwind()
.await;
if let Err(payload) = result {
let msg = crate::util::scrub_credentials(&panic_message(&*payload));
error!(
ticket = %ticket_for_failure.id,
panic = %msg,
"Dispatch panicked — transitioning ticket to Failed",
);
if crate::shutdown::aborting() {
warn!(
ticket = %ticket_for_failure.id,
"Dispatch panic during drain — leaving ticket for boot resume"
);
return;
}
let pause_note =
pause_workspace_on_failure(&ticket_for_failure, "dispatch panic").await;
let panic_comment = format!("❌ Dispatch panicked: {msg}{pause_note}");
let _ = comment_and_transition(
TransitionCtx {
ticket: &ticket_for_failure,
source: expected_phase,
target: TicketPhase::Failed,
notify: NotifyPolicy::Notify,
log_label: "dispatch panic",
breaker_trip: false,
},
SYSTEM_ROLE,
&panic_comment,
)
.await;
}
});
}
#[derive(Copy, Clone)]
struct VerifierInfo {
role: Role,
log_label: &'static str,
success_phase: TicketPhase,
active_phase: TicketPhase,
prompt_template: &'static str,
extraction_prompt_path: &'static str,
}
const REVIEWER_VI: VerifierInfo = VerifierInfo {
role: Role::Reviewer,
log_label: "Reviewers",
success_phase: TicketPhase::Reviewed,
active_phase: TicketPhase::InReview,
prompt_template: "review.md",
extraction_prompt_path: "extraction/reviewer.md",
};
const QA_VI: VerifierInfo = VerifierInfo {
role: Role::Qa,
log_label: "QA",
success_phase: TicketPhase::QaPassed,
active_phase: TicketPhase::InQa,
prompt_template: "qa.md",
extraction_prompt_path: "extraction/qa.md",
};
#[derive(Copy, Clone)]
struct PollPhaseInfo {
expected_phase: TicketPhase,
pipeline_check: PipelineCheck,
circuit_breaker_kind: Option<CircuitBreakerKind>,
claim_grace: Option<ChronoDuration>,
log_label: &'static str,
}
impl PollPhaseInfo {
const fn new(expected_phase: TicketPhase, log_label: &'static str) -> Self {
Self {
expected_phase,
pipeline_check: PipelineCheck::Skip,
circuit_breaker_kind: None,
claim_grace: None,
log_label,
}
}
}
#[derive(Copy, Clone)]
enum PollPhase {
BacklogAnalysis,
EngineerDevelopment,
SanitationCheck,
DiagnosticsCheck,
VerifierCheck(VerifierInfo),
}
impl PollPhase {
fn info(self) -> PollPhaseInfo {
match self {
Self::BacklogAnalysis => PollPhaseInfo {
claim_grace: Some(BoardStore::BACKLOG_CLAIM_GRACE),
..PollPhaseInfo::new(TicketPhase::Analysis, "Analyst")
},
Self::EngineerDevelopment => PollPhaseInfo {
pipeline_check: PipelineCheck::Enforce,
..PollPhaseInfo::new(TicketPhase::InDevelopment, "Engineer")
},
Self::SanitationCheck => PollPhaseInfo {
circuit_breaker_kind: Some(CircuitBreakerKind::Sanitation),
..PollPhaseInfo::new(TicketPhase::InSanitation, "Sanitation")
},
Self::DiagnosticsCheck => PollPhaseInfo {
circuit_breaker_kind: Some(CircuitBreakerKind::Diagnostics),
..PollPhaseInfo::new(TicketPhase::InDiagnostics, "Diagnostics")
},
Self::VerifierCheck(vi) => PollPhaseInfo::new(vi.active_phase, vi.log_label),
}
}
}
const CLAIM_PHASES: &[(TicketPhase, PollPhase)] = &[
(TicketPhase::Backlog, PollPhase::BacklogAnalysis),
(
TicketPhase::ReadyForDevelopment,
PollPhase::EngineerDevelopment,
),
(
TicketPhase::DiagnosticsDone,
PollPhase::VerifierCheck(REVIEWER_VI),
),
(TicketPhase::Reviewed, PollPhase::VerifierCheck(QA_VI)),
];
async fn spawn_for_each_ticket_in_phase<F, Fut>(phase: TicketPhase, ws: &Workspace, f: F)
where
F: Fn(Ticket, Workspace) -> Fut + Clone + Send + 'static,
Fut: Future<Output = ()> + Send + 'static,
{
match board().list_all_tickets(Some(&ws.name), Some(phase)).await {
Ok(tickets) => {
for ticket in tickets {
let f = f.clone();
let ws = ws.clone();
tokio::spawn(async move {
f(ticket, ws).await;
});
}
}
Err(e) => {
error!(workspace = %ws.name, phase = %phase, error = %e, "Phase listing failed");
}
}
}
async fn poll_round() {
let workspaces = match crate::workspace::store().list().await {
Ok(ws_list) => ws_list,
Err(e) => {
error!(error = %e, "Failed to list workspaces");
return;
}
};
let tasks: Vec<_> = workspaces
.into_iter()
.map(|ws| {
tokio::spawn(async move {
process_single_workspace(ws).await;
})
})
.collect();
let results = join_all(tasks).await;
crate::util::log_join_failures(
results,
"Panic in workspace poll round — management loop continues",
"Workspace poll task was cancelled — management loop continues",
);
}
async fn process_single_workspace(ws: Workspace) {
let ws = match pickup_pending_workspace(&ws).await {
Some(claimed) => claimed,
None => ws,
};
run_claim_pipeline(&ws).await;
spawn_for_each_ticket_in_phase(TicketPhase::InDiagnostics, &ws, |ticket, ws| async move {
if ticket.assigned_to.is_some() {
return;
}
spawn_dispatch(PollPhase::DiagnosticsCheck, ticket, ws);
})
.await;
spawn_for_each_ticket_in_phase(
TicketPhase::SanitationPassed,
&ws,
|ticket, ws| async move {
let Some(porcelain) = ensure_git_or_done_and_get_status(
&ticket,
&ws,
TicketPhase::SanitationPassed,
"finalize",
)
.await
else {
return;
};
finalize_ticket_with_git_status(ticket, ws, TicketPhase::SanitationPassed, &porcelain)
.await;
},
)
.await;
spawn_for_each_ticket_in_phase(TicketPhase::QaPassed, &ws, handle_qa_passed).await;
spawn_for_each_ticket_in_phase(TicketPhase::InSanitation, &ws, |ticket, ws| async move {
if ticket.assigned_to.is_some() {
return;
}
spawn_dispatch(PollPhase::SanitationCheck, ticket, ws);
})
.await;
}
async fn pickup_pending_workspace(ws: &Workspace) -> Option<Workspace> {
let (generation, discover_diagnostics) = pickup_claim(ws).await?;
info!(
workspace = %ws.name,
generation,
discover_diagnostics,
"Pickup: pending workspace claimed into discovery"
);
spawn_workspace_discovery(ws, generation, discover_diagnostics);
crate::workspace::store()
.get_by_name(&ws.name)
.await
.ok()
.flatten()
}
async fn pickup_claim(ws: &Workspace) -> Option<(i64, bool)> {
if ws.status != WorkspaceStatus::Pending {
return None;
}
if !crate::config::provider_configured() {
return None;
}
if crate::workspace::pending_pickup_cooldown_active(&ws.name) {
return None;
}
let storage = crate::workspace::store();
let generation = match storage.claim_pending_for_discovery(&ws.name).await {
Ok(Some(generation)) => generation,
Ok(None) => return None,
Err(e) => {
warn!(
workspace = %ws.name,
error = %e,
"Pickup: failed to claim pending workspace — retrying next poll cycle"
);
return None;
}
};
let discover_diagnostics = match storage.get_by_name(&ws.name).await {
Ok(Some(fresh)) => fresh.diagnostics.is_none(),
Ok(None) | Err(_) => ws.diagnostics.is_none(),
};
Some((generation, discover_diagnostics))
}
fn blocks_claim(ws: &Workspace, phase: PollPhase) -> bool {
if !matches!(
phase,
PollPhase::BacklogAnalysis | PollPhase::EngineerDevelopment
) {
return false;
}
ws.paused || ws.status != WorkspaceStatus::Ready
}
async fn run_claim_pipeline(ws: &Workspace) {
let board = board();
for &(source, phase) in CLAIM_PHASES {
if blocks_claim(ws, phase) {
continue;
}
let info = phase.info();
let ticket = match board
.claim_ticket_in_workspace(
source,
info.expected_phase,
&ws.name,
info.pipeline_check,
info.claim_grace,
)
.await
{
Ok(Some(t)) => {
ticket_buffer::push(
&ws.name,
&t.id,
source,
t.phase,
ticket_buffer::TransitionOrigin::Pipeline,
);
t
}
Ok(None) => continue,
Err(e) => {
error!(
workspace = %ws.name,
phase = %info.log_label,
error = %e,
"Claim failed, skipping remaining claim phases for workspace",
);
break;
}
};
spawn_dispatch(phase, ticket, ws.clone());
}
}
async fn run_single_agent(
agent_id: String,
role: Role,
ws: &Workspace,
ticket: &Ticket,
message: &str,
incoming_rx: tokio::sync::mpsc::UnboundedReceiver<crate::message_router::AgentJob>,
resume: bool,
) -> (Agent, Option<String>) {
run_agent(
agent_id,
role,
ws,
Some(ticket),
message,
String::new(),
String::new(),
Some(incoming_rx),
resume,
None,
None,
None,
)
.await
}
fn engineer_failure_comment(shutdown: bool, cancelled: bool, error: Option<&str>) -> String {
if shutdown {
return "Engineer failed: service shutting down — the run was interrupted \
by process shutdown."
.to_string();
}
if cancelled {
return "Engineer failed: cancelled by user.".to_string();
}
let Some(detail) = error else {
return load_prompt("pipeline/engineer_failed.md");
};
let detail = crate::util::scrub_credentials(detail);
let detail = crate::util::truncate_sandwich(
&detail,
crate::util::FAILURE_DETAIL_CAP,
"engineer failure",
);
if detail.contains(RETRY_EXHAUSTION_MARKER) {
format!("Engineer failed: LLM provider retry exhaustion.\n\n{detail}")
} else {
format!("Engineer failed.\n\n{detail}")
}
}
async fn engineer_comment_text(agent: &Agent, raw: &str) -> String {
let ticket_id = agent.ticket.as_ref().map_or("?", |t| t.id.as_str());
let policy = crate::retry::RetryPolicy::comment();
let extraction_prompt = load_prompt("extraction/engineer.md");
let summary = match agent
.extract_verdict::<crate::EngineerSummary>(&extraction_prompt, None, Some(&policy))
.await
{
Ok(summary) => summary,
Err(e) => {
warn!(
ticket = %ticket_id,
error = %e,
"Engineer summary extraction failed — using raw response for ticket comment"
);
return crate::util::scrub_credentials(raw);
}
};
let items: Vec<&str> = summary
.items
.iter()
.map(String::as_str)
.filter(|s| !s.trim().is_empty())
.collect();
if items.is_empty() {
warn!(
ticket = %ticket_id,
"Engineer summary extraction returned no usable items — using raw response for ticket comment"
);
return crate::util::scrub_credentials(raw);
}
let mut out = String::from("Implemented / fixed / executed:");
for item in items {
let _ = write!(out, "\n- {}", item.replace('\n', " "));
}
let synopsis = summary.summary.as_deref().unwrap_or("").trim();
if !synopsis.is_empty() {
let _ = write!(out, "\n\n### Summary\n{synopsis}");
}
crate::util::truncate_sandwich(
&crate::util::scrub_credentials(&out),
crate::util::FAILURE_DETAIL_CAP,
"engineer summary comment",
)
}
fn stage_round_drain_cut(
ticket_id: &str,
label: &str,
response: Option<&str>,
resumed: bool,
) -> bool {
let drain_cut = response.is_none() && crate::shutdown::aborting();
if drain_cut && !resumed {
info!(
ticket = %ticket_id,
"{label} round cut short by drain — job stays launched for boot resume",
);
}
drain_cut
}
async fn finalize_engineer_round(
ticket: &Ticket,
agent: &Agent,
response: Option<&str>,
job_id: &str,
resumed: bool,
) {
if phase_changed_and_clear_assignment(&ticket.id, TicketPhase::InDevelopment).await {
complete_ticket_stage_job(job_id).await;
return;
}
if stage_round_drain_cut(&ticket.id, "Engineer", response, resumed) {
return;
}
if let Some(text) = response {
let comment_text = engineer_comment_text(agent, text).await;
comment_and_transition_or_bail(
TransitionCtx {
ticket,
source: TicketPhase::InDevelopment,
target: TicketPhase::InDiagnostics,
notify: NotifyPolicy::Buffer,
log_label: "Engineer",
breaker_trip: false,
},
Role::Engineer.as_str(),
&comment_text,
if resumed {
"Resumed engineer finished — transitioned ticket"
} else {
"Engineer finished — transitioned ticket"
},
)
.await;
complete_ticket_stage_job(job_id).await;
return;
}
if !handle_engineer_failure(ticket, agent, resumed).await {
return;
}
complete_ticket_stage_job(job_id).await;
}
async fn handle_engineer_failure(ticket: &Ticket, agent: &Agent, resumed: bool) -> bool {
let cancelled = agent.is_cancelled();
let pause_reason = if cancelled {
"user cancelled the agent run"
} else {
"engineer agent failure"
};
let pause_note = pause_workspace_on_failure(ticket, pause_reason).await;
if crate::shutdown::aborting() {
info!(
ticket = %ticket.id,
"Engineer failure cut short by shutdown/drain after the pause — job stays launched for boot resume",
);
return false;
}
let failure_comment = engineer_failure_comment(
crate::shutdown::shutdown_token().is_cancelled(),
cancelled,
agent.failure.as_deref(),
);
let comment_text = format!("{failure_comment}{pause_note}");
if cancelled {
comment_and_transition_or_bail(
TransitionCtx {
ticket,
source: TicketPhase::InDevelopment,
target: TicketPhase::Failed,
notify: NotifyPolicy::Notify,
log_label: "Engineer",
breaker_trip: false,
},
SYSTEM_ROLE,
&comment_text,
if resumed {
"Resumed engineer cancelled — transitioned ticket"
} else {
"Engineer cancelled — transitioned ticket"
},
)
.await;
} else {
bounce_engineer_hard_failure(ticket, &comment_text, resumed).await;
}
true
}
async fn bounce_engineer_hard_failure(ticket: &Ticket, comment_text: &str, resumed: bool) {
let bounce_trip = usize::try_from(ticket.bounce_count).unwrap_or(usize::MAX)
>= crate::joint_verdict::MAX_BOUNCES;
let trip_comment = bounce_trip.then(bounce_breaker_trip_comment);
let target = if bounce_trip {
TicketPhase::Failed
} else {
TicketPhase::ReadyForDevelopment
};
let outcome = with_comment_and_transition(
TransitionCtx {
ticket,
source: TicketPhase::InDevelopment,
target,
notify: NotifyPolicy::Notify,
log_label: "Engineer",
breaker_trip: false,
},
async |tx| {
if let Some(comment) = &trip_comment {
BoardStore::add_comment_tx(tx, &ticket.id, SYSTEM_ROLE, comment).await?;
}
BoardStore::add_comment_tx(tx, &ticket.id, SYSTEM_ROLE, comment_text).await?;
if !bounce_trip {
BoardStore::increment_bounce_count_tx(tx, &ticket.id).await?;
}
Ok(())
},
)
.await;
if matches!(outcome, FinalizeOutcome::Applied) {
let resumed_prefix = if resumed { "Resumed " } else { "" };
if bounce_trip {
info!(
ticket = %ticket.id,
"{resumed_prefix}Engineer hard failure exhausted the bounce budget — ticket failed",
);
} else {
info!(
ticket = %ticket.id,
"{resumed_prefix}Engineer hard failure — ticket bounced to ReadyForDevelopment for retry",
);
}
}
}
async fn dispatch_engineer(ticket: Arc<Ticket>, ws: Workspace) {
let agent_id = ticket_agent_id(&ticket.id, Role::Engineer.as_str());
let last_eng_pos = ticket
.comments
.iter()
.rposition(|c| c.role == Role::Engineer.as_str());
let feedback: Vec<&str> = ticket
.comments
.iter()
.skip(last_eng_pos.map_or(0, |i| i + 1))
.map(|c| c.content.as_str())
.collect();
let message = if feedback.is_empty() {
load_prompt("implement.md")
} else {
substitute(
&load_prompt("pipeline/bounce_feedback.md"),
&[("{{feedback}}", &feedback.join("\n---\n"))],
)
};
let job_id = crate::generate_id();
let Ok(_slot) = spawn_single_slot_round(
&job_id,
&ticket,
&ws,
"engineer",
TicketPhase::InDevelopment,
Role::Engineer,
&message,
&agent_id,
)
.await
else {
error!(
ticket = %ticket.id,
"Failed to spawn engineer job — aborting dispatch",
);
return;
};
run_stage_agent_round(
&ticket,
&ws,
&job_id,
&message,
false,
StageRoundKind::Engineer,
)
.await;
}
#[derive(Clone, Copy)]
enum StageRoundKind {
Engineer,
Sanitation,
}
async fn run_stage_agent_round(
ticket: &Ticket,
ws: &Workspace,
job_id: &str,
task: &str,
resumed: bool,
kind: StageRoundKind,
) {
if let StageRoundKind::Engineer = kind
&& let Err(e) = crate::jobs::upsert_engineer_anchor(
&crate::session::store().conn,
&ticket.id,
task,
crate::jobs::RowStatus::Launched,
)
.await
{
warn!(
ticket = %ticket.id,
job = %job_id,
error = %e,
"Failed to upsert engineer anchor — session continuity across bounces degraded",
);
}
let agent_id = match kind {
StageRoundKind::Engineer => crate::jobs::engineer_anchor_id(&ticket.id),
StageRoundKind::Sanitation => format!("ticket_{job_id}_sanitation"),
};
let incoming_rx = register_agent_and_assign(
&ticket.id,
&agent_id,
match kind {
StageRoundKind::Engineer if resumed => {
"Failed to persist assigned_to for resumed engineer — comments may not route"
}
StageRoundKind::Engineer => {
"Failed to persist assigned_to — stale agent already cancelled at dispatch, proceeding without DB assignment"
}
StageRoundKind::Sanitation => {
"Failed to persist assigned_to for sanitation agent — mid-run comments may not route"
}
},
)
.await;
let has_session = resumed && crate::session::store().has_content(&agent_id).await;
let message = if has_session {
String::new()
} else {
task.to_string()
};
let (agent, response) = run_single_agent(
agent_id,
match kind {
StageRoundKind::Engineer => Role::Engineer,
StageRoundKind::Sanitation => Role::Sanitation,
},
ws,
ticket,
&message,
incoming_rx,
resumed,
)
.await;
match kind {
StageRoundKind::Engineer => {
finalize_engineer_round(ticket, &agent, response.as_deref(), job_id, resumed).await;
}
StageRoundKind::Sanitation => {
finalize_sanitation_round(ticket, &agent, response.as_deref(), job_id, resumed).await;
}
}
}
async fn determine_notify_policy(workspace_name: &str, ticket_id: &str) -> NotifyPolicy {
match board()
.has_active_tickets_excluding(workspace_name, ticket_id)
.await
{
Ok(true) => {
debug!(
ticket = %ticket_id,
workspace = %workspace_name,
"Other active tickets remain — buffering Done notification",
);
NotifyPolicy::Buffer
}
Ok(false) => NotifyPolicy::Notify,
Err(e) => {
warn!(
ticket = %ticket_id,
workspace = %workspace_name,
error = %e,
"Failed to check active tickets — notifying to be safe",
);
NotifyPolicy::Notify
}
}
}
async fn transition_ticket_to_done(ticket: &Ticket, source: TicketPhase, comment: &str) {
let notify_policy = determine_notify_policy(&ticket.workspace_name, &ticket.id).await;
let log_label = "Finalize";
if matches!(
comment_and_transition(
TransitionCtx {
ticket,
source,
target: TicketPhase::Done,
notify: notify_policy,
log_label,
breaker_trip: false,
},
SYSTEM_ROLE,
comment,
)
.await,
FinalizeOutcome::Applied
) {
info!(ticket = %ticket.id, "{comment}");
}
}
async fn ensure_git_or_done_and_get_status(
ticket: &Ticket,
ws: &Workspace,
phase: TicketPhase,
error_context: &'static str,
) -> Option<String> {
let repo_path = ws.as_path();
if !crate::git_commands::git_is_installed().await {
transition_ticket_to_done(
ticket,
phase,
"Git not installed — moving to Done without commit",
)
.await;
return None;
}
if !crate::git_commands::is_git_repo(repo_path) {
transition_ticket_to_done(
ticket,
phase,
"Not a git repo — moving to Done without commit",
)
.await;
return None;
}
match run_git_status(repo_path).await {
Ok(porcelain) => Some(porcelain),
Err(e) => {
warn!(
ticket = %ticket.id,
error = %e,
"Failed to check git status — staying in {} for retry: {}",
phase.as_ref(),
error_context,
);
None
}
}
}
async fn finalize_ticket_with_git_status(
ticket: Ticket,
ws: Workspace,
source: TicketPhase,
porcelain: &str,
) {
let repo_path = ws.as_path();
if porcelain.trim().is_empty() {
transition_ticket_to_done(
&ticket,
source,
"Clean working tree — moving to Done without commit",
)
.await;
return;
}
match crate::git_commands::run_git_commit(repo_path, &ticket.title).await {
Ok(commit_info) => {
finalize_commit_and_transition(&ticket, commit_info, source).await;
}
Err(e) => {
error!(
ticket = %ticket.id,
error = %e,
"Commit failed — staying in {} for retry",
source.as_ref(),
);
}
}
}
async fn finalize_commit_and_transition(
ticket: &Ticket,
commit_info: crate::git_commands::CommitInfo,
source: TicketPhase,
) {
let comment = format_commit_summary(
commit_info.short_hash(),
commit_info.lines_added,
commit_info.lines_removed,
);
let phase_label = source.as_ref();
crate::registry::AGENT_REGISTRY.cancel_by_ticket_id(&ticket.id);
let notify_policy = determine_notify_policy(&ticket.workspace_name, &ticket.id).await;
let log_label = format!(
"finalize Done transition from {phase_label} ({})",
commit_info.short_hash(),
);
if matches!(
with_comment_and_transition(
TransitionCtx {
ticket,
source,
target: TicketPhase::Done,
notify: notify_policy,
log_label: &log_label,
breaker_trip: false,
},
async |tx| {
BoardStore::set_commit_info_tx(
tx,
&ticket.id,
&commit_info.hash,
commit_info.lines_added,
commit_info.lines_removed,
)
.await?;
BoardStore::add_comment_tx(tx, &ticket.id, SYSTEM_ROLE, &comment).await?;
Ok(())
},
)
.await,
FinalizeOutcome::Applied
) {
info!(ticket = %ticket.id, "Committed {}, moving to Done", commit_info.short_hash());
}
}
fn format_commit_summary(short_hash: &str, added: i64, removed: i64) -> String {
match (added, removed) {
(0, 0) => format!("Committed as `{short_hash}` (no changes)"),
(a, 0) => format!("Committed as `{short_hash}` (+{a})"),
(0, r) => format!("Committed as `{short_hash}` (-{r})"),
(a, r) => format!("Committed as `{short_hash}` (+{a}/-{r})"),
}
}
async fn handle_qa_passed(ticket: Ticket, ws: Workspace) {
let Some(porcelain) = ensure_git_or_done_and_get_status(
&ticket,
&ws,
TicketPhase::QaPassed,
"untracked files check",
)
.await
else {
return;
};
let untracked = parse_new_files_from_porcelain(&porcelain);
if untracked.is_empty() {
finalize_ticket_with_git_status(ticket, ws, TicketPhase::QaPassed, &porcelain).await;
} else {
let agent_id = ticket_agent_id(&ticket.id, Role::Sanitation.as_str());
let claimed = match board().claim_sanitation(&ticket.id, &agent_id).await {
Ok(c) => c,
Err(e) => {
warn!(
ticket = %ticket.id,
error = %e,
"Failed to transition QaPassed ticket to InSanitation"
);
return;
}
};
if !claimed {
debug!(
ticket = %ticket.id,
"QaPassed ticket moved externally — skipping sanitation dispatch",
);
return;
}
ticket_buffer::push(
&ticket.workspace_name,
&ticket.id,
TicketPhase::QaPassed,
TicketPhase::InSanitation,
ticket_buffer::TransitionOrigin::Pipeline,
);
spawn_dispatch(PollPhase::SanitationCheck, ticket, ws);
}
}
async fn record_sanitation_failure(
ticket_id: &str,
reason: impl std::fmt::Display,
raw_dump: Option<&crate::retry::RetryExhausted>,
) {
let reason_str = match raw_dump {
Some(failure) => format!(
"{} — {reason}\n\n{}",
load_prompt("pipeline/sanitation_failed.md"),
raw_response_dump_section(failure)
),
None => format!(
"{} — {reason}",
load_prompt("pipeline/sanitation_failed.md")
),
};
if let Err(e) = crate::turso::with_tx(
&board().conn,
ticket_id,
"record sanitation failure",
async |tx| {
BoardStore::add_comment_tx(tx, ticket_id, SANITATION_ROLE, &reason_str).await?;
BoardStore::set_assigned_to_tx(tx, ticket_id, None).await?;
Ok(())
},
)
.await
{
warn!(
ticket = %ticket_id,
error = %e,
"Failed to record sanitation failure (circuit-breaker comment + assigned_to clear)",
);
}
}
async fn register_agent_and_assign(
ticket_id: &str,
agent_id: &str,
warn_message: &str,
) -> tokio::sync::mpsc::UnboundedReceiver<crate::message_router::AgentJob> {
let incoming_rx = message_router::register_agent(agent_id);
if let Err(e) = board()
.set_assigned_to_no_cancel(ticket_id, Some(agent_id))
.await
{
warn!(
ticket = ticket_id,
error = %e,
"{warn_message}",
);
}
incoming_rx
}
async fn finalize_sanitation_round(
ticket: &Ticket,
agent: &Agent,
response: Option<&str>,
job_id: &str,
resumed: bool,
) {
if phase_changed_and_clear_assignment(&ticket.id, TicketPhase::InSanitation).await {
complete_ticket_stage_job(job_id).await;
return;
}
if stage_round_drain_cut(&ticket.id, "Sanitation", response, resumed) {
return;
}
let resumed_suffix = if resumed { " (resumed)" } else { "" };
if response.is_none() {
warn!(
ticket = %ticket.id,
"Sanitation agent returned no output{resumed_suffix} — clearing assigned_to for retry"
);
record_sanitation_failure(
&ticket.id,
format!("agent returned no output{resumed_suffix}"),
None,
)
.await;
complete_ticket_stage_job(job_id).await;
return;
}
let extraction_prompt = crate::prompt::load_prompt("extraction/sanitation.md");
match agent
.extract_verdict::<crate::SanitationVerdict>(&extraction_prompt, None, None)
.await
{
Ok(verdict) => process_sanitation_verdict(ticket, verdict).await,
Err(failure) => {
warn!(
ticket = %ticket.id,
error = %failure,
"Failed to extract sanitation verdict{resumed_suffix} — clearing assigned_to for retry"
);
record_sanitation_failure(
&ticket.id,
format!("verdict extraction error{resumed_suffix}: {failure}"),
Some(&failure),
)
.await;
}
}
complete_ticket_stage_job(job_id).await;
}
async fn dispatch_sanitation(ticket: Arc<Ticket>, ws: Workspace) {
let untracked_files = match list_new_or_untracked_files(ws.as_path()).await {
Ok(files) => files.join("\n"),
Err(e) => {
warn!(
ticket = %ticket.id,
error = %e,
"Failed to list untracked files — proceeding with empty list",
);
String::from("(could not list untracked files)")
}
};
let prompt = substitute(
&crate::prompt::load_prompt("sanitation.md"),
&[
("{{ticket_title}}", &ticket.title),
("{{ticket_description}}", &ticket.description),
("{{untracked_files}}", &untracked_files),
],
);
let job_id = crate::generate_id();
let Ok(_slot) = spawn_single_slot_round(
&job_id,
&ticket,
&ws,
"sanitation",
TicketPhase::InSanitation,
Role::Sanitation,
&prompt,
&format!("ticket_{job_id}_sanitation"),
)
.await
else {
error!(
ticket = %ticket.id,
"Failed to spawn sanitation job — aborting dispatch",
);
return;
};
run_stage_agent_round(
&ticket,
&ws,
&job_id,
&prompt,
false,
StageRoundKind::Sanitation,
)
.await;
}
async fn process_sanitation_verdict(ticket: &Ticket, verdict: crate::SanitationVerdict) {
if verdict.pass {
let passed_suffix = if verdict.garbage_files.is_empty() {
""
} else {
" (files reviewed)"
};
let comment = format!(
"🧹 Sanitation passed{passed_suffix}: {rationale}",
rationale = verdict.rationale
);
comment_and_transition_or_bail(
TransitionCtx {
ticket,
source: TicketPhase::InSanitation,
target: TicketPhase::SanitationPassed,
notify: NotifyPolicy::Buffer,
log_label: "Sanitation",
breaker_trip: false,
},
Role::Sanitation.as_str(),
&comment,
"Sanitation passed — transitioned to SanitationPassed",
)
.await;
} else {
let garbage_list = verdict.garbage_files.join("\n- ");
let comment = substitute(
&load_prompt("pipeline/sanitation_failed_comment.md"),
&[
("{{garbage_list}}", &garbage_list),
("{{rationale}}", &verdict.rationale),
],
);
let count_str = verdict.garbage_files.len().to_string();
let sys_comment = substitute(
&load_prompt("pipeline/sanitation_circuit_breaker_comment.md"),
&[
(
"{{sanitation_failed_marker}}",
&load_prompt("pipeline/sanitation_failed.md"),
),
("{{count}}", &count_str),
],
);
if !matches!(
with_comment_and_transition(
TransitionCtx {
ticket,
source: TicketPhase::InSanitation,
target: TicketPhase::ReadyForDevelopment,
notify: NotifyPolicy::Buffer,
log_label: "Sanitation",
breaker_trip: false,
},
async |tx| {
BoardStore::add_comment_tx(
tx,
&ticket.id,
Role::Sanitation.as_str(),
comment.as_str(),
)
.await?;
BoardStore::add_comment_tx(
tx,
&ticket.id,
SANITATION_ROLE,
sys_comment.as_str(),
)
.await?;
Ok(())
},
)
.await,
FinalizeOutcome::Applied
) {
return;
}
info!(
ticket = %ticket.id,
"Sanitation failed — bounced back to ReadyForDevelopment with pipeline reservation",
);
}
}
async fn run_diagnostics_commands(diag: &DiagnosticsCommands, ws: &Workspace) -> (String, bool) {
let mut comment = String::new();
let mut all_passed = true;
let mut failed_at: &str = "";
for (label, cmd_opt) in diag.commands() {
let Some(cmd) = cmd_opt else {
continue;
};
let started = std::time::Instant::now();
match ShellTool::new(ShellMode::Full)
.execute_with_status(ws, serde_json::json!({"command": cmd}))
.await
{
Ok((_output, Some(0))) => {
let _ = write!(
comment,
"\n\n{label} ({cmd}): PASSED in {:.1}s",
started.elapsed().as_secs_f64(),
);
}
Ok((output, _exit_code)) => {
let _ = write!(comment, "\n\n{label} ({cmd}):\n");
let display = if output.is_empty() {
"(no output)".to_string()
} else {
output
};
comment.push_str(&display);
all_passed = false;
failed_at = label;
break;
}
Err(e) => {
let _ = write!(comment, "\n\n{label} ({cmd}):\n");
comment.push_str(&e.to_string());
all_passed = false;
failed_at = label;
break;
}
}
}
if all_passed {
comment.push_str("\n\n---\n");
comment.push_str(&load_prompt("pipeline/diagnostics_passed.md"));
} else {
let _ = write!(
comment,
"\n\n---\n{} {failed_at}",
load_prompt("pipeline/diagnostics_failed.md"),
);
}
crate::tools::shell::cleanup_agent_spills(crate::tools::shell::NON_AGENT_SPILL_OWNER);
(comment.trim_start_matches('\n').to_string(), all_passed)
}
async fn dispatch_diagnostics(ticket: Arc<Ticket>, ws: Workspace) {
match board()
.claim_diagnostics(&ticket.id, DIAGNOSTICS_ROLE)
.await
{
Err(e) => {
error!(
ticket = %ticket.id,
error = %e,
"Diagnostics claim error — bailing out",
);
return;
}
Ok(false) => {
warn!(
ticket = %ticket.id,
"Diagnostics claim failed — ticket already claimed or moved out of InDiagnostics"
);
return;
}
Ok(true) => {}
}
let (target_phase, comment_body, outcome_log): (TicketPhase, String, &str) =
match crate::workspace::store().get_diagnostics(&ws.name).await {
Ok(Some(cmds)) if !cmds.is_empty() => {
let (comment, all_passed) = run_diagnostics_commands(&cmds, &ws).await;
if phase_changed_and_clear_assignment(&ticket.id, TicketPhase::InDiagnostics).await
{
return;
}
if all_passed {
(
TicketPhase::DiagnosticsDone,
comment,
"Diagnostics finished — transitioned ticket",
)
} else {
(
TicketPhase::ReadyForDevelopment,
comment,
"Diagnostics failed — transitioned ticket",
)
}
}
Ok(_) => {
(
TicketPhase::DiagnosticsDone,
"No diagnostics commands are configured for this workspace \
— diagnostics skipped."
.to_string(),
"Diagnostics skipped — transitioned ticket",
)
}
Err(e) => {
warn!(
ticket = %ticket.id,
error = %e,
"Failed to load diagnostics for workspace — transitioning to DiagnosticsDone",
);
(
TicketPhase::DiagnosticsDone,
format!("Could not load diagnostics commands due to a database error: {e}"),
"Diagnostics failed — transitioned ticket",
)
}
};
comment_and_transition_or_bail(
TransitionCtx {
ticket: &ticket,
source: TicketPhase::InDiagnostics,
target: target_phase,
notify: NotifyPolicy::Buffer,
log_label: "Diagnostics",
breaker_trip: false,
},
DIAGNOSTICS_ROLE,
&comment_body,
outcome_log,
)
.await;
}
#[derive(Clone)]
pub(crate) enum ParallelVerdict {
NoResponse(String),
ParseFailed(crate::retry::RetryExhausted),
Verdict(crate::Verdict),
}
fn validate_verdict_score(v: &crate::Verdict) -> Result<(), String> {
if v.score <= 10 {
Ok(())
} else {
Err(format!("verdict score {} out of range [0,10]", v.score))
}
}
#[must_use]
fn stage_name(role: Role) -> &'static str {
match role {
Role::Analyst => "Analysis",
Role::Reviewer => "Review",
Role::Qa => "QA",
_ => unreachable!("stage_name called with a non-verdict role"),
}
}
#[must_use]
pub(crate) fn stage_role(name: &str) -> Option<Role> {
match name {
"Analysis" => Some(Role::Analyst),
"Review" => Some(Role::Reviewer),
"QA" => Some(Role::Qa),
_ => None,
}
}
#[expect(clippy::too_many_arguments)]
async fn build_round_joint_comment(
stage: &str,
results: &[ParallelVerdict],
threshold: u8,
role: Role,
header: &str,
ws: &Workspace,
ticket_id: &str,
ticket_title: &str,
) -> String {
let mut verdicts: Vec<crate::joint_verdict::JointVerdict<'_>> = Vec::new();
let mut failures: Vec<crate::joint_verdict::JointFailure> = Vec::new();
for (i, r) in results.iter().enumerate() {
match r {
ParallelVerdict::Verdict(v) => verdicts.push(crate::joint_verdict::JointVerdict {
agent_index: i,
verdict: v,
}),
ParallelVerdict::NoResponse(reason) => {
failures.push(crate::joint_verdict::JointFailure {
agent_index: i,
dump: reason.clone(),
});
}
ParallelVerdict::ParseFailed(f) => failures.push(crate::joint_verdict::JointFailure {
agent_index: i,
dump: crate::util::scrub_credentials(&raw_response_dump_section(f)),
}),
}
}
let round = crate::joint_verdict::JointRound {
stage,
dispatched: results.len(),
verdicts,
failures,
header: header.to_string(),
threshold,
};
if round
.verdicts
.iter()
.all(|v| v.verdict.issues_detected.is_empty())
{
crate::joint_verdict::render_joint_comment(
&round,
&crate::consensus::RepairOutcome::Fallback,
&crate::consensus::ItemTable::new(&crate::joint_verdict::issues_by_agent(&round)),
)
} else {
crate::joint_verdict::build_joint_comment(&round, role, ws, ticket_id, ticket_title).await
}
}
fn load_verifier_angles(role: Role) -> Vec<String> {
match role {
Role::Reviewer => load_prompt_sections("review_angles.md"),
Role::Qa => load_prompt_sections("qa_angles.md"),
_ => Vec::new(),
}
}
#[expect(clippy::too_many_lines)]
async fn run_parallel_agents(
ticket: &Arc<Ticket>,
ws: &Workspace,
role: Role,
extraction_prompt: &str,
job_id: &str,
slots: &[TicketStageSlot],
resume: bool,
) -> Vec<ParallelVerdict> {
let launched: Vec<&TicketStageSlot> = slots
.iter()
.filter(|s| s.status != crate::jobs::RowStatus::Done)
.collect();
let receivers: Vec<_> = launched
.iter()
.map(|s| message_router::register_agent(&s.agent_id))
.collect();
let assigned_to_str = launched
.iter()
.map(|s| s.agent_id.as_str())
.collect::<Vec<_>>()
.join(",");
if !assigned_to_str.is_empty()
&& let Err(e) = board()
.set_assigned_to_no_cancel(&ticket.id, Some(&assigned_to_str))
.await
{
warn!(
ticket = %ticket.id,
error = %e,
"Failed to set assigned_to for parallel agents",
);
}
let mut results: Vec<ParallelVerdict> = Vec::with_capacity(slots.len());
{
let members: Vec<_> = launched
.iter()
.zip(receivers)
.map(|(slot, rx)| {
let ticket = Arc::clone(ticket);
let ws = ws.clone();
let extraction_prompt = extraction_prompt.to_string();
let agent_id = slot.agent_id.clone();
let task = slot.task.clone();
move |round: crate::agent::RoundOpts| async move {
if !is_ticket_in_phase(&ticket.id, ticket.phase).await {
if let Some(notify) = &round.first_call_notify {
notify.notify_one();
}
message_router::unregister_agent(&agent_id);
return ParallelVerdict::NoResponse(PHASE_GATE_BAIL_REASON.to_string());
}
let has_session =
resume && crate::session::store().has_content(&agent_id).await;
let (agent, response) = run_agent(
agent_id.clone(),
role,
&ws,
Some(&ticket),
if has_session { "" } else { &task },
String::new(),
String::new(),
Some(rx),
resume,
Some(round),
None,
None,
)
.await;
let response = response.unwrap_or_default();
if response.is_empty() {
let reason = agent.failure_reason("agent produced no response");
ParallelVerdict::NoResponse(crate::util::scrub_credentials(&reason))
} else {
let verdict = agent
.extract_verdict::<crate::Verdict>(
&extraction_prompt,
Some(&validate_verdict_score),
None,
)
.await;
match verdict {
Ok(v) => ParallelVerdict::Verdict(v),
Err(e) => ParallelVerdict::ParseFailed(e),
}
}
}
})
.collect();
let handles = crate::agent::spawn_staggered_round(members, resume).await;
let mut run_results: Vec<ParallelVerdict> = Vec::with_capacity(handles.len());
for handle in handles {
match handle.await {
Ok(v) => run_results.push(v),
Err(e) => run_results.push(round_member_failed(e)),
}
}
let conn = &crate::session::store().conn;
let mut by_agent: std::collections::HashMap<&str, &ParallelVerdict> =
std::collections::HashMap::new();
for (slot, result) in launched.iter().zip(&run_results) {
let outcome = serialize_verdict_outcome(result);
let status = if matches!(result, ParallelVerdict::NoResponse(_)) {
crate::jobs::RowStatus::Failed
} else {
crate::jobs::RowStatus::Done
};
if let Err(e) = crate::jobs::write_agent_outcome(
conn,
job_id,
&slot.agent_id,
status,
Some(&outcome),
)
.await
{
warn!(
job = %job_id,
agent = %slot.agent_id,
error = %e,
"Failed to checkpoint agent outcome",
);
}
by_agent.insert(slot.agent_id.as_str(), result);
}
for slot in slots {
if slot.status == crate::jobs::RowStatus::Done {
let outcome = slot.outcome.clone().unwrap_or_default();
results.push(deserialize_verdict_outcome(&outcome));
} else if let Some(r) = by_agent.get(slot.agent_id.as_str()) {
results.push((*r).clone());
} else {
unreachable!("every non-Done slot is launched and recorded 1:1 in by_agent");
}
}
}
if let Err(e) = board().set_assigned_to_no_cancel(&ticket.id, None).await {
warn!(
ticket = %ticket.id,
error = %e,
"Failed to clear assigned_to after parallel agents",
);
}
results
}
fn round_member_failed(e: tokio::task::JoinError) -> ParallelVerdict {
let reason = crate::util::scrub_credentials(&crate::util::panic_message(&*e.into_panic()));
tracing::warn!(%reason, "round member task failed");
ParallelVerdict::NoResponse(reason)
}
#[must_use]
fn verdict_passes(verdict: &crate::Verdict) -> bool {
verdict.score >= REVIEW_QA_THRESHOLD
}
fn raw_response_dump_section(failure: &crate::retry::RetryExhausted) -> String {
match failure.last_raw.as_deref() {
Some(text) if !text.trim().is_empty() => format!(
"Raw agent response (last attempt):\n```\n{}\n```",
crate::util::truncate_sandwich(
text,
crate::util::FAILURE_DETAIL_CAP,
"verdict response"
)
),
Some(_) => "Final attempt was a tool call — no text response produced.".to_string(),
None => format!(
"Extraction failed after {} attempt(s) — final failure: {} ({})",
failure.failures.len(),
failure.final_class.label(),
failure.detail,
),
}
}
struct TicketStageSlot {
idx: i64,
agent_id: String,
task: String,
status: crate::jobs::RowStatus,
outcome: Option<String>,
}
fn serialize_verdict_outcome(result: &ParallelVerdict) -> String {
match result {
ParallelVerdict::Verdict(v) => serde_json::json!({ "verdict": v }).to_string(),
ParallelVerdict::NoResponse(reason) => {
serde_json::json!({ "no_response": reason }).to_string()
}
ParallelVerdict::ParseFailed(f) => {
serde_json::json!({ "parse_failed": raw_response_dump_section(f) }).to_string()
}
}
}
fn deserialize_verdict_outcome(outcome: &str) -> ParallelVerdict {
let Ok(v) = serde_json::from_str::<serde_json::Value>(outcome) else {
return ParallelVerdict::NoResponse("unreadable stored outcome".to_string());
};
if let Some(verdict) = v.get("verdict") {
match serde_json::from_value(verdict.clone()) {
Ok(v) => ParallelVerdict::Verdict(v),
Err(_) => ParallelVerdict::NoResponse("unreadable stored verdict".to_string()),
}
} else if let Some(r) = v.get("no_response").and_then(serde_json::Value::as_str) {
ParallelVerdict::NoResponse(r.to_string())
} else if let Some(p) = v.get("parse_failed").and_then(serde_json::Value::as_str) {
ParallelVerdict::NoResponse(p.to_string())
} else {
ParallelVerdict::NoResponse("unrecognized stored outcome".to_string())
}
}
const fn agent_kind_for_role(role: Role) -> crate::jobs::AgentKind {
match role {
Role::Reviewer | Role::Qa => crate::jobs::AgentKind::Verifier,
Role::Engineer => crate::jobs::AgentKind::Engineer,
Role::Sanitation => crate::jobs::AgentKind::Sanitation,
_ => crate::jobs::AgentKind::Analyst,
}
}
async fn next_ticket_stage_round(
conn: &crate::turso::Connection,
ticket_id: &str,
stage: &str,
) -> i64 {
conn.query_row(
"SELECT COALESCE(MAX(round), 0) + 1 FROM ticket_stage_jobs WHERE ticket_id = ?1 AND stage = ?2",
crate::turso::params![ticket_id, stage],
|row| row.get::<i64>(0),
)
.await
.unwrap_or(1)
}
#[must_use]
fn ticket_stage_agent_id(ticket_id: &str, idx: i64, suffix: &str, role: Role) -> String {
format!("ticket_{}_{}_{}_{}", ticket_id, idx, suffix, role.as_str())
}
#[must_use]
fn ticket_stage_slot_task(
prompt: &str,
angles: &[String],
slot_count: usize,
global_idx: usize,
) -> String {
if angles.is_empty() {
prompt.to_string()
} else if slot_count == 1 {
format!("{prompt}\n\n{}", angles.join("\n\n"))
} else {
format!("{prompt}\n\n{}", angles[global_idx % angles.len()])
}
}
async fn spawn_ticket_stage_round(
ticket: &Ticket,
ws: &Workspace,
stage: &'static str,
phase: TicketPhase,
role: Role,
prompt: &str,
count: usize,
) -> anyhow::Result<(String, Vec<TicketStageSlot>)> {
let job_id = crate::generate_id();
let suffix = crate::generate_suffix();
let angles = load_verifier_angles(role);
let mut slots = Vec::with_capacity(count);
for i in 0..count {
let idx = i64::try_from(i).unwrap_or(i64::MAX);
let agent_id = ticket_stage_agent_id(&ticket.id, idx, &suffix, role);
let task = ticket_stage_slot_task(prompt, &angles, count, i);
slots.push(TicketStageSlot {
idx,
agent_id,
task,
status: crate::jobs::RowStatus::Launched,
outcome: None,
});
}
let agents: Vec<crate::jobs::NewAgent> = slots
.iter()
.map(|s| crate::jobs::NewAgent {
agent_id: s.agent_id.clone(),
kind: agent_kind_for_role(role),
idx: Some(s.idx),
task: s.task.clone(),
})
.collect();
let round = next_ticket_stage_round(&crate::session::store().conn, &ticket.id, stage).await;
crate::jobs::spawn_job(
&crate::session::store().conn,
&job_id,
prompt,
&ws.name,
"",
"",
role,
&agents,
&crate::jobs::SpawnChild::TicketStage {
ticket_id: ticket.id.clone(),
stage: stage.to_string(),
phase: phase.as_ref().to_string(),
round,
},
)
.await?;
Ok((job_id, slots))
}
#[expect(clippy::too_many_arguments)]
async fn spawn_single_slot_round(
job_id: &str,
ticket: &Ticket,
ws: &Workspace,
stage: &'static str,
phase: TicketPhase,
role: Role,
prompt: &str,
agent_id: &str,
) -> anyhow::Result<TicketStageSlot> {
let slot = TicketStageSlot {
idx: 0,
agent_id: agent_id.to_string(),
task: prompt.to_string(),
status: crate::jobs::RowStatus::Launched,
outcome: None,
};
crate::jobs::spawn_job(
&crate::session::store().conn,
job_id,
prompt,
&ws.name,
"",
"",
role,
&[crate::jobs::NewAgent {
agent_id: agent_id.to_string(),
kind: agent_kind_for_role(role),
idx: Some(0),
task: prompt.to_string(),
}],
&crate::jobs::SpawnChild::TicketStage {
ticket_id: ticket.id.clone(),
stage: stage.to_string(),
phase: phase.as_ref().to_string(),
round: next_ticket_stage_round(&crate::session::store().conn, &ticket.id, stage).await,
},
)
.await?;
Ok(slot)
}
async fn append_ticket_stage_slots(
ticket: &Ticket,
job_id: &str,
role: Role,
prompt: &str,
count: usize,
) -> anyhow::Result<Vec<TicketStageSlot>> {
let roster = crate::jobs::list_agents_for_job(&crate::session::store().conn, job_id).await?;
let roster_len = roster.len();
let next_idx = i64::try_from(roster_len).unwrap_or(i64::MAX);
let suffix = crate::generate_suffix();
let angles = load_verifier_angles(role);
let slot_count = roster_len + count;
let mut slots = Vec::with_capacity(count);
for (k, i) in (next_idx..next_idx + i64::try_from(count).unwrap_or(i64::MAX)).enumerate() {
let agent_id = ticket_stage_agent_id(&ticket.id, i, &suffix, role);
let task = ticket_stage_slot_task(prompt, &angles, slot_count, roster_len + k);
crate::session::store()
.conn
.execute(
crate::jobs::AGENT_INSERT_SQL,
crate::jobs::agent_params(
job_id,
&agent_id,
agent_kind_for_role(role),
Some(i),
&task,
),
)
.await?;
slots.push(TicketStageSlot {
idx: i,
agent_id,
task,
status: crate::jobs::RowStatus::Launched,
outcome: None,
});
}
Ok(slots)
}
async fn complete_ticket_stage_job(job_id: &str) {
if let Err(e) =
crate::jobs::complete_ticket_stage_job(&crate::session::store().conn, job_id).await
{
warn!(job = %job_id, error = %e, "Failed to terminalize ticket_stage job");
}
}
async fn load_ticket_stage_slots(job_id: &str) -> anyhow::Result<Vec<TicketStageSlot>> {
let rows = crate::jobs::list_agents_for_job(&crate::session::store().conn, job_id).await?;
Ok(rows
.into_iter()
.map(|r| TicketStageSlot {
idx: r.idx.unwrap_or(0),
agent_id: r.agent_id,
task: r.task,
status: r.status.parse().unwrap_or(crate::jobs::RowStatus::Launched),
outcome: r.outcome,
})
.collect())
}
#[must_use]
async fn maybe_escalate_analysis(
ticket: &Arc<Ticket>,
ws: &Workspace,
job_id: &str,
extraction_prompt: &str,
task: &str,
resume: bool,
results: &mut Vec<ParallelVerdict>,
) -> bool {
if results.len() == DEFAULT_PARALLEL_AGENT_COUNT
&& crate::joint_verdict::analysis_escalation_needed(results, DEFAULT_PARALLEL_AGENT_COUNT)
&& !crate::shutdown::aborting()
{
if complete_job_and_bail_if_phase_moved(&ticket.id, TicketPhase::Analysis, job_id).await {
return false;
}
if resume {
info!(ticket = %ticket.id, job = %job_id, "Resume: escalating with 2 additional analysts");
} else {
info!(ticket = %ticket.id, "All analysts flagged blockers — escalating with 2 additional analysts");
}
let extra_slots = match append_ticket_stage_slots(ticket, job_id, Role::Analyst, task, 2)
.await
{
Ok(s) => s,
Err(e) => {
if resume {
warn!(job = %job_id, error = %e, "Resume: escalation slot append failed — proceeding");
} else {
warn!(ticket = %ticket.id, error = %e, "Failed to append escalation slots — proceeding with base round");
}
Vec::new()
}
};
let extra = if extra_slots.is_empty() {
Vec::new()
} else {
run_parallel_agents(
ticket,
ws,
Role::Analyst,
extraction_prompt,
job_id,
&extra_slots,
resume,
)
.await
};
results.extend(extra);
}
true
}
async fn dispatch_backlog_analysts(ticket: Arc<Ticket>, ws: Workspace) {
let prompt_key = if ticket.reporter == Role::Maintainer.as_str() {
"analyze/maintainer_ticket.md"
} else {
"analyze/manager_ticket.md"
};
let message = load_prompt(prompt_key);
let Ok((job_id, slots)) = spawn_ticket_stage_round(
&ticket,
&ws,
"analysis",
TicketPhase::Analysis,
Role::Analyst,
&message,
DEFAULT_PARALLEL_AGENT_COUNT,
)
.await
else {
error!(
ticket = %ticket.id,
"Failed to spawn analysis job — aborting dispatch",
);
return;
};
run_analysis_round(&ticket, &ws, &job_id, &slots, &message, false).await;
}
async fn run_analysis_round(
ticket: &Arc<Ticket>,
ws: &Workspace,
job_id: &str,
slots: &[TicketStageSlot],
task: &str,
resumed: bool,
) {
let extraction_prompt = load_prompt("extraction/analyst.md");
let mut results = run_parallel_agents(
ticket,
ws,
Role::Analyst,
&extraction_prompt,
job_id,
slots,
resumed,
)
.await;
if !maybe_escalate_analysis(
ticket,
ws,
job_id,
&extraction_prompt,
task,
resumed,
&mut results,
)
.await
{
return;
}
finalize_analysis_round(ws, ticket, &results, job_id).await;
}
async fn process_analyst_verdicts(ws: &Workspace, ticket: &Ticket, results: &[ParallelVerdict]) {
let nonempty_count = results
.iter()
.filter(|r| !matches!(r, ParallelVerdict::NoResponse(_)))
.count();
let dispatched = results.len();
let mut lgtm = 0usize;
let mut minor_issues = 0usize;
let mut potential_blockers = 0usize;
let mut missing_analysis = 0usize;
for r in results {
match r {
ParallelVerdict::Verdict(v)
if v.score >= ANALYST_PASS_THRESHOLD && v.issues_detected.is_empty() =>
{
lgtm += 1;
}
ParallelVerdict::Verdict(v) if v.score >= ANALYST_PASS_THRESHOLD => minor_issues += 1,
ParallelVerdict::Verdict(_) => potential_blockers += 1,
ParallelVerdict::NoResponse(_) | ParallelVerdict::ParseFailed(_) => {
missing_analysis += 1;
}
}
}
let summary = format_analyst_summary(
dispatched,
lgtm,
minor_issues,
potential_blockers,
missing_analysis,
);
let extracted_count = dispatched - missing_analysis;
let passing_count = lgtm + minor_issues;
let all_passed = passing_count == dispatched;
if crate::shutdown::aborting() {
return;
}
let joint_comment = build_round_joint_comment(
stage_name(Role::Analyst),
results,
ANALYST_PASS_THRESHOLD,
Role::Analyst,
&summary,
ws,
&ticket.id,
&ticket.title,
)
.await;
if !matches!(
with_comment_and_transition(
TransitionCtx {
ticket,
source: TicketPhase::Analysis,
target: TicketPhase::Planning,
notify: NotifyPolicy::Notify,
log_label: "Analyst",
breaker_trip: false,
},
async |tx| {
BoardStore::add_comment_tx(
tx,
&ticket.id,
stage_name(Role::Analyst),
&joint_comment,
)
.await?;
Ok(())
},
)
.await,
FinalizeOutcome::Applied
) {
return;
}
if all_passed {
info!(
ticket = %ticket.id,
nonempty_count,
"Backlog analysis complete — all analysts passed (≥ {ANALYST_PASS_THRESHOLD}/10)",
);
} else {
info!(
ticket = %ticket.id,
nonempty_count,
extracted_count,
passing_count,
"Backlog analysis incomplete — moved to planning ({nonempty_count}/{dispatched} responded, \
{extracted_count} extracted, {passing_count} passed)",
);
}
}
async fn finalize_analysis_round(
ws: &Workspace,
ticket: &Ticket,
results: &[ParallelVerdict],
job_id: &str,
) {
if complete_job_and_bail_if_phase_moved(&ticket.id, TicketPhase::Analysis, job_id).await {
return;
}
if crate::shutdown::aborting() {
return;
}
process_analyst_verdicts(ws, ticket, results).await;
if crate::shutdown::aborting() {
return;
}
complete_ticket_stage_job(job_id).await;
}
fn format_analyst_summary(
total: usize,
lgtm: usize,
minor_issues: usize,
potential_blockers: usize,
missing_analysis: usize,
) -> String {
let description = [
(lgtm, "LGTM"),
(minor_issues, "found minor issues"),
(potential_blockers, "flagged potential blockers"),
(missing_analysis, "provided no analysis"),
]
.iter()
.filter(|&&(count, _label)| count > 0)
.map(|&(count, label)| {
if count == total {
format!("All {label}")
} else {
format!("{count} {label}")
}
})
.collect::<Vec<_>>()
.join(", ");
format!("{total} analysts reviewed this ticket. {description}.")
}
async fn drain_ready_for_development_siblings(ticket: &Ticket) {
match board()
.drain_ready_for_development_to_planning(&ticket.workspace_name)
.await
{
Ok(updated) if updated > 0 => {
info!(
tickets = updated,
workspace = %ticket.workspace_name,
"Moved {updated} ReadyForDevelopment ticket(s) to Planning after circuit breaker trip",
);
}
Ok(_) => {
debug!(
workspace = %ticket.workspace_name,
"No ReadyForDevelopment siblings to drain after circuit breaker trip",
);
}
Err(e) => {
warn!(
ticket = %ticket.id,
workspace = %ticket.workspace_name,
error = %e,
"Failed to move ReadyForDevelopment tickets to Planning \
— breaker trip proceeds without moving siblings",
);
}
}
}
#[must_use]
async fn try_trip_circuit_breaker(
ticket: &Ticket,
source_phase: TicketPhase,
kind: Option<CircuitBreakerKind>,
log_label: &str,
) -> bool {
let Some(kind) = kind else {
return false;
};
let comments = if ticket.comments.is_empty() {
match board().get_comments(&ticket.id).await {
Ok(c) => std::borrow::Cow::Owned(c),
Err(e) => {
warn!(
ticket = %ticket.id,
error = %e,
"Failed to fetch comments for circuit breaker — proceeding anyway"
);
return false;
}
}
} else {
std::borrow::Cow::Borrowed(ticket.comments.as_slice())
};
let Some((count, max_count)) = kind.should_trip(&comments) else {
return false;
};
let msg = kind.trip_message(count, max_count);
info!(
ticket = %ticket.id,
count,
max_count,
log_label,
"Circuit breaker tripped at {count}/{max_count} ({log_label}) — failing ticket"
);
let breaker_label = format!("{log_label} circuit breaker");
match comment_and_transition(
TransitionCtx {
ticket,
source: source_phase,
target: TicketPhase::Failed,
notify: NotifyPolicy::Notify,
log_label: &breaker_label,
breaker_trip: true,
},
SYSTEM_ROLE,
&msg,
)
.await
{
FinalizeOutcome::Applied | FinalizeOutcome::Moved => {}
FinalizeOutcome::Failed => {
error!(
ticket = %ticket.id,
source_phase = %source_phase,
log_label = %breaker_label,
"Circuit breaker transition to Failed failed — ticket may loop indefinitely",
);
}
}
true
}
fn verifier_failure_reasons(results: &[ParallelVerdict]) -> String {
let per_agent_cap = crate::util::FAILURE_DETAIL_CAP / results.len().max(1);
let reasons: Vec<String> = results
.iter()
.enumerate()
.filter_map(|(i, r)| match r {
ParallelVerdict::NoResponse(reason) => Some(format!("{}. {reason}", i + 1)),
ParallelVerdict::ParseFailed(f) => Some(format!(
"{}. verdict extraction failed: {}",
i + 1,
crate::util::truncate_sandwich(
&raw_response_dump_section(f),
per_agent_cap,
"agent failure",
)
)),
ParallelVerdict::Verdict(_) => None,
})
.collect();
format!("\n\nPer-agent failures:\n{}", reasons.join("\n"))
}
#[must_use]
fn bounce_breaker_trip_comment() -> String {
let max = crate::joint_verdict::MAX_BOUNCES;
format!(
"Failed after {max} bounces — ticket bounced back too many times \
(circuit breaker, max: {max}). Ticket failed — Manager will triage."
)
}
#[expect(clippy::too_many_lines)]
async fn process_verifier_verdicts(
ws: &Workspace,
ticket: &Ticket,
results: &[ParallelVerdict],
verifier: VerifierInfo,
) -> bool {
let all_failed = results
.iter()
.all(|r| !matches!(r, ParallelVerdict::Verdict(_)));
let any_failed = results.iter().any(|r| match r {
ParallelVerdict::Verdict(v) => !verdict_passes(v),
_ => true,
});
let mut bounce_trip = false;
let (target, notify) = if all_failed {
(TicketPhase::Failed, NotifyPolicy::Notify)
} else if any_failed {
if usize::try_from(ticket.bounce_count).unwrap_or(usize::MAX)
>= crate::joint_verdict::MAX_BOUNCES
{
bounce_trip = true;
(TicketPhase::Failed, NotifyPolicy::Notify)
} else {
(TicketPhase::ReadyForDevelopment, NotifyPolicy::Buffer)
}
} else {
(verifier.success_phase, NotifyPolicy::Buffer)
};
if crate::shutdown::aborting() {
info!(
ticket = %ticket.id,
stage = %verifier.log_label,
"Verifier round cut short by drain — job stays launched for boot resume",
);
return false;
}
let (failure_comment, reasons) = if all_failed {
let pause_note =
pause_workspace_on_failure(ticket, "all verifier agents failed to produce verdicts")
.await;
let reasons = verifier_failure_reasons(results);
let header = substitute(
&load_prompt("pipeline/verifiers_all_failed.md"),
&[("{{agent_type}}", verifier.log_label)],
);
let body = format!("{reasons}{pause_note}");
let failure_comment = crate::util::truncate_sandwich(
&crate::util::scrub_credentials(&format!("{header}\n{body}")),
crate::util::FAILURE_DETAIL_CAP,
"verifier failure",
);
(failure_comment, reasons)
} else {
(String::new(), String::new())
};
let joint_comment = if all_failed {
None
} else {
Some(
build_round_joint_comment(
stage_name(verifier.role),
results,
REVIEW_QA_THRESHOLD,
verifier.role,
"",
ws,
&ticket.id,
&ticket.title,
)
.await,
)
};
let bounce_breaker_comment = bounce_trip.then(bounce_breaker_trip_comment);
if !matches!(
with_comment_and_transition(
TransitionCtx {
ticket,
source: verifier.active_phase,
target,
notify,
log_label: verifier.log_label,
breaker_trip: bounce_trip,
},
async |tx| {
if let Some(comment) = &joint_comment {
BoardStore::add_comment_tx(tx, &ticket.id, stage_name(verifier.role), comment)
.await?;
}
if let Some(comment) = &bounce_breaker_comment {
BoardStore::add_comment_tx(tx, &ticket.id, SYSTEM_ROLE, comment).await?;
}
if all_failed {
BoardStore::add_comment_tx(tx, &ticket.id, SYSTEM_ROLE, &failure_comment)
.await?;
}
if target == TicketPhase::ReadyForDevelopment {
BoardStore::increment_bounce_count_tx(tx, &ticket.id).await?;
}
Ok(())
},
)
.await,
FinalizeOutcome::Applied
) {
return false;
}
if bounce_trip {
drain_ready_for_development_siblings(ticket).await;
}
if all_failed {
info!(
ticket = %ticket.id,
reasons = %crate::util::truncate_sandwich(
&crate::util::scrub_credentials(&reasons),
crate::util::FAILURE_DETAIL_CAP,
"verifier failure",
),
"{log_label}: all verifier agents failed to produce verdicts — ticket moved to Failed",
log_label = verifier.log_label,
);
} else if bounce_trip {
info!(
ticket = %ticket.id,
"Bounce circuit breaker tripped ({MAX_BOUNCES} bounces) — ticket failed",
MAX_BOUNCES = crate::joint_verdict::MAX_BOUNCES,
);
} else if any_failed {
info!(
ticket = %ticket.id,
"{log_label} failed — pipeline reservation set for rework priority",
log_label = verifier.log_label,
);
} else {
info!(
ticket = %ticket.id,
"{log_label}: all passed (≥ {REVIEW_QA_THRESHOLD}/10)",
log_label = verifier.log_label,
);
}
true
}
#[must_use]
fn should_skip_review(
reviewed_head: Option<&str>,
reviewed_tree: Option<&str>,
current_head: Option<&str>,
current_tree: Option<&str>,
porcelain: &str,
) -> bool {
let (Some(base_head), Some(base_tree)) = (reviewed_head, reviewed_tree) else {
return false;
};
let (Some(head), Some(tree)) = (current_head, current_tree) else {
return false;
};
head == base_head && tree == base_tree && !has_unstaged_changes(porcelain)
}
async fn compute_review_skip(ticket: &Ticket, repo_path: &Path) -> anyhow::Result<bool> {
let porcelain = run_git_status(repo_path).await?;
let head = run_git_head(repo_path).await.ok();
let tree = run_git_write_tree(repo_path).await.ok();
if (head.is_none() || tree.is_none()) && ticket.reviewed_head.is_some() {
warn!(
ticket = %ticket.id,
head = head.is_some(),
tree = tree.is_some(),
"Could not compute full content identity — running full review",
);
}
Ok(should_skip_review(
ticket.reviewed_head.as_deref(),
ticket.reviewed_tree.as_deref(),
head.as_deref(),
tree.as_deref(),
&porcelain,
))
}
async fn working_tree_churn(repo_path: &Path) -> anyhow::Result<i64> {
let (added, removed) = run_git_diff_stats(repo_path).await?;
Ok(added + removed)
}
async fn compute_reviewer_count(ticket: &Ticket, repo_path: &Path) -> usize {
let low =
i64::try_from(crate::joint_verdict::DEFAULT_REVIEW_COUNT_LOW_CHURN).unwrap_or(i64::MAX);
let high =
i64::try_from(crate::joint_verdict::DEFAULT_REVIEW_COUNT_HIGH_CHURN).unwrap_or(i64::MAX);
match working_tree_churn(repo_path).await {
Ok(total) => {
let base = crate::joint_verdict::review_base_from_signals(total, low, high);
info!(
ticket = %ticket.id,
total_churn = total,
reviewer_base = base,
"Reviewer count calibration: base {base} from total churn",
);
crate::joint_verdict::review_agent_count(base, ticket.priority)
}
Err(e) => {
warn!(
ticket = %ticket.id,
error = %e,
"Could not compute working-tree churn — reviewer base defaults to 3",
);
3
}
}
}
async fn resume_ticket_stage_round(stage: String, job_id: String, ticket: Ticket, ws: Workspace) {
match stage.as_str() {
"analysis" => resume_analysis_round(&job_id, ticket, ws).await,
"review" => resume_verifier_round(&job_id, ticket, ws, REVIEWER_VI).await,
"qa" => resume_verifier_round(&job_id, ticket, ws, QA_VI).await,
"engineer" => resume_stage_round(&job_id, ticket, ws, StageRoundKind::Engineer).await,
"sanitation" => resume_stage_round(&job_id, ticket, ws, StageRoundKind::Sanitation).await,
other => {
warn!(stage = %other, job = %job_id, "Unknown ticket_stage on resume — completing job");
complete_ticket_stage_job(&job_id).await;
}
}
}
async fn resume_analysis_round(job_id: &str, ticket: Ticket, ws: Workspace) {
if complete_job_and_bail_if_phase_moved(&ticket.id, TicketPhase::Analysis, job_id).await {
return;
}
let Ok(slots) = load_ticket_stage_slots(job_id).await else {
complete_ticket_stage_job(job_id).await;
return;
};
let ticket_arc = Arc::new(ticket);
let task = job_task(job_id).await;
run_analysis_round(&ticket_arc, &ws, job_id, &slots, &task, true).await;
}
async fn record_reviewed_base_after_review(
repo_path: &Path,
ticket_id: &str,
git_available: bool,
transitioned: bool,
results: &[ParallelVerdict],
log_prefix: &str,
) {
let reviewed = results
.iter()
.any(|r| matches!(r, ParallelVerdict::Verdict(_)));
if git_available && transitioned && reviewed {
if let Err(e) = run_git_add_all(repo_path).await {
warn!(
ticket = %ticket_id,
error = %e,
"{log_prefix}Failed to stage changes after review — reviewed base not recorded",
);
} else {
let head = run_git_head(repo_path).await.ok();
let tree = run_git_write_tree(repo_path).await.ok();
if head.is_none() || tree.is_none() {
warn!(
ticket = %ticket_id,
head = head.is_some(),
tree = tree.is_some(),
"{log_prefix}Could not compute content identity after review — reviewed base not recorded",
);
} else if let Err(e) = board()
.set_reviewed_base(ticket_id, head.as_deref(), tree.as_deref())
.await
{
warn!(
ticket = %ticket_id,
error = %e,
"{log_prefix}Failed to record reviewed base — later rounds will re-review",
);
} else {
debug!(ticket = %ticket_id, "{log_prefix}Recorded reviewed base after review");
}
}
}
}
async fn resume_verifier_round(job_id: &str, ticket: Ticket, ws: Workspace, vi: VerifierInfo) {
if complete_job_and_bail_if_phase_moved(&ticket.id, vi.active_phase, job_id).await {
return;
}
let Ok(slots) = load_ticket_stage_slots(job_id).await else {
complete_ticket_stage_job(job_id).await;
return;
};
let extraction_prompt = load_prompt(vi.extraction_prompt_path);
let ticket_arc = Arc::new(ticket);
let results = run_parallel_agents(
&ticket_arc,
&ws,
vi.role,
&extraction_prompt,
job_id,
&slots,
true,
)
.await;
if complete_job_and_bail_if_phase_moved(&ticket_arc.id, vi.active_phase, job_id).await {
return;
}
if crate::shutdown::aborting() {
return;
}
let (_is_reviewer, _repo_path, git_available) = verifier_git_state(&ws, vi).await;
finalize_verifier_round(&ws, &ticket_arc, vi, &results, job_id, true, git_available).await;
}
async fn resume_stage_round(job_id: &str, ticket: Ticket, ws: Workspace, kind: StageRoundKind) {
let phase = match kind {
StageRoundKind::Engineer => TicketPhase::InDevelopment,
StageRoundKind::Sanitation => TicketPhase::InSanitation,
};
if complete_job_and_bail_if_phase_moved(&ticket.id, phase, job_id).await {
return;
}
let task = job_task(job_id).await;
run_stage_agent_round(&ticket, &ws, job_id, &task, true, kind).await;
}
async fn job_task(job_id: &str) -> String {
crate::session::store()
.conn
.query_optional(
"SELECT task FROM jobs WHERE id = ?1",
crate::turso::params![job_id],
|row| row.get::<String>(0),
)
.await
.ok()
.flatten()
.unwrap_or_default()
}
async fn finalize_verifier_round(
ws: &Workspace,
ticket: &Ticket,
vi: VerifierInfo,
results: &[ParallelVerdict],
job_id: &str,
resumed: bool,
git_available: bool,
) {
let transitioned = process_verifier_verdicts(ws, ticket, results, vi).await;
if !transitioned && crate::shutdown::aborting() {
let cut_short = if resumed {
"Resumed verifier round cut short by drain — job stays launched for boot resume"
} else {
"Verifier round cut short by drain — job stays launched for boot resume"
};
info!(ticket = %ticket.id, "{cut_short}");
return;
}
record_reviewed_base_after_review(
ws.as_path(),
&ticket.id,
git_available,
transitioned,
results,
if resumed { "Resume: " } else { "" },
)
.await;
complete_ticket_stage_job(job_id).await;
}
async fn verifier_git_state(ws: &Workspace, vi: VerifierInfo) -> (bool, &Path, bool) {
let is_reviewer = vi.role == Role::Reviewer;
let repo_path = ws.as_path();
let git_available = is_reviewer
&& crate::git_commands::git_is_installed().await
&& crate::git_commands::is_git_repo(repo_path);
(is_reviewer, repo_path, git_available)
}
async fn dispatch_verifiers(ticket: Arc<Ticket>, ws: Workspace, vi: VerifierInfo) {
let (is_reviewer, repo_path, git_available) = verifier_git_state(&ws, vi).await;
if git_available {
match compute_review_skip(&ticket, repo_path).await {
Ok(true) => {
info!(
ticket = %ticket.id,
"Content identical to reviewed base — skipping reviewer dispatch",
);
let _ = comment_and_transition(
TransitionCtx {
ticket: &ticket,
source: vi.active_phase,
target: TicketPhase::Reviewed,
notify: NotifyPolicy::Buffer,
log_label: vi.log_label,
breaker_trip: false,
},
SYSTEM_ROLE,
"Content is identical to the reviewed base recorded for this ticket \
(same HEAD commit and index tree, no working-tree changes). \
Skipping reviewer dispatch.",
)
.await;
return;
}
Ok(false) => {}
Err(e) => {
warn!(
ticket = %ticket.id,
error = %e,
"Git status check failed for skip-review — proceeding with normal review",
);
}
}
}
let engineer_response = ticket
.comments
.iter()
.rev()
.find(|c| c.role == Role::Engineer.as_str())
.map(|c| &c.content)
.map_or("(no output)", String::as_str);
let prompt = substitute(
&crate::prompt::load_prompt(vi.prompt_template),
&[("{{agent_response}}", engineer_response)],
);
let extraction_prompt = crate::prompt::load_prompt(vi.extraction_prompt_path);
let count = if is_reviewer {
compute_reviewer_count(&ticket, repo_path).await
} else {
QA_PARALLEL_AGENT_COUNT
};
let verifier_label = if count == 1 { "verifier" } else { "verifiers" };
info!(
ticket = %ticket.id,
role = %vi.role.as_str(),
count,
verifier_label,
"Dispatching {count} parallel {verifier_label}",
);
let stage = if is_reviewer { "review" } else { "qa" };
let Ok((job_id, slots)) = spawn_ticket_stage_round(
&ticket,
&ws,
stage,
vi.active_phase,
vi.role,
&prompt,
count,
)
.await
else {
error!(
ticket = %ticket.id,
"Failed to spawn verifier job — aborting dispatch",
);
return;
};
let results = run_parallel_agents(
&ticket,
&ws,
vi.role,
&extraction_prompt,
&job_id,
&slots,
false,
)
.await;
if complete_job_and_bail_if_phase_moved(&ticket.id, vi.active_phase, &job_id).await {
return;
}
finalize_verifier_round(&ws, &ticket, vi, &results, &job_id, false, git_available).await;
}
#[cfg(test)]
#[path = "management_tests.rs"]
mod tests;