use std::fmt::Write;
use std::path::Path;
use std::sync::Arc;
use std::time::Duration;
use tracing::{debug, error, info, warn};
use futures_util::FutureExt;
use futures_util::future::join_all;
use crate::agent::run_agent;
use crate::board::{BOARD, BoardStore, PipelineCheck, Ticket, TicketComment, TicketPhase};
use crate::git_commands::{
list_new_or_untracked_files, parse_new_files_from_porcelain, run_git_status,
};
use crate::manager_queue::{JobKind, ManagerJob};
use crate::prompt::{load_prompt, substitute};
use crate::role::{DIAGNOSTICS_ROLE, SYSTEM_ROLE};
use crate::session::ticket_session_key;
use crate::ticket_buffer;
use crate::tools::shell::{ShellMode, ShellTool};
use crate::turso::TxGuard;
use crate::util::panic_message;
use crate::{DiagnosticsCommands, Role, Workspace};
const PARALLEL_AGENT_COUNT: usize = 3;
const DIAGNOSTICS_COMMENT_PREFIX: &str = "🔍 Auto-diagnostics";
const DIAGNOSTICS_PASSED_MARKER: &str = "✅ All diagnostics passed";
const DIAGNOSTICS_FAILED_MARKER: &str = "❌ Diagnostics failed at";
const SANITATION_FAILED_MARKER: &str = "Sanitation failed";
const ANALYST_PASS_THRESHOLD: u8 = 7;
const REVIEW_QA_THRESHOLD: u8 = 9;
#[inline]
fn board() -> &'static BoardStore {
crate::board::store()
}
async fn clear_assigned_to(ticket_id: &str, context: &str) {
if let Err(e) = board().clear_assigned_to_no_cancel(ticket_id).await {
warn!(
ticket = %ticket_id,
error = %e,
"Failed to clear assigned_to: {context}",
);
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, strum::EnumIter)]
enum CircuitBreakerKind {
General,
Sanitation,
Diagnostics,
}
impl CircuitBreakerKind {
const fn threshold(self) -> usize {
match self {
Self::General => 30,
Self::Sanitation => 3,
Self::Diagnostics => 4,
}
}
fn should_trip(self, comments: &[TicketComment]) -> Option<(usize, usize, String)> {
let threshold = self.threshold();
let (count, msg) = match self {
Self::General => {
let count = comments.len();
(
count,
format!(
"Failed after {count} comments — ticket has accumulated too many comments \
(circuit breaker, threshold: {threshold}). \
Ticket failed — Manager will triage."
),
)
}
Self::Sanitation => {
let count =
count_matching_comments(comments, SYSTEM_ROLE, SANITATION_FAILED_MARKER);
(
count,
format!(
"❌ Sanitation circuit breaker tripped after {count} cumulative failures. \
(threshold: {threshold})",
),
)
}
Self::Diagnostics => {
let count =
count_matching_comments(comments, DIAGNOSTICS_ROLE, DIAGNOSTICS_FAILED_MARKER);
(
count,
format!(
"{DIAGNOSTICS_COMMENT_PREFIX}\n\n❌ Circuit breaker: {count} prior diagnostic \
failures. Failing ticket."
),
)
}
};
if count <= threshold {
None
} else {
Some((count, threshold, msg))
}
}
}
fn count_matching_comments(comments: &[TicketComment], role: &str, marker: &str) -> usize {
comments
.iter()
.filter(|c| c.role == role && c.content.contains(marker))
.count()
}
#[must_use]
async fn is_ticket_in_phase(ticket_id: &str, expected_phase: TicketPhase) -> bool {
match board().get_ticket_phase(ticket_id).await {
Ok(Some(phase)) => {
let ok = phase == expected_phase;
if !ok {
debug!(
ticket = %ticket_id,
expected_phase = %expected_phase,
actual = %phase,
"Ticket moved externally — bailing out",
);
}
ok
}
Ok(None) => {
debug!(ticket = %ticket_id, "Ticket not found — row missing (violates architecture invariant)");
false
}
Err(e) => {
warn!(ticket = %ticket_id, error = %e, "Failed to check ticket phase");
false
}
}
}
#[must_use]
async fn guard_ticket_in_phase(ticket_id: &str, expected: TicketPhase) -> bool {
if !is_ticket_in_phase(ticket_id, expected).await {
clear_assigned_to(ticket_id, &format!("ticket left {expected:?}")).await;
return false;
}
true
}
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
enum NotifyPolicy {
Notify,
Buffer,
}
async fn dispatch_notification(
ticket: &Ticket,
source: TicketPhase,
target: TicketPhase,
notify: NotifyPolicy,
) {
match notify {
NotifyPolicy::Notify => notify_ticket(ticket, target).await,
NotifyPolicy::Buffer => {
ticket_buffer::push(&ticket.workspace_name, &ticket.id, source, target);
}
}
}
#[derive(Debug)]
struct TransitionCtx<'a> {
ticket: &'a Ticket,
source: TicketPhase,
target: TicketPhase,
notify: NotifyPolicy,
log_label: &'a str,
}
#[must_use]
async fn with_comment_and_transition<F>(args: TransitionCtx<'_>, write_comments: F) -> bool
where
F: AsyncFnOnce(&TxGuard<'_>) -> anyhow::Result<()>,
{
let pipeline_reservation = (args.target == TicketPhase::ReadyForDevelopment).then_some(true);
if let Err(e) = crate::turso::with_tx(
&board().conn,
&args.ticket.id,
args.log_label,
async move |tx| {
write_comments(tx).await?;
BoardStore::transition_to_tx(
tx,
&args.ticket.id,
Some(args.source),
args.target,
pipeline_reservation,
)
.await?;
Ok(())
},
)
.await
{
warn!(
ticket = %args.ticket.id,
error = %e,
"{}: transition to {} failed — ticket stuck in {}",
args.log_label, args.target, args.source,
);
clear_assigned_to(&args.ticket.id, args.log_label).await;
return false;
}
dispatch_notification(args.ticket, args.source, args.target, args.notify).await;
true
}
#[must_use]
async fn comment_and_transition(ctx: TransitionCtx<'_>, comment: (&str, &str)) -> bool {
let ticket = ctx.ticket;
with_comment_and_transition(ctx, async |tx| {
let (role, text) = comment;
BoardStore::add_comment_tx(tx, &ticket.id, role, text).await?;
Ok(())
})
.await
}
#[must_use]
async fn resolve_ticket_workspace(ticket: &Ticket, log_label: &str) -> Option<crate::Workspace> {
match crate::workspace::get_by_name(&ticket.workspace_name).await {
Ok(Some(ws)) => Some(ws),
Ok(None) => {
warn!(
ticket = %ticket.id,
workspace_name = %ticket.workspace_name,
"Workspace not found for ticket — {log_label}",
);
None
}
Err(e) => {
warn!(
ticket = %ticket.id,
workspace_name = %ticket.workspace_name,
error = %e,
"Failed to look up workspace for ticket — {log_label}",
);
None
}
}
}
async fn notify_ticket(ticket: &Ticket, target_phase: TicketPhase) {
let Some(ws) = resolve_ticket_workspace(ticket, "skipping notification").await else {
error!(
ticket = %ticket.id,
workspace_name = %ticket.workspace_name,
"Workspace resolution failed — notification skipped"
);
return;
};
let transition_log = format!(
"[{}] {}: {} → {}",
ticket.reporter,
ticket.id,
ticket.phase,
target_phase.as_ref()
);
let drained = crate::ticket_buffer::drain(&ws.name);
let mut message = substitute(
&load_prompt("notification.md"),
&[
("{{ticket_id}}", &ticket.id),
("{{ticket_title}}", &ticket.title),
("{{ticket_phase}}", target_phase.as_ref()),
("{{transition_log}}", &transition_log),
("{{ticket_updates}}", &drained),
],
);
if target_phase == TicketPhase::Failed {
let failure_details: String = match board().get_comments(&ticket.id).await {
Ok(comments) => comments.last().map_or_else(
|| "(unknown failure reason)".to_string(),
|c| c.content.clone(),
),
Err(_) => "(unknown failure reason)".to_string(),
};
let warning = substitute(
&load_prompt("warning.md"),
&[("{{failure_details}}", &failure_details)],
);
message.push_str("\n\n");
message.push_str(&warning);
}
crate::manager_queue::manager_queue().enqueue(ManagerJob {
content: message,
workspace_name: ws.name,
kind: JobKind::TicketNotify,
});
}
pub async fn run_management() {
if let Some(board) = BOARD.get()
&& let Err(e) = board.reset_inflight_tickets().await
{
error!(error = %e, "Failed to reset in-flight tickets");
}
let interval = Duration::from_secs(1);
loop {
if !crate::shutdown::sleep_or_shutdown(interval).await {
break;
}
if let Err(e) = poll_round().await {
error!(error = %e, "Board poller round failed");
}
}
}
fn spawn_dispatch(phase: PollPhase, ticket: Ticket, ws: Workspace) {
let phase_info = phase.info();
let expected_phase = phase_info.expected_phase;
let kind = phase_info.circuit_breaker_kind;
let log_label = phase_info.log_label;
info!(
ticket = %ticket.id,
title = %ticket.title,
workspace = %ws.name,
"Dispatching {} ticket",
phase_info.log_label,
);
crate::registry::AGENT_REGISTRY.cancel_by_ticket_id(&ticket.id);
let ticket = Arc::new(ticket);
let ticket_for_failure = Arc::clone(&ticket);
tokio::spawn(async move {
if !is_ticket_in_phase(&ticket.id, expected_phase).await {
return;
}
if try_trip_circuit_breaker(&ticket, expected_phase, kind, log_label).await {
return;
}
let result = std::panic::AssertUnwindSafe(async move {
match phase {
PollPhase::BacklogAnalysis => dispatch_backlog_analysts(ticket, ws).await,
PollPhase::EngineerDevelopment => dispatch_engineer(ticket, ws).await,
PollPhase::SanitationCheck => dispatch_sanitation(ticket, ws).await,
PollPhase::DiagnosticsCheck => dispatch_diagnostics(ticket, ws).await,
PollPhase::VerifierCheck(vi) => dispatch_verifiers(ticket, ws, vi).await,
}
})
.catch_unwind()
.await;
if let Err(payload) = result {
let msg = panic_message(&*payload);
error!(
ticket = %ticket_for_failure.id,
panic = %msg,
"Dispatch panicked — transitioning ticket to Failed",
);
let _ = comment_and_transition(
TransitionCtx {
ticket: &ticket_for_failure,
source: expected_phase,
target: TicketPhase::Failed,
notify: NotifyPolicy::Notify,
log_label: "dispatch panic",
},
(SYSTEM_ROLE, &format!("❌ Dispatch panicked: {msg}")),
)
.await;
}
});
}
#[derive(Copy, Clone)]
struct VerifierInfo {
role: Role,
log_label: &'static str,
success_phase: TicketPhase,
source_phase: TicketPhase,
prompt_template: &'static str,
extraction_prompt_path: &'static str,
}
const REVIEWER_VI: VerifierInfo = VerifierInfo {
role: Role::Reviewer,
log_label: "Reviewers",
success_phase: TicketPhase::Reviewed,
source_phase: TicketPhase::InReview,
prompt_template: "review.md",
extraction_prompt_path: "extraction/reviewer.md",
};
const QA_VI: VerifierInfo = VerifierInfo {
role: Role::Qa,
log_label: "QA",
success_phase: TicketPhase::QaPassed,
source_phase: TicketPhase::InQa,
prompt_template: "qa.md",
extraction_prompt_path: "extraction/qa.md",
};
#[derive(Copy, Clone)]
struct PollPhaseInfo {
expected_phase: TicketPhase,
pipeline_check: PipelineCheck,
circuit_breaker_kind: CircuitBreakerKind,
log_label: &'static str,
}
impl PollPhaseInfo {
const fn new(expected_phase: TicketPhase, log_label: &'static str) -> Self {
Self {
expected_phase,
pipeline_check: PipelineCheck::Skip,
circuit_breaker_kind: CircuitBreakerKind::General,
log_label,
}
}
}
#[derive(Copy, Clone)]
enum PollPhase {
BacklogAnalysis,
EngineerDevelopment,
SanitationCheck,
DiagnosticsCheck,
VerifierCheck(VerifierInfo),
}
impl PollPhase {
fn info(self) -> PollPhaseInfo {
match self {
Self::BacklogAnalysis => PollPhaseInfo::new(TicketPhase::Analysis, "Analyst"),
Self::EngineerDevelopment => PollPhaseInfo {
pipeline_check: PipelineCheck::Enforce,
..PollPhaseInfo::new(TicketPhase::InDevelopment, "Engineer")
},
Self::SanitationCheck => PollPhaseInfo {
circuit_breaker_kind: CircuitBreakerKind::Sanitation,
..PollPhaseInfo::new(TicketPhase::InSanitation, "Sanitation")
},
Self::DiagnosticsCheck => PollPhaseInfo {
circuit_breaker_kind: CircuitBreakerKind::Diagnostics,
..PollPhaseInfo::new(TicketPhase::InDiagnostics, "Diagnostics")
},
Self::VerifierCheck(vi) => PollPhaseInfo::new(vi.source_phase, vi.log_label),
}
}
}
const CLAIM_PHASES: &[(TicketPhase, PollPhase)] = &[
(TicketPhase::Backlog, PollPhase::BacklogAnalysis),
(
TicketPhase::ReadyForDevelopment,
PollPhase::EngineerDevelopment,
),
(
TicketPhase::DiagnosticsDone,
PollPhase::VerifierCheck(REVIEWER_VI),
),
(TicketPhase::Reviewed, PollPhase::VerifierCheck(QA_VI)),
];
async fn for_tickets_in_phase(phase: TicketPhase, workspace_name: &str, action: impl Fn(Ticket)) {
match board()
.list_all_tickets(Some(workspace_name), Some(phase))
.await
{
Ok(tickets) => {
for ticket in tickets {
action(ticket);
}
}
Err(e) => {
error!(workspace = workspace_name, phase = %phase, error = %e, "Phase listing failed");
}
}
}
async fn spawn_for_each_ticket_in_phase<F, Fut>(phase: TicketPhase, ws: &Workspace, f: F)
where
F: Fn(Ticket, Workspace) -> Fut + Clone + Send + 'static,
Fut: Future<Output = ()> + Send + 'static,
{
for_tickets_in_phase(phase, &ws.name, |ticket| {
let f = f.clone();
let ws = ws.clone();
tokio::spawn(async move {
f(ticket, ws).await;
});
})
.await;
}
async fn dispatch_unassigned_in_phase(
phase: TicketPhase,
dispatch_phase: PollPhase,
ws: &Workspace,
) {
for_tickets_in_phase(phase, &ws.name, |ticket| {
if ticket.assigned_to.is_some() {
return;
}
spawn_dispatch(dispatch_phase, ticket, ws.clone());
})
.await;
}
async fn poll_round() -> anyhow::Result<()> {
let board = board();
let workspaces = match crate::workspace::store().list().await {
Ok(ws_list) => ws_list,
Err(e) => {
error!(error = %e, "Failed to list workspaces");
return Ok(());
}
};
for ws in &workspaces {
for &(source, phase) in CLAIM_PHASES {
if ws.paused && matches!(phase, PollPhase::EngineerDevelopment) {
continue;
}
let info = phase.info();
let ticket = match board
.claim_ticket_in_workspace(
source,
info.expected_phase,
&ws.name,
info.pipeline_check,
)
.await
{
Ok(Some(t)) => {
ticket_buffer::push(&ws.name, &t.id, source, t.phase);
t
}
Ok(None) => continue,
Err(e) => {
error!(
workspace = %ws.name,
phase = %info.log_label,
error = %e,
"Claim failed, skipping remaining claim phases for workspace",
);
break;
}
};
spawn_dispatch(phase, ticket, ws.clone());
}
dispatch_unassigned_in_phase(TicketPhase::InDiagnostics, PollPhase::DiagnosticsCheck, ws)
.await;
spawn_for_each_ticket_in_phase(TicketPhase::SanitationPassed, ws, |ticket, ws| {
finalize_ticket_from_phase(ticket, ws, TicketPhase::SanitationPassed)
})
.await;
spawn_for_each_ticket_in_phase(TicketPhase::QaPassed, ws, |ticket, ws| {
handle_qa_passed(ticket, ws)
})
.await;
dispatch_unassigned_in_phase(TicketPhase::InSanitation, PollPhase::SanitationCheck, ws)
.await;
}
Ok(())
}
async fn dispatch_engineer(ticket: Arc<Ticket>, ws: Workspace) {
let session_key = ticket_session_key(&ticket.id, Role::Engineer.as_str());
let last_eng_pos = ticket
.comments
.iter()
.rposition(|c| c.role == Role::Engineer.as_str());
let feedback: Vec<&str> = ticket
.comments
.iter()
.skip(last_eng_pos.map_or(0, |i| i + 1))
.map(|c| c.content.as_str())
.collect();
let message = if feedback.is_empty() {
"Implement the ticket described in the system prompt.".to_string()
} else {
format!("New feedback to address:\n{}", feedback.join("\n---\n"))
};
if let Err(e) = board()
.set_assigned_to(&ticket.id, Some(&session_key))
.await
{
warn!(
ticket = %ticket.id,
error = %e,
"Failed to set assigned_to for engineer — stale agent not cancelled",
);
}
let (_agent, response) =
run_agent(session_key, Role::Engineer, &ws, Some(&ticket), &message).await;
if !guard_ticket_in_phase(&ticket.id, TicketPhase::InDevelopment).await {
return;
}
let (comment_text, target_phase, notify) = if let Some(ref text) = response {
(
text.as_str(),
TicketPhase::InDiagnostics,
NotifyPolicy::Buffer,
)
} else {
("Agent failed", TicketPhase::Failed, NotifyPolicy::Notify)
};
if !comment_and_transition(
TransitionCtx {
ticket: &ticket,
source: TicketPhase::InDevelopment,
target: target_phase,
notify,
log_label: "Engineer",
},
(Role::Engineer.as_str(), comment_text),
)
.await
{
return;
}
info!(
ticket = %ticket.id,
target = %target_phase,
"Engineer finished — transitioned ticket",
);
}
async fn determine_notify_policy(workspace_name: &str, ticket_id: &str) -> NotifyPolicy {
match board()
.has_active_tickets_excluding(workspace_name, ticket_id)
.await
{
Ok(true) => {
debug!(
ticket = %ticket_id,
workspace = %workspace_name,
"Other active tickets remain — buffering Done notification",
);
NotifyPolicy::Buffer
}
Ok(false) => NotifyPolicy::Notify,
Err(e) => {
warn!(
ticket = %ticket_id,
workspace = %workspace_name,
error = %e,
"Failed to check active tickets — notifying to be safe",
);
NotifyPolicy::Notify
}
}
}
async fn transition_ticket_to_done(ticket: &Ticket, source: TicketPhase, comment: &str) {
let notify_policy = determine_notify_policy(&ticket.workspace_name, &ticket.id).await;
let log_label = source.as_ref();
if comment_and_transition(
TransitionCtx {
ticket,
source,
target: TicketPhase::Done,
notify: notify_policy,
log_label,
},
(SYSTEM_ROLE, comment),
)
.await
{
info!(ticket = %ticket.id, "{comment}");
}
}
#[must_use]
async fn transition_ticket_to_done_if_git_unavailable(
ticket: &Ticket,
repo_path: &Path,
source: TicketPhase,
) -> bool {
if !crate::git_commands::git_is_installed().await {
transition_ticket_to_done(
ticket,
source,
"Git not installed — moving to Done without commit",
)
.await;
return true;
}
if !crate::git_commands::is_git_repo(repo_path) {
transition_ticket_to_done(
ticket,
source,
"Not a git repo — moving to Done without commit",
)
.await;
return true;
}
false
}
async fn finalize_ticket_with_status(
ticket: Ticket,
ws: Workspace,
source: TicketPhase,
porcelain: &str,
) {
let repo_path = ws.as_path();
if porcelain.trim().is_empty() {
transition_ticket_to_done(
&ticket,
source,
"Clean working tree — moving to Done without commit",
)
.await;
return;
}
match crate::git_commands::run_git_commit(repo_path, &ticket.title).await {
Ok(commit_info) => {
finalize_commit_and_transition(&ticket, commit_info, source).await;
}
Err(e) => {
error!(
ticket = %ticket.id,
error = %e,
"Commit failed — staying in {} for retry",
source.as_ref(),
);
}
}
}
async fn finalize_ticket_from_phase(ticket: Ticket, ws: Workspace, source: TicketPhase) {
let repo_path = ws.as_path();
let phase_label = source.as_ref();
if transition_ticket_to_done_if_git_unavailable(&ticket, repo_path, source).await {
return;
}
let porcelain = match run_git_status(repo_path).await {
Ok(output) => output,
Err(e) => {
warn!(
ticket = %ticket.id,
error = %e,
"Failed to check git status — staying in {phase_label} for retry"
);
return;
}
};
finalize_ticket_with_status(ticket, ws, source, &porcelain).await;
}
async fn finalize_commit_and_transition(
ticket: &Ticket,
commit_info: crate::git_commands::CommitInfo,
source: TicketPhase,
) {
let comment = format_commit_summary(
commit_info.short_hash(),
commit_info.lines_added,
commit_info.lines_removed,
);
let phase_label = source.as_ref();
crate::registry::AGENT_REGISTRY.cancel_by_ticket_id(&ticket.id);
if crate::turso::with_tx(
&board().conn,
&ticket.id,
&format!(
"finalize Done transition from {phase_label} ({})",
commit_info.short_hash()
),
async |tx| {
BoardStore::finalize_done_tx(
tx,
&ticket.id,
&commit_info.hash,
commit_info.lines_added,
commit_info.lines_removed,
&comment,
source,
)
.await
},
)
.await
.is_ok()
{
info!(ticket = %ticket.id, "Committed {}, moving to Done", commit_info.short_hash());
let notify_policy = determine_notify_policy(&ticket.workspace_name, &ticket.id).await;
dispatch_notification(ticket, source, TicketPhase::Done, notify_policy).await;
} else {
warn!(
ticket = %ticket.id,
short_hash = commit_info.short_hash(),
"Commit was written to git but board transaction failed — \
orphan commit in repo, will retry on next poll cycle",
);
}
}
fn format_commit_summary(short_hash: &str, added: i64, removed: i64) -> String {
match (added, removed) {
(0, 0) => format!("Committed as `{short_hash}` (no changes)"),
(a, 0) => format!("Committed as `{short_hash}` (+{a})"),
(0, r) => format!("Committed as `{short_hash}` (-{r})"),
(a, r) => format!("Committed as `{short_hash}` (+{a}/-{r})"),
}
}
async fn handle_qa_passed(ticket: Ticket, ws: Workspace) {
let repo_path = ws.as_path();
if transition_ticket_to_done_if_git_unavailable(&ticket, repo_path, TicketPhase::QaPassed).await
{
return;
}
let porcelain = match run_git_status(repo_path).await {
Ok(out) => out,
Err(e) => {
warn!(
ticket = %ticket.id,
error = %e,
"Failed to check git status for untracked files — staying in QaPassed for retry"
);
return;
}
};
let untracked = parse_new_files_from_porcelain(&porcelain);
if untracked.is_empty() {
finalize_ticket_with_status(ticket, ws, TicketPhase::QaPassed, &porcelain).await;
} else {
let session_key = ticket_session_key(&ticket.id, Role::Sanitation.as_str());
let claimed = match board().claim_sanitation(&ticket.id, &session_key).await {
Ok(c) => c,
Err(e) => {
warn!(
ticket = %ticket.id,
error = %e,
"Failed to transition QaPassed ticket to InSanitation"
);
return;
}
};
if !claimed {
debug!(
ticket = %ticket.id,
"QaPassed ticket moved externally — skipping sanitation dispatch",
);
return;
}
ticket_buffer::push(
&ticket.workspace_name,
&ticket.id,
TicketPhase::QaPassed,
TicketPhase::InSanitation,
);
spawn_dispatch(PollPhase::SanitationCheck, ticket, ws);
}
}
async fn record_sanitation_failure(ticket_id: &str, reason: impl std::fmt::Display) {
let reason_str = format!("{SANITATION_FAILED_MARKER} — {reason}");
if let Err(e) = crate::turso::with_tx(
&board().conn,
ticket_id,
"record sanitation failure",
async |tx| {
BoardStore::add_comment_tx(tx, ticket_id, SYSTEM_ROLE, &reason_str).await?;
BoardStore::set_assigned_to_tx(tx, ticket_id, None).await?;
Ok(())
},
)
.await
{
warn!(
ticket = %ticket_id,
error = %e,
"Failed to record sanitation failure (circuit-breaker comment + assigned_to clear)",
);
}
}
async fn dispatch_sanitation(ticket: Arc<Ticket>, ws: Workspace) {
let session_key = ticket_session_key(&ticket.id, Role::Sanitation.as_str());
let untracked_files = match list_new_or_untracked_files(ws.as_path()).await {
Ok(files) => files.join("\n"),
Err(e) => {
warn!(
ticket = %ticket.id,
error = %e,
"Failed to list untracked files — proceeding with empty list",
);
String::from("(could not list untracked files)")
}
};
let prompt = substitute(
&crate::prompt::load_prompt("sanitation.md"),
&[
("{{ticket_title}}", &ticket.title),
("{{ticket_description}}", &ticket.description),
("{{untracked_files}}", &untracked_files),
],
);
let (agent, response) =
run_agent(session_key, Role::Sanitation, &ws, Some(&ticket), &prompt).await;
if !guard_ticket_in_phase(&ticket.id, TicketPhase::InSanitation).await {
return;
}
if response.is_none() {
warn!(
ticket = %ticket.id,
"Sanitation agent returned no output — clearing assigned_to for retry"
);
record_sanitation_failure(&ticket.id, "agent returned no output").await;
return;
}
let extraction_prompt = crate::prompt::load_prompt("extraction/sanitation.md");
let retry_prompt = crate::prompt::load_prompt("extraction/retry.md");
let verdict: crate::SanitationVerdict = match agent
.extract_structured(&extraction_prompt, &retry_prompt, 5)
.await
{
Ok(v) => v,
Err(e) => {
warn!(
ticket = %ticket.id,
error = %e,
"Failed to extract sanitation verdict — clearing assigned_to for retry"
);
record_sanitation_failure(&ticket.id, format!("verdict extraction error: {e}")).await;
return;
}
};
process_sanitation_verdict(&ticket, verdict).await;
}
async fn process_sanitation_verdict(ticket: &Ticket, verdict: crate::SanitationVerdict) {
if verdict.pass {
let passed_suffix = if verdict.garbage_files.is_empty() {
""
} else {
" (files reviewed)"
};
let comment = format!(
"🧹 Sanitation passed{passed_suffix}: {rationale}",
rationale = verdict.rationale
);
if !comment_and_transition(
TransitionCtx {
ticket,
source: TicketPhase::InSanitation,
target: TicketPhase::SanitationPassed,
notify: NotifyPolicy::Buffer,
log_label: "Sanitation",
},
(Role::Sanitation.as_str(), &comment),
)
.await
{
return;
}
info!(
ticket = %ticket.id,
"Sanitation passed — transitioned to SanitationPassed",
);
} else {
let garbage_list = verdict.garbage_files.join("\n- ");
let comment = format!(
"🗑️ Sanitation failed — garbage files detected:\n- {garbage_list}\n\nRationale: {rationale}\n\n\
These files might have been accidentally generated by other agents in the workspace for testing purposes. Engineer needs to clean them up if they are not required in the scope of the ticket.",
rationale = verdict.rationale,
garbage_list = garbage_list,
);
let sys_comment = format!(
"{SANITATION_FAILED_MARKER} — garbage files: {count}",
count = verdict.garbage_files.len(),
);
if !with_comment_and_transition(
TransitionCtx {
ticket,
source: TicketPhase::InSanitation,
target: TicketPhase::ReadyForDevelopment,
notify: NotifyPolicy::Buffer,
log_label: "Sanitation",
},
async |tx| {
BoardStore::add_comment_tx(
tx,
&ticket.id,
Role::Sanitation.as_str(),
comment.as_str(),
)
.await?;
BoardStore::add_comment_tx(tx, &ticket.id, SYSTEM_ROLE, sys_comment.as_str())
.await?;
Ok(())
},
)
.await
{
return;
}
info!(
ticket = %ticket.id,
"Sanitation failed — bounced back to ReadyForDevelopment with pipeline reservation",
);
}
}
async fn run_diagnostics_commands(diag: &DiagnosticsCommands, ws: &Workspace) -> (String, bool) {
let mut comment = String::from(DIAGNOSTICS_COMMENT_PREFIX);
let mut all_passed = true;
let mut failed_at: &str = "";
for (label, cmd_opt) in diag.commands() {
let Some(cmd) = cmd_opt else {
continue;
};
let _ = write!(comment, "\n\n{label} ({cmd}):\n");
match ShellTool::new(ShellMode::Full)
.execute_with_status(ws, serde_json::json!({"command": cmd}))
.await
{
Ok((output, exit_code)) => {
let display = if output.is_empty() {
"(no output)".to_string()
} else {
output
};
comment.push_str(&display);
if exit_code != Some(0) {
all_passed = false;
failed_at = label;
break;
}
}
Err(e) => {
comment.push_str(&e.to_string());
all_passed = false;
failed_at = label;
break;
}
}
}
if all_passed {
comment.push_str("\n\n---\n");
comment.push_str(DIAGNOSTICS_PASSED_MARKER);
} else {
let _ = write!(comment, "\n\n---\n{DIAGNOSTICS_FAILED_MARKER} {failed_at}");
}
(comment, all_passed)
}
async fn dispatch_diagnostics(ticket: Arc<Ticket>, ws: Workspace) {
match board()
.claim_diagnostics(&ticket.id, DIAGNOSTICS_ROLE)
.await
{
Err(e) => {
error!(
ticket = %ticket.id,
error = %e,
"Diagnostics claim error — bailing out",
);
return;
}
Ok(false) => {
warn!(
ticket = %ticket.id,
"Diagnostics claim failed — ticket already claimed or moved out of InDiagnostics"
);
return;
}
Ok(true) => {}
}
let (target_phase, comment_body): (TicketPhase, String) =
match crate::workspace::store().get_diagnostics(&ws.name).await {
Ok(Some(cmds)) if !cmds.is_empty() => {
let (comment, all_passed) = run_diagnostics_commands(&cmds, &ws).await;
if !guard_ticket_in_phase(&ticket.id, TicketPhase::InDiagnostics).await {
return;
}
if all_passed {
(TicketPhase::DiagnosticsDone, comment)
} else {
(TicketPhase::ReadyForDevelopment, comment)
}
}
Ok(_) => {
(
TicketPhase::DiagnosticsDone,
"No diagnostics commands are configured for this workspace \
— diagnostics skipped."
.to_string(),
)
}
Err(e) => {
warn!(
ticket = %ticket.id,
error = %e,
"Failed to load diagnostics for workspace — transitioning to DiagnosticsDone",
);
(
TicketPhase::DiagnosticsDone,
format!("Could not load diagnostics commands due to a database error: {e}"),
)
}
};
if !comment_and_transition(
TransitionCtx {
ticket: &ticket,
source: TicketPhase::InDiagnostics,
target: target_phase,
notify: NotifyPolicy::Buffer,
log_label: "Diagnostics",
},
(DIAGNOSTICS_ROLE, &comment_body),
)
.await
{
return;
}
info!(
ticket = %ticket.id,
target = %target_phase,
"Diagnostics finished — transitioned ticket",
);
}
#[derive(Clone)]
enum ParallelVerdict {
NoResponse,
ParseFailed,
Verdict(crate::Verdict),
}
async fn run_parallel_agents(
ticket: &Arc<Ticket>,
ws: &Workspace,
role: Role,
prompt: &str,
extraction_prompt: &str,
) -> Vec<ParallelVerdict> {
let suffix = crate::generate_suffix();
let retry_prompt = load_prompt("extraction/retry.md");
let futures: Vec<_> = (0..PARALLEL_AGENT_COUNT)
.map(move |i| {
let ticket = Arc::clone(ticket);
let prompt = prompt.to_string();
let ws = ws.clone();
let base = ticket_session_key(&ticket.id, role.as_str());
let session_key = format!("{base}_{i}_{suffix}");
let extraction_prompt = extraction_prompt.to_string();
let retry_prompt = retry_prompt.clone();
async move {
let (agent, response) =
run_agent(session_key, role, &ws, Some(&ticket), &prompt).await;
let response = response.unwrap_or_default();
if response.is_empty() {
return ParallelVerdict::NoResponse;
}
let verdict = agent
.extract_structured::<crate::Verdict>(&extraction_prompt, &retry_prompt, 5)
.await
.ok();
match verdict {
Some(v) => ParallelVerdict::Verdict(v),
None => ParallelVerdict::ParseFailed,
}
}
})
.collect();
join_all(futures).await
}
#[must_use]
fn verdict_passes(verdict: Option<&crate::Verdict>) -> bool {
verdict.is_some_and(|v| v.score >= REVIEW_QA_THRESHOLD)
}
fn format_verdict_body(verdict: &crate::Verdict) -> String {
let mut text = verdict.critique.clone().unwrap_or_default();
if !verdict.issues_detected.is_empty() {
if !text.is_empty() {
text.push_str("\n\n");
}
text.push_str("Issues:\n");
for issue in &verdict.issues_detected {
let _ = writeln!(text, "- {issue}");
}
}
text
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum VerdictFilter {
All,
FailingOnly,
}
fn format_verdict_comment(
r: &ParallelVerdict,
comment_role: &str,
filter: VerdictFilter,
) -> Option<String> {
match r {
ParallelVerdict::Verdict(v) => {
if filter == VerdictFilter::FailingOnly && verdict_passes(Some(v)) {
return None; }
let comment = format_verdict_body(v);
if comment.is_empty() {
return Some(format!(
"{} agent scored {}/10 with no specific critique provided.",
comment_role, v.score
));
}
Some(comment)
}
ParallelVerdict::ParseFailed => Some(format!(
"{comment_role} produced a response but verdict extraction failed — \
treating as a failure."
)),
ParallelVerdict::NoResponse => Some(format!(
"{comment_role} agent failed to produce a response — counting as a failure."
)),
}
}
async fn record_verdict_comments_tx(
tx: &TxGuard<'_>,
ticket_id: &str,
results: &[ParallelVerdict],
role_str: &str,
filter: VerdictFilter,
) -> anyhow::Result<()> {
for (i, r) in results.iter().enumerate() {
let role_label = format!("{role_str}_{}", i + 1);
if let Some(comment) = format_verdict_comment(r, &role_label, filter) {
BoardStore::insert_comment_tx(tx, ticket_id, &role_label, &comment).await?;
}
}
Ok(())
}
async fn dispatch_backlog_analysts(ticket: Arc<Ticket>, ws: Workspace) {
let prompt_key = if ticket.reporter == Role::Maintainer.as_str() {
"analyze/maintainer_ticket.md"
} else {
"analyze/manager_ticket.md"
};
let message = load_prompt(prompt_key);
let extraction_prompt = load_prompt("extraction/analyst.md");
let results =
run_parallel_agents(&ticket, &ws, Role::Analyst, &message, &extraction_prompt).await;
if !is_ticket_in_phase(&ticket.id, TicketPhase::Analysis).await {
return;
}
process_analyst_verdicts(&ticket, &results).await;
}
async fn process_analyst_verdicts(ticket: &Ticket, results: &[ParallelVerdict]) {
let nonempty_count = results
.iter()
.filter(|r| !matches!(r, ParallelVerdict::NoResponse))
.count();
let total = results.len();
let mut lgtm = 0usize;
let mut minor_issues = 0usize;
let mut potential_blockers = 0usize;
let mut missing_analysis = 0usize;
for r in results {
match r {
ParallelVerdict::Verdict(v)
if v.score >= ANALYST_PASS_THRESHOLD && v.issues_detected.is_empty() =>
{
lgtm += 1;
}
ParallelVerdict::Verdict(v) if v.score >= ANALYST_PASS_THRESHOLD => minor_issues += 1,
ParallelVerdict::Verdict(_) => potential_blockers += 1,
ParallelVerdict::NoResponse | ParallelVerdict::ParseFailed => missing_analysis += 1,
}
}
let summary = format_analyst_summary(
total,
lgtm,
minor_issues,
potential_blockers,
missing_analysis,
);
let extracted_count = total - missing_analysis;
let passing_count = lgtm + minor_issues;
let all_passed = passing_count == PARALLEL_AGENT_COUNT;
if !with_comment_and_transition(
TransitionCtx {
ticket,
source: TicketPhase::Analysis,
target: TicketPhase::Planning,
notify: NotifyPolicy::Notify,
log_label: "Analyst",
},
async |tx| {
record_verdict_comments_tx(
tx,
&ticket.id,
results,
Role::Analyst.as_str(),
VerdictFilter::All,
)
.await?;
BoardStore::add_comment_tx(tx, &ticket.id, SYSTEM_ROLE, &summary).await?;
Ok(())
},
)
.await
{
return;
}
if all_passed {
info!(
ticket = %ticket.id,
nonempty_count,
"Backlog analysis complete — all analysts passed (≥ {ANALYST_PASS_THRESHOLD}/10)",
);
} else {
info!(
ticket = %ticket.id,
nonempty_count,
extracted_count,
passing_count,
"Backlog analysis incomplete — moved to planning ({nonempty_count}/{PARALLEL_AGENT_COUNT} responded, \
{extracted_count} extracted, {passing_count} passed)",
);
}
}
fn format_analyst_summary(
total: usize,
lgtm: usize,
minor_issues: usize,
potential_blockers: usize,
missing_analysis: usize,
) -> String {
let description = [
(lgtm, "LGTM"),
(minor_issues, "found minor issues"),
(potential_blockers, "flagged potential blockers"),
(missing_analysis, "provided no analysis"),
]
.iter()
.filter(|&&(count, _label)| count > 0)
.map(|&(count, label)| {
if count == total {
format!("All {label}")
} else {
format!("{count} {label}")
}
})
.collect::<Vec<_>>()
.join(", ");
format!("{total} analysts reviewed this ticket. {description}.")
}
async fn drain_ready_for_development_siblings(ticket: &Ticket) {
match board()
.drain_ready_for_development_to_planning(&ticket.workspace_name)
.await
{
Ok(updated) if updated > 0 => {
info!(
tickets = updated,
workspace = %ticket.workspace_name,
"Moved {updated} ReadyForDevelopment ticket(s) to Planning after circuit breaker trip",
);
}
Ok(_) => {
debug!(
workspace = %ticket.workspace_name,
"No ReadyForDevelopment siblings to drain after circuit breaker trip",
);
}
Err(e) => {
warn!(
ticket = %ticket.id,
workspace = %ticket.workspace_name,
error = %e,
"Failed to move ReadyForDevelopment tickets to Planning \
— breaker trip proceeds without moving siblings",
);
}
}
}
#[must_use]
async fn try_trip_circuit_breaker(
ticket: &Ticket,
source_phase: TicketPhase,
kind: CircuitBreakerKind,
log_label: &str,
) -> bool {
let comments = match board().get_comments(&ticket.id).await {
Ok(c) => c,
Err(e) => {
warn!(
ticket = %ticket.id,
error = %e,
"Failed to fetch comments for circuit breaker — proceeding anyway"
);
return false;
}
};
let Some((count, threshold, msg)) = kind.should_trip(&comments) else {
return false;
};
info!(
ticket = %ticket.id,
count,
threshold,
log_label,
"Circuit breaker tripped at {count}/{threshold} ({log_label}) — failing ticket"
);
if comment_and_transition(
TransitionCtx {
ticket,
source: source_phase,
target: TicketPhase::Failed,
notify: NotifyPolicy::Notify,
log_label: &format!("{log_label} circuit breaker"),
},
(SYSTEM_ROLE, &msg),
)
.await
{
drain_ready_for_development_siblings(ticket).await;
}
true
}
async fn process_verifier_verdicts(
ticket: &Ticket,
results: &[ParallelVerdict],
verifier: VerifierInfo,
) {
let all_failed = results
.iter()
.all(|r| !matches!(r, ParallelVerdict::Verdict(_)));
let any_failed = results.iter().any(|r| match r {
ParallelVerdict::Verdict(v) => !verdict_passes(Some(v)),
_ => true,
});
let (target, notify) = if all_failed {
(TicketPhase::Failed, NotifyPolicy::Notify)
} else if any_failed {
(TicketPhase::ReadyForDevelopment, NotifyPolicy::Buffer)
} else {
(verifier.success_phase, NotifyPolicy::Buffer)
};
let failure_comment = if all_failed {
Some(format!(
"❌ All {} agents failed to produce verdicts — \
ticket marked as Failed.",
verifier.log_label,
))
} else {
None
};
if !with_comment_and_transition(
TransitionCtx {
ticket,
source: verifier.source_phase,
target,
notify,
log_label: verifier.log_label,
},
async |tx| {
record_verdict_comments_tx(
tx,
&ticket.id,
results,
verifier.role.as_str(),
VerdictFilter::FailingOnly,
)
.await?;
if let Some(ref fc) = failure_comment {
BoardStore::add_comment_tx(tx, &ticket.id, SYSTEM_ROLE, fc).await?;
}
Ok(())
},
)
.await
{
return;
}
if all_failed {
info!(
ticket = %ticket.id,
"{log_label}: all verifier agents failed to produce verdicts — ticket moved to Failed",
log_label = verifier.log_label,
);
} else if any_failed {
info!(
ticket = %ticket.id,
"{log_label} failed — pipeline reservation set for rework priority",
log_label = verifier.log_label,
);
} else {
info!(
ticket = %ticket.id,
"{log_label}: all passed (≥ {REVIEW_QA_THRESHOLD}/10)",
log_label = verifier.log_label,
);
}
}
async fn dispatch_verifiers(ticket: Arc<Ticket>, ws: Workspace, vi: VerifierInfo) {
let engineer_response = ticket
.comments
.iter()
.rev()
.find(|c| c.role == Role::Engineer.as_str())
.map(|c| &c.content)
.map_or("(no output)", String::as_str);
let prompt = substitute(
&crate::prompt::load_prompt(vi.prompt_template),
&[("{{agent_response}}", engineer_response)],
);
let extraction_prompt = crate::prompt::load_prompt(vi.extraction_prompt_path);
let results = run_parallel_agents(&ticket, &ws, vi.role, &prompt, &extraction_prompt).await;
if !is_ticket_in_phase(&ticket.id, vi.source_phase).await {
return;
}
process_verifier_verdicts(&ticket, &results, vi).await;
}
#[cfg(test)]
#[path = "management_tests.rs"]
mod tests;