use std::fmt::Write;
use std::future::Future;
use std::sync::Arc;
use std::time::Duration;
use tracing::{debug, error, info, warn};
use futures_util::FutureExt;
use futures_util::future::join_all;
use crate::agent::run_agent;
use crate::board::{BOARD, BoardStore, Ticket, TicketComment, TicketPhase};
use crate::diff_parse::list_untracked_files;
use crate::manager_queue::{JobKind, ManagerJob};
use crate::prompt::{load_prompt, substitute};
use crate::role::{DIAGNOSTICS_ROLE, SYSTEM_ROLE};
use crate::session::ticket_session_key;
use crate::ticket_buffer;
use crate::tools::shell::{ShellMode, ShellTool};
use crate::util::panic_message;
use crate::{Role, Tool, Workspace};
const PARALLEL_AGENT_COUNT: usize = 3;
const CIRCUIT_BREAKER_COMMENT_THRESHOLD: usize = 50;
const DIAGNOSTICS_CIRCUIT_BREAKER_THRESHOLD: usize = 4;
const SANITATION_CIRCUIT_BREAKER_THRESHOLD: usize = 3;
const _: () = assert!(SANITATION_CIRCUIT_BREAKER_THRESHOLD < CIRCUIT_BREAKER_COMMENT_THRESHOLD);
const _: () = assert!(DIAGNOSTICS_CIRCUIT_BREAKER_THRESHOLD < CIRCUIT_BREAKER_COMMENT_THRESHOLD);
const DIAGNOSTICS_COMMENT_PREFIX: &str = "🔍 Auto-diagnostics";
const DIAGNOSTICS_PASSED_MARKER: &str = "✅ All diagnostics passed";
const DIAGNOSTICS_FAILED_MARKER: &str = "❌ Diagnostics failed at";
const SANITATION_FAILED_PREFIX: &str = "Sanitation failed";
const ANALYSIS_THRESHOLD: u8 = 7;
const REVIEW_QA_THRESHOLD: u8 = 9;
#[inline]
fn board() -> &'static BoardStore {
crate::board::store()
}
fn count_sanitation_failures(comments: &[TicketComment]) -> usize {
comments
.iter()
.filter(|c| c.role == SYSTEM_ROLE && c.content.contains(SANITATION_FAILED_PREFIX))
.count()
}
fn general_breaker_comment(count: usize) -> String {
format!(
"Failed after {count} comments — ticket has accumulated too many comments \
(circuit breaker, threshold: {CIRCUIT_BREAKER_COMMENT_THRESHOLD}). \
Ticket failed — Manager will triage."
)
}
fn sanitation_breaker_comment(count: usize) -> String {
format!(
"❌ Sanitation circuit breaker tripped after {count} consecutive failures. \
(threshold: {SANITATION_CIRCUIT_BREAKER_THRESHOLD})",
)
}
#[must_use]
async fn is_ticket_in_phase(ticket_id: &str, expected: TicketPhase) -> bool {
match board().get_ticket_status(ticket_id).await {
Ok(Some(status)) => {
let ok = status == expected;
if !ok {
debug!(
ticket = %ticket_id,
expected = %expected,
actual = %status,
"Ticket moved externally — bailing out",
);
}
ok
}
Ok(None) => {
warn!(ticket = %ticket_id, "Ticket not found — may have been deleted");
false
}
Err(e) => {
warn!(ticket = %ticket_id, error = %e, "Failed to check ticket status");
false
}
}
}
#[must_use]
async fn guard_phase_and_circuit_breaker(
ticket: &Ticket,
expected: TicketPhase,
label: &str,
) -> bool {
if !is_ticket_in_phase(&ticket.id, expected).await {
return false;
}
if run_circuit_breaker(
ticket,
expected,
CIRCUIT_BREAKER_COMMENT_THRESHOLD,
<[TicketComment]>::len,
general_breaker_comment,
label,
)
.await
{
return false;
}
true
}
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
enum NotifyPolicy {
Notify,
Buffer,
}
async fn dispatch_notification(
ticket: &Ticket,
target: TicketPhase,
source: TicketPhase,
notify: NotifyPolicy,
log_label: &'static str,
) {
match notify {
NotifyPolicy::Notify => notify_ticket(ticket, target).await,
NotifyPolicy::Buffer => {
if let Some(ws) = resolve_ticket_workspace(ticket, log_label).await {
ticket_buffer::push(&ws.name, &ticket.id, source, target);
}
}
}
}
async fn transition_ticket(
ticket: &Ticket,
expected: TicketPhase,
target: TicketPhase,
notify: NotifyPolicy,
pipeline_reservation: Option<bool>,
) -> anyhow::Result<()> {
match board()
.transition_to(&ticket.id, Some(expected), target, pipeline_reservation)
.await
{
Ok(()) => {
dispatch_notification(ticket, target, expected, notify, "cannot buffer transition")
.await;
Ok(())
}
Err(e) => {
debug!(
ticket = %ticket.id,
expected = %expected,
target = %target,
error = %e,
"Failed to update ticket status",
);
Err(e)
}
}
}
#[must_use]
async fn resolve_ticket_workspace(
ticket: &Ticket,
log_label: &'static str,
) -> Option<crate::Workspace> {
match crate::workspace::get_by_name(&ticket.workspace_name).await {
Ok(Some(ws)) => Some(ws),
Ok(None) => {
warn!(
ticket = %ticket.id,
workspace_name = %ticket.workspace_name,
"Workspace not found for ticket — {log_label}",
);
None
}
Err(e) => {
warn!(
ticket = %ticket.id,
workspace_name = %ticket.workspace_name,
error = %e,
"Failed to look up workspace for ticket — {log_label}",
);
None
}
}
}
async fn bounce_back_to_development(ticket: &Ticket, source: TicketPhase, log_label: &str) {
if let Err(e) = transition_ticket(
ticket,
source,
TicketPhase::ReadyForDevelopment,
NotifyPolicy::Buffer,
Some(true),
)
.await
{
warn!(
ticket = %ticket.id,
error = %e,
"{log_label} failed but transition to ReadyForDevelopment also failed",
log_label = log_label,
);
} else {
info!(
ticket = %ticket.id,
"{log_label} failed — pipeline reservation set for rework priority",
log_label = log_label,
);
}
}
async fn notify_ticket(ticket: &Ticket, status: TicketPhase) {
let Some(ws) = resolve_ticket_workspace(ticket, "skipping notification").await else {
error!(
ticket = %ticket.id,
workspace_name = %ticket.workspace_name,
"Workspace resolution failed — notification skipped"
);
return;
};
let transition_log = format!(
"[{}] {}: {} → {}",
ticket.reporter,
ticket.id,
ticket.status,
status.as_ref()
);
let drained = crate::ticket_buffer::drain(&ws.name);
let mut message = substitute(
&load_prompt("notification.md"),
&[
("{{ticket_id}}", &ticket.id),
("{{ticket_title}}", &ticket.title),
("{{ticket_status}}", status.as_ref()),
("{{transition_log}}", &transition_log),
("{{ticket_updates}}", &drained),
],
);
if status == TicketPhase::Failed {
let failure_details = match board().get_comments(&ticket.id).await {
Ok(comments) => comments.last().map_or_else(
|| "No failure details available.".to_string(),
|c| c.content.clone(),
),
Err(e) => {
warn!(
ticket = %ticket.id,
error = %e,
"Failed to load comments for failure notification",
);
"No failure details available.".to_string()
}
};
let warning = substitute(
&load_prompt("warning.md"),
&[("{{failure_details}}", &failure_details)],
);
message.push_str("\n\n");
message.push_str(&warning);
}
crate::manager_queue::manager_queue().enqueue(ManagerJob {
content: message,
workspace_name: ws.name,
kind: JobKind::TicketNotify,
});
}
pub async fn run_management() {
if let Some(board) = BOARD.get()
&& let Err(e) = board.reset_inflight_tickets().await
{
error!(error = %e, "Failed to reset in-flight tickets");
}
let interval = Duration::from_secs(1);
loop {
if !crate::shutdown::sleep_or_shutdown(interval).await {
break;
}
if let Err(e) = poll_round().await {
error!(error = %e, "Board poller round failed");
}
}
}
fn spawn_dispatch(phase: PollPhase, ticket: Ticket, ws: Workspace) {
let phase_info = phase.info();
let active_phase = phase_info.active_phase;
info!(
ticket = %ticket.id,
title = %ticket.title,
workspace = %ws.name,
"Dispatching {} ticket",
phase_info.role_label,
);
let ticket = Arc::new(ticket);
let ticket_for_failure = Arc::clone(&ticket);
tokio::spawn(async move {
let result = std::panic::AssertUnwindSafe(async move {
match phase {
PollPhase::BacklogAnalysis => dispatch_backlog_analysts(ticket, ws).await,
PollPhase::EngineerDevelopment => dispatch_engineer(ticket, ws).await,
PollPhase::SanitationCheck => dispatch_sanitation(ticket, ws).await,
PollPhase::DiagnosticsCheck => dispatch_diagnostics(ticket, ws).await,
PollPhase::VerifierCheck(vi) => dispatch_verifiers(ticket, ws, vi).await,
}
})
.catch_unwind()
.await;
if let Err(payload) = result {
let msg = panic_message(&*payload);
error!(
ticket = %ticket_for_failure.id,
panic = %msg,
"Dispatch panicked — transitioning ticket to Failed",
);
let _ = board()
.add_comment(
&ticket_for_failure.id,
SYSTEM_ROLE,
&format!("❌ Dispatch panicked: {msg}"),
)
.await;
if let Err(e) = transition_ticket(
&ticket_for_failure,
active_phase,
TicketPhase::Failed,
NotifyPolicy::Notify,
None,
)
.await
{
warn!(
ticket = %ticket_for_failure.id,
error = %e,
"Failed to transition ticket to Failed after dispatch panic",
);
}
}
});
}
#[derive(Copy, Clone)]
struct VerifierInfo {
role: Role,
log_label: &'static str,
success_phase: TicketPhase,
active_phase: TicketPhase,
prompt_template: &'static str,
extraction_prompt_path: &'static str,
}
const REVIEWER_VI: VerifierInfo = VerifierInfo {
role: Role::Reviewer,
log_label: "Reviewers",
success_phase: TicketPhase::Reviewed,
active_phase: TicketPhase::InReview,
prompt_template: "review.md",
extraction_prompt_path: "extraction/reviewer.md",
};
const QA_VI: VerifierInfo = VerifierInfo {
role: Role::Qa,
log_label: "QA",
success_phase: TicketPhase::QaPassed,
active_phase: TicketPhase::InQa,
prompt_template: "qa.md",
extraction_prompt_path: "extraction/qa.md",
};
#[derive(Copy, Clone)]
struct PollPhaseInfo {
active_phase: TicketPhase,
require_clear_pipeline: bool,
role_label: &'static str,
}
#[derive(Copy, Clone)]
enum PollPhase {
BacklogAnalysis,
EngineerDevelopment,
SanitationCheck,
DiagnosticsCheck,
VerifierCheck(VerifierInfo),
}
impl PollPhase {
fn info(self) -> PollPhaseInfo {
match self {
Self::BacklogAnalysis => PollPhaseInfo {
active_phase: TicketPhase::Analysis,
require_clear_pipeline: false,
role_label: Role::Analyst.as_str(),
},
Self::EngineerDevelopment => PollPhaseInfo {
active_phase: TicketPhase::InDevelopment,
require_clear_pipeline: true,
role_label: Role::Engineer.as_str(),
},
Self::SanitationCheck => PollPhaseInfo {
active_phase: TicketPhase::InSanitation,
require_clear_pipeline: false,
role_label: Role::Sanitation.as_str(),
},
Self::DiagnosticsCheck => PollPhaseInfo {
active_phase: TicketPhase::InDiagnostics,
require_clear_pipeline: false,
role_label: DIAGNOSTICS_ROLE,
},
Self::VerifierCheck(vi) => PollPhaseInfo {
active_phase: vi.active_phase,
require_clear_pipeline: false,
role_label: vi.role.as_str(),
},
}
}
}
const CLAIM_PHASES: &[(TicketPhase, PollPhase)] = &[
(TicketPhase::Backlog, PollPhase::BacklogAnalysis),
(
TicketPhase::ReadyForDevelopment,
PollPhase::EngineerDevelopment,
),
(
TicketPhase::DiagnosticsDone,
PollPhase::VerifierCheck(REVIEWER_VI),
),
(TicketPhase::Reviewed, PollPhase::VerifierCheck(QA_VI)),
];
async fn for_tickets_in_phase(phase: TicketPhase, ws_name: &str, mut action: impl FnMut(Ticket)) {
match board().list_tickets_in_phase(phase, ws_name).await {
Ok(tickets) => {
for ticket in tickets {
action(ticket);
}
}
Err(e) => error!(workspace = ws_name, phase = %phase, error = %e, "Phase listing failed"),
}
}
async fn spawn_for_each_ticket_in_phase<F, Fut>(phase: TicketPhase, ws: &Workspace, f: F)
where
F: Fn(Ticket, Workspace) -> Fut + Clone + Send + 'static,
Fut: Future<Output = ()> + Send + 'static,
{
for_tickets_in_phase(phase, &ws.name, |ticket| {
let f = f.clone();
let ws = ws.clone();
tokio::spawn(async move {
f(ticket, ws).await;
});
})
.await;
}
async fn dispatch_unassigned_in_phase(
phase: TicketPhase,
dispatch_phase: PollPhase,
ws: &Workspace,
) {
for_tickets_in_phase(phase, &ws.name, |ticket| {
if ticket.assigned_to.is_some() {
return;
}
spawn_dispatch(dispatch_phase, ticket, ws.clone());
})
.await;
}
async fn poll_round() -> anyhow::Result<()> {
let board = board();
let workspaces = match crate::workspace::store().list().await {
Ok(ws_list) => ws_list,
Err(e) => {
error!(error = %e, "Failed to list workspaces");
return Ok(());
}
};
for ws in &workspaces {
for &(source, phase) in CLAIM_PHASES {
if ws.paused && matches!(phase, PollPhase::EngineerDevelopment) {
continue;
}
let info = phase.info();
let ticket = match board
.claim_ticket_in_workspace(
source,
info.active_phase,
&ws.name,
info.require_clear_pipeline,
)
.await
{
Ok(Some(t)) => {
ticket_buffer::push(&ws.name, &t.id, source, t.status);
t
}
Ok(None) => continue,
Err(e) => {
error!(
workspace = %ws.name,
phase = %info.role_label,
error = %e,
"Claim failed, skipping remaining phases for workspace",
);
break;
}
};
spawn_dispatch(phase, ticket, ws.clone());
}
dispatch_unassigned_in_phase(TicketPhase::InDiagnostics, PollPhase::DiagnosticsCheck, ws)
.await;
spawn_for_each_ticket_in_phase(TicketPhase::SanitationPassed, ws, |ticket, ws| {
finalize_ticket_from_phase(ticket, ws, TicketPhase::SanitationPassed)
})
.await;
spawn_for_each_ticket_in_phase(TicketPhase::QaPassed, ws, |ticket, ws| {
handle_qa_passed(ticket, ws)
})
.await;
dispatch_unassigned_in_phase(TicketPhase::InSanitation, PollPhase::SanitationCheck, ws)
.await;
}
Ok(())
}
async fn dispatch_engineer(ticket: Arc<Ticket>, ws: Workspace) {
let session_key = ticket_session_key(&ticket.id, Role::Engineer.as_str());
if !guard_phase_and_circuit_breaker(&ticket, TicketPhase::InDevelopment, "Engineer").await {
return;
}
let last_eng_pos = ticket
.comments
.iter()
.rposition(|c| c.role == Role::Engineer.as_str());
let feedback: Vec<&str> = ticket
.comments
.iter()
.skip(last_eng_pos.map_or(0, |i| i + 1))
.map(|c| c.content.as_str())
.collect();
let message = if feedback.is_empty() {
"Implement the ticket described in the system prompt.".to_string()
} else {
format!("New feedback to address:\n{}", feedback.join("\n---\n"))
};
let _ = board()
.set_assigned_to(&ticket.id, Some(&session_key))
.await;
let (_agent, response) =
run_agent(session_key, Role::Engineer, &ws, Some(&ticket), &message).await;
if !is_ticket_in_phase(&ticket.id, TicketPhase::InDevelopment).await {
return;
}
let (comment_text, target_phase, notify) = if let Some(ref text) = response {
(
text.as_str(),
TicketPhase::InDiagnostics,
NotifyPolicy::Buffer,
)
} else {
("Agent failed", TicketPhase::Failed, NotifyPolicy::Notify)
};
let _ = board()
.add_comment(&ticket.id, Role::Engineer.as_str(), comment_text)
.await;
if let Err(e) = transition_ticket(
&ticket,
TicketPhase::InDevelopment,
target_phase,
notify,
None,
)
.await
{
let verb = match target_phase {
TicketPhase::InDiagnostics => "completed",
_ => "failed",
};
warn!(
ticket = %ticket.id,
error = %e,
"Engineer {verb} but transition to {phase} failed — ticket stuck in {stuck}",
phase = target_phase.as_ref(),
stuck = TicketPhase::InDevelopment.as_ref(),
);
}
}
async fn determine_notify_policy(workspace_name: &str, ticket_id: &str) -> NotifyPolicy {
match board()
.has_active_tickets_excluding(workspace_name, ticket_id)
.await
{
Ok(true) => {
debug!(
ticket = %ticket_id,
workspace = %workspace_name,
"Other active tickets remain — buffering Done notification",
);
NotifyPolicy::Buffer
}
Ok(false) => NotifyPolicy::Notify,
Err(e) => {
warn!(
ticket = %ticket_id,
workspace = %workspace_name,
error = %e,
"Failed to check active tickets — notifying to be safe",
);
NotifyPolicy::Notify
}
}
}
async fn transition_ticket_to_done(ticket: &Ticket, source: TicketPhase, reason: &str) {
info!(ticket = %ticket.id, "{reason}");
let notify_policy = determine_notify_policy(&ticket.workspace_name, &ticket.id).await;
if let Err(e) = transition_ticket(ticket, source, TicketPhase::Done, notify_policy, None).await
{
let phase_label = source.as_ref();
warn!(
ticket = %ticket.id,
error = %e,
"{phase_label} passed but transition to Done failed",
);
}
}
async fn finalize_ticket_from_phase(ticket: Ticket, ws: Workspace, source: TicketPhase) {
let repo_path = ws.as_path();
let phase_label = source.as_ref();
if !crate::diff_parse::git_is_installed().await {
transition_ticket_to_done(
&ticket,
source,
"Git not installed — moving to Done without commit",
)
.await;
return;
}
if !crate::diff_parse::is_git_repo(repo_path) {
transition_ticket_to_done(
&ticket,
source,
"Not a git repo — moving to Done without commit",
)
.await;
return;
}
let has_changes = match crate::diff_parse::run_git_status(repo_path).await {
Ok(output) => !output.trim().is_empty(),
Err(e) => {
warn!(
ticket = %ticket.id,
error = %e,
"Failed to check git status — staying in {phase_label} for retry"
);
return;
}
};
if !has_changes {
transition_ticket_to_done(
&ticket,
source,
"Clean working tree — moving to Done without commit",
)
.await;
return;
}
match crate::diff_parse::run_git_commit(repo_path, &ticket.title).await {
Ok(commit_info) => {
commit_and_transition_ticket_from(&ticket, commit_info, source).await;
}
Err(e) => {
error!(
ticket = %ticket.id,
error = %e,
"Commit failed — staying in {phase_label} for retry"
);
}
}
}
async fn commit_and_transition_ticket_from(
ticket: &Ticket,
commit_info: crate::diff_parse::CommitInfo,
source: TicketPhase,
) {
let short_hash = commit_info.hash.get(..7).unwrap_or(&commit_info.hash);
let comment = format_commit_summary(
short_hash,
commit_info.lines_added,
commit_info.lines_removed,
);
let phase_label = source.as_ref();
crate::registry::AGENT_REGISTRY.cancel_by_ticket_id(&ticket.id);
let tx = match board().conn.begin_tx().await {
Ok(tx) => tx,
Err(e) => {
warn!(
ticket = %ticket.id,
error = %e,
"Failed to begin transaction — staying in {phase_label} for retry",
);
return;
}
};
let outcome: anyhow::Result<()> = async {
BoardStore::set_commit_info_tx(
&tx,
&ticket.id,
&commit_info.hash,
commit_info.lines_added,
commit_info.lines_removed,
)
.await?;
BoardStore::add_comment_tx(&tx, &ticket.id, SYSTEM_ROLE, &comment).await?;
BoardStore::transition_to_tx(&tx, &ticket.id, Some(source), TicketPhase::Done, None)
.await?;
Ok(())
}
.await;
match outcome {
Ok(()) => {
if let Err(e) = tx.commit().await {
error!(
ticket = %ticket.id,
error = %e,
"Commit succeeded ({short_hash}) but DB transaction commit failed — \
ticket stays in {phase_label} for retry, orphan commit in repo",
);
return;
}
info!(ticket = %ticket.id, "Committed {short_hash}, moving to Done");
let notify_policy = determine_notify_policy(&ticket.workspace_name, &ticket.id).await;
dispatch_notification(
ticket,
TicketPhase::Done,
source,
notify_policy,
"cannot buffer Done transition",
)
.await;
}
Err(e) => {
warn!(
ticket = %ticket.id,
error = %e,
"Failed to finalize Done transition — transaction rolled back, \
ticket stays in {phase_label} for retry",
);
}
}
}
fn format_commit_summary(short_hash: &str, added: i64, removed: i64) -> String {
match (added, removed) {
(0, 0) => format!("Committed as `{short_hash}` (no changes)"),
(a, 0) => format!("Committed as `{short_hash}` (+{a})"),
(0, r) => format!("Committed as `{short_hash}` (-{r})"),
(a, r) => format!("Committed as `{short_hash}` (+{a}/-{r})"),
}
}
async fn handle_qa_passed(ticket: Ticket, ws: Workspace) {
let repo_path = ws.as_path();
if !crate::diff_parse::git_is_installed().await || !crate::diff_parse::is_git_repo(repo_path) {
finalize_ticket_from_phase(ticket, ws, TicketPhase::QaPassed).await;
return;
}
let untracked = match list_untracked_files(repo_path).await {
Ok(files) => files,
Err(e) => {
warn!(
ticket = %ticket.id,
error = %e,
"Failed to check git status for untracked files — staying in QaPassed for retry"
);
return;
}
};
if untracked.is_empty() {
finalize_ticket_from_phase(ticket, ws, TicketPhase::QaPassed).await;
return;
}
let claimed = match board().claim_sanitation(&ticket.id).await {
Ok(c) => c,
Err(e) => {
warn!(
ticket = %ticket.id,
error = %e,
"Failed to transition QaPassed ticket to InSanitation"
);
return;
}
};
if !claimed {
debug!(
ticket = %ticket.id,
"QaPassed ticket moved externally — skipping sanitation dispatch",
);
return;
}
ticket_buffer::push(
&ticket.workspace_name,
&ticket.id,
TicketPhase::QaPassed,
TicketPhase::InSanitation,
);
spawn_dispatch(PollPhase::SanitationCheck, ticket, ws);
}
#[allow(clippy::too_many_lines)]
async fn dispatch_sanitation(ticket: Arc<Ticket>, ws: Workspace) {
let session_key = ticket_session_key(&ticket.id, Role::Sanitation.as_str());
if !is_ticket_in_phase(&ticket.id, TicketPhase::InSanitation).await {
return;
}
if run_circuit_breaker(
&ticket,
TicketPhase::InSanitation,
SANITATION_CIRCUIT_BREAKER_THRESHOLD,
count_sanitation_failures,
sanitation_breaker_comment,
"Sanitation",
)
.await
{
return;
}
let untracked_files = match list_untracked_files(ws.as_path()).await {
Ok(files) => files.join("\n"),
Err(e) => {
warn!(
ticket = %ticket.id,
error = %e,
"Failed to list untracked files — proceeding with empty list",
);
String::from("(could not list untracked files)")
}
};
let prompt = substitute(
&crate::prompt::load_prompt("sanitation.md"),
&[
("{{ticket_title}}", &ticket.title),
("{{ticket_description}}", &ticket.description),
("{{untracked_files}}", &untracked_files),
],
);
let (agent, response) =
run_agent(session_key, Role::Sanitation, &ws, Some(&ticket), &prompt).await;
if !is_ticket_in_phase(&ticket.id, TicketPhase::InSanitation).await {
return;
}
let Some(ref _text) = response else {
warn!(
ticket = %ticket.id,
"Sanitation agent returned no output — clearing assigned_to for retry"
);
let _ = board()
.add_comment(
&ticket.id,
SYSTEM_ROLE,
&format!("{SANITATION_FAILED_PREFIX} — agent returned no output"),
)
.await;
let _ = board().set_assigned_to(&ticket.id, None).await;
return;
};
let extraction_prompt = crate::prompt::load_prompt("extraction/sanitation.md");
let retry_prompt = crate::prompt::load_prompt("extraction/retry.md");
let verdict: crate::SanitationVerdict = match agent
.extract_structured(&extraction_prompt, &retry_prompt, 5)
.await
{
Ok(v) => v,
Err(e) => {
warn!(
ticket = %ticket.id,
error = %e,
"Failed to extract sanitation verdict — clearing assigned_to for retry"
);
let _ = board()
.add_comment(
&ticket.id,
SYSTEM_ROLE,
&format!("{SANITATION_FAILED_PREFIX} — verdict extraction error: {e}"),
)
.await;
let _ = board().set_assigned_to(&ticket.id, None).await;
return;
}
};
if verdict.pass {
info!(
ticket = %ticket.id,
"Sanitation passed — transitioning to SanitationPassed",
);
let comment = if verdict.garbage_files.is_empty() {
format!(
"🧹 Sanitation passed: {rationale}",
rationale = verdict.rationale
)
} else {
format!(
"🧹 Sanitation passed (files reviewed): {rationale}",
rationale = verdict.rationale
)
};
let _ = board()
.add_comment(&ticket.id, Role::Sanitation.as_str(), &comment)
.await;
if let Err(e) = transition_ticket(
&ticket,
TicketPhase::InSanitation,
TicketPhase::SanitationPassed,
NotifyPolicy::Buffer,
None,
)
.await
{
warn!(
ticket = %ticket.id,
error = %e,
"Sanitation passed but transition to SanitationPassed failed — \
ticket stuck in InSanitation"
);
}
} else {
let garbage_list = verdict.garbage_files.join("\n- ");
let comment = format!(
"🗑️ Sanitation failed — garbage files detected:\n- {garbage_list}\n\nRationale: {rationale}",
rationale = verdict.rationale,
);
let _ = board()
.add_comment(&ticket.id, Role::Sanitation.as_str(), &comment)
.await;
let _ = board()
.add_comment(
&ticket.id,
SYSTEM_ROLE,
&format!(
"{SANITATION_FAILED_PREFIX} — garbage files: {count}",
count = verdict.garbage_files.len(),
),
)
.await;
bounce_back_to_development(&ticket, TicketPhase::InSanitation, "Sanitation").await;
}
}
#[allow(clippy::too_many_lines)]
async fn dispatch_diagnostics(ticket: Arc<Ticket>, ws: Workspace) {
match board().claim_diagnostics(&ticket.id).await {
Err(e) => {
error!(
ticket = %ticket.id,
error = %e,
"Diagnostics claim error — bailing out",
);
return;
}
Ok(false) => {
warn!(
ticket = %ticket.id,
"Diagnostics claim failed — ticket already claimed or moved out of InDiagnostics"
);
return;
}
Ok(true) => {}
}
let diag = match crate::workspace::store().get_diagnostics(&ws.name).await {
Ok(Some(cmds)) if !cmds.is_empty() => Some(cmds),
Ok(Some(_) | None) => None,
Err(e) => {
warn!(
ticket = %ticket.id,
error = %e,
"Failed to load diagnostics for workspace — transitioning to DiagnosticsDone"
);
None
}
};
let Some(diag) = diag else {
if let Err(e) = transition_ticket(
&ticket,
TicketPhase::InDiagnostics,
TicketPhase::DiagnosticsDone,
NotifyPolicy::Buffer,
None,
)
.await
{
warn!(
ticket = %ticket.id,
error = %e,
"No diagnostics commands — failed to transition to DiagnosticsDone",
);
}
return;
};
if run_circuit_breaker(
&ticket,
TicketPhase::InDiagnostics,
DIAGNOSTICS_CIRCUIT_BREAKER_THRESHOLD,
|comments| {
comments
.iter()
.filter(|c| {
c.role == DIAGNOSTICS_ROLE
&& c.content.starts_with(DIAGNOSTICS_COMMENT_PREFIX)
&& c.content.contains(DIAGNOSTICS_FAILED_MARKER)
})
.count()
},
|count| {
format!(
"{DIAGNOSTICS_COMMENT_PREFIX}\n\n❌ Circuit breaker: {count} prior diagnostic \
failures. Failing ticket."
)
},
"Diagnostics",
)
.await
{
return;
}
let mut comment = String::from(DIAGNOSTICS_COMMENT_PREFIX);
let mut all_passed = true;
let mut failed_at: &str = "";
for (label, cmd_opt) in diag.commands() {
let Some(cmd) = cmd_opt else {
continue;
};
let _ = write!(comment, "\n\n{label} ({cmd}):\n");
match ShellTool::new(ShellMode::Full)
.execute(&ws, serde_json::json!({"command": cmd}))
.await
{
Ok(output) => {
let failed = output.contains("[exit status: ");
let display = if output.is_empty() {
"(no output)".to_string()
} else {
output
};
comment.push_str(&display);
if failed {
all_passed = false;
failed_at = label;
break;
}
}
Err(e) => {
comment.push_str(&e.to_string());
all_passed = false;
failed_at = label;
break;
}
}
}
if all_passed {
comment.push_str("\n\n---\n");
comment.push_str(DIAGNOSTICS_PASSED_MARKER);
} else {
let _ = write!(comment, "\n\n---\n{DIAGNOSTICS_FAILED_MARKER} {failed_at}");
}
let _ = board()
.add_comment(&ticket.id, DIAGNOSTICS_ROLE, &comment)
.await;
if all_passed {
if let Err(e) = transition_ticket(
&ticket,
TicketPhase::InDiagnostics,
TicketPhase::DiagnosticsDone,
NotifyPolicy::Buffer,
None,
)
.await
{
warn!(
ticket = %ticket.id,
error = %e,
"Diagnostics completed but transition to DiagnosticsDone \
failed — ticket stuck in DiagnosticsDone",
);
}
} else {
bounce_back_to_development(&ticket, TicketPhase::InDiagnostics, "Diagnostics").await;
}
}
struct ParallelVerdict {
response: String,
verdict: Option<crate::Verdict>,
}
async fn extract_parallel_verdicts(
results: Vec<(crate::Agent, String)>,
extraction_prompt: &str,
) -> Vec<ParallelVerdict> {
let retry_prompt = crate::prompt::load_prompt("extraction/retry.md");
let futures: Vec<_> = results
.into_iter()
.map(|(agent, response)| {
let extraction_prompt = extraction_prompt.to_string();
let retry_prompt = retry_prompt.clone();
async move {
if response.is_empty() {
return ParallelVerdict {
response,
verdict: None,
};
}
let verdict = agent
.extract_structured::<crate::Verdict>(&extraction_prompt, &retry_prompt, 5)
.await
.ok();
ParallelVerdict { response, verdict }
}
})
.collect();
join_all(futures).await
}
async fn run_parallel_with_extraction(
ticket: &Arc<Ticket>,
ws: &Workspace,
role: Role,
prompt: &str,
extraction_prompt: &str,
) -> Vec<ParallelVerdict> {
let suffix = crate::generate_suffix();
let futures: Vec<_> = (0..PARALLEL_AGENT_COUNT)
.map(move |i| {
let ticket = Arc::clone(ticket);
let prompt = prompt.to_string();
let ws = ws.clone();
let base = ticket_session_key(&ticket.id, role.as_str());
let session_key = format!("{base}_{i}_{suffix}");
async move {
let (agent, response) =
run_agent(session_key, role, &ws, Some(&ticket), &prompt).await;
(agent, response.unwrap_or_default())
}
})
.collect();
let results = join_all(futures).await;
extract_parallel_verdicts(results, extraction_prompt).await
}
#[must_use]
fn verdict_passes(verdict: Option<&crate::Verdict>) -> bool {
verdict.is_some_and(|v| v.score >= REVIEW_QA_THRESHOLD)
}
fn format_verdict_body(verdict: &crate::Verdict) -> String {
let mut text = verdict.critique.clone().unwrap_or_default();
if !verdict.issues_detected.is_empty() {
if !text.is_empty() {
text.push_str("\n\n");
}
text.push_str("Issues:\n");
for issue in &verdict.issues_detected {
let _ = writeln!(text, "- {issue}");
}
}
text
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum VerdictFilter {
All,
FailingOnly,
}
fn format_verdict_comment(
r: &ParallelVerdict,
comment_role: &str,
filter: VerdictFilter,
) -> Option<String> {
if let Some(v) = &r.verdict {
if filter == VerdictFilter::FailingOnly && verdict_passes(Some(v)) {
return None; }
let comment = format_verdict_body(v);
if comment.is_empty() {
return Some(format!(
"{} agent scored {}/10 with no specific critique provided.",
comment_role, v.score
));
}
return Some(comment);
}
if r.response.is_empty() {
Some(format!(
"{comment_role} agent failed to produce a response — counting as a failure."
))
} else {
Some(format!(
"{comment_role} produced a response but verdict extraction failed — \
treating as a failure."
))
}
}
async fn record_verdict_comments(
ticket_id: &str,
results: &[ParallelVerdict],
role_str: &str,
filter: VerdictFilter,
) {
for (i, r) in results.iter().enumerate() {
let role_label = format!("{role_str}_{}", i + 1);
if let Some(comment) = format_verdict_comment(r, &role_label, filter) {
let _ = board().add_comment(ticket_id, &role_label, &comment).await;
}
}
}
async fn dispatch_backlog_analysts(ticket: Arc<Ticket>, ws: Workspace) {
if !guard_phase_and_circuit_breaker(&ticket, TicketPhase::Analysis, "Analysts").await {
return;
}
let message = load_prompt("analyze.md");
let extraction_prompt = load_prompt("extraction/analyst.md");
let parallel_results =
run_parallel_with_extraction(&ticket, &ws, Role::Analyst, &message, &extraction_prompt)
.await;
if !is_ticket_in_phase(&ticket.id, TicketPhase::Analysis).await {
return;
}
handle_analyst_verdicts(&ticket, ¶llel_results).await;
}
async fn handle_analyst_verdicts(ticket: &Ticket, results: &[ParallelVerdict]) {
record_verdict_comments(
&ticket.id,
results,
Role::Analyst.as_str(),
VerdictFilter::All,
)
.await;
let nonempty_count = results.iter().filter(|r| !r.response.is_empty()).count();
let total = results.len();
let mut lgtm = 0usize;
let mut minor_issues = 0usize;
let mut potential_blockers = 0usize;
let mut missing_analysis = 0usize;
for r in results {
match &r.verdict {
Some(v) if v.score >= ANALYSIS_THRESHOLD && v.issues_detected.is_empty() => lgtm += 1,
Some(v) if v.score >= ANALYSIS_THRESHOLD => minor_issues += 1,
Some(_) => potential_blockers += 1,
None => missing_analysis += 1,
}
}
let summary = build_analyst_summary(
total,
lgtm,
minor_issues,
potential_blockers,
missing_analysis,
);
let _ = board().add_comment(&ticket.id, SYSTEM_ROLE, &summary).await;
let extracted_count = total - missing_analysis;
let passing_count = lgtm + minor_issues;
let all_passed = passing_count == PARALLEL_AGENT_COUNT;
let target = TicketPhase::Planning;
if let Err(e) = transition_ticket(
ticket,
TicketPhase::Analysis,
target,
NotifyPolicy::Notify,
None,
)
.await
{
warn!(
ticket = %ticket.id,
error = %e,
"Analyst verdicts completed but transition to {phase} failed — ticket stuck in {stuck}",
phase = target.as_ref(),
stuck = TicketPhase::Analysis.as_ref(),
);
return;
}
if all_passed {
info!(
ticket = %ticket.id,
nonempty_count,
"Backlog analysis complete — all analysts passed (≥ {ANALYSIS_THRESHOLD}/10)",
);
} else {
info!(
ticket = %ticket.id,
nonempty_count,
extracted_count,
passing_count,
"Backlog analysis incomplete — moved to planning ({nonempty_count}/{PARALLEL_AGENT_COUNT} responded, \
{extracted_count} extracted, {passing_count} passed)",
);
}
}
fn build_analyst_summary(
total: usize,
lgtm: usize,
minor_issues: usize,
potential_blockers: usize,
missing_analysis: usize,
) -> String {
let description = [
(lgtm, "LGTM"),
(minor_issues, "found minor issues"),
(potential_blockers, "flagged potential blockers"),
(missing_analysis, "provided no analysis"),
]
.iter()
.filter(|&&(count, _label)| count > 0)
.map(|&(count, label)| {
if count == total {
format!("All {label}")
} else {
format!("{count} {label}")
}
})
.collect::<Vec<_>>()
.join(", ");
format!("{total} analysts reviewed this ticket. {description}.")
}
async fn drain_ready_for_development_siblings(ticket: &Ticket) {
let other_tickets = match board()
.list_tickets_in_phase(TicketPhase::ReadyForDevelopment, &ticket.workspace_name)
.await
{
Ok(tickets) => tickets,
Err(e) => {
warn!(
ticket = %ticket.id,
workspace = %ticket.workspace_name,
error = %e,
"Failed to list ReadyForDevelopment tickets for moving to planning \
— breaker trip proceeds without moving siblings",
);
return;
}
};
let planning_move_comment = format!(
"Moved to planning due to circuit breaker trip on {}: {}. Re-advance to ReadyForDevelopment after Manager resolves the failure.",
ticket.id, ticket.title,
);
for other in other_tickets.iter().filter(|t| t.id != ticket.id) {
if let Err(e) = transition_ticket(
other,
TicketPhase::ReadyForDevelopment,
TicketPhase::Planning,
NotifyPolicy::Buffer,
None,
)
.await
{
debug!(
other_ticket = %other.id,
error = %e,
"Failed to move other ReadyForDevelopment ticket to planning — likely raced by external move",
);
continue;
}
let _ = board()
.add_comment(&other.id, SYSTEM_ROLE, &planning_move_comment)
.await;
}
}
#[must_use]
async fn run_circuit_breaker(
ticket: &Ticket,
expected: TicketPhase,
threshold: usize,
count_fn: impl Fn(&[TicketComment]) -> usize,
comment_text: impl Fn(usize) -> String,
log_label: &str,
) -> bool {
let comments = match board().get_comments(&ticket.id).await {
Ok(c) => c,
Err(e) => {
warn!(
ticket = %ticket.id,
error = %e,
"Failed to fetch comments for circuit breaker — proceeding anyway"
);
return false;
}
};
let count = count_fn(&comments);
if count <= threshold {
return false;
}
info!(
ticket = %ticket.id,
count,
threshold,
log_label,
"Circuit breaker tripped at {count}/{threshold} ({log_label}) — failing ticket"
);
let _ = board()
.add_comment(&ticket.id, SYSTEM_ROLE, &comment_text(count))
.await;
if let Err(e) = transition_ticket(
ticket,
expected,
TicketPhase::Failed,
NotifyPolicy::Notify,
None,
)
.await
{
warn!(
ticket = %ticket.id,
error = %e,
"Circuit breaker tripped but transition to Failed failed",
);
return true;
}
drain_ready_for_development_siblings(ticket).await;
true
}
async fn process_verdict_results(
ticket: &Ticket,
results: &[ParallelVerdict],
verifier: VerifierInfo,
) {
record_verdict_comments(
&ticket.id,
results,
verifier.role.as_str(),
VerdictFilter::FailingOnly,
)
.await;
let all_failed = results.iter().all(|r| r.verdict.is_none());
if all_failed {
let _ = board()
.add_comment(
&ticket.id,
SYSTEM_ROLE,
&format!(
"❌ All {label} agents failed to produce verdicts — \
ticket marked as Failed.",
label = verifier.log_label,
),
)
.await;
if let Err(e) = transition_ticket(
ticket,
verifier.active_phase,
TicketPhase::Failed,
NotifyPolicy::Notify,
None,
)
.await
{
warn!(
ticket = %ticket.id,
error = %e,
role = %verifier.log_label,
"All {label} agents failed but transition to Failed also failed",
label = verifier.log_label,
);
}
return;
}
if results.iter().any(|r| !verdict_passes(r.verdict.as_ref())) {
bounce_back_to_development(ticket, verifier.active_phase, verifier.log_label).await;
return;
}
if let Err(e) = transition_ticket(
ticket,
verifier.active_phase,
verifier.success_phase,
NotifyPolicy::Buffer,
None,
)
.await
{
warn!(
ticket = %ticket.id,
error = %e,
"{role} verdicts completed but transition to {phase} failed — ticket stuck in {stuck}",
role = verifier.log_label,
phase = verifier.success_phase.as_ref(),
stuck = verifier.active_phase.as_ref(),
);
} else {
info!(
ticket = %ticket.id,
"{log_label}: all passed (≥ {REVIEW_QA_THRESHOLD}/10)",
log_label = verifier.log_label,
);
}
}
async fn dispatch_verifiers(ticket: Arc<Ticket>, ws: Workspace, vi: VerifierInfo) {
if !guard_phase_and_circuit_breaker(&ticket, vi.active_phase, vi.log_label).await {
return;
}
let engineer_response = ticket
.comments
.iter()
.rev()
.find(|c| c.role == Role::Engineer.as_str())
.map(|c| &c.content)
.map_or("(no output)", String::as_str);
let prompt = substitute(
&crate::prompt::load_prompt(vi.prompt_template),
&[("{{agent_response}}", engineer_response)],
);
let extraction_prompt = crate::prompt::load_prompt(vi.extraction_prompt_path);
let results =
run_parallel_with_extraction(&ticket, &ws, vi.role, &prompt, &extraction_prompt).await;
if !is_ticket_in_phase(&ticket.id, vi.active_phase).await {
return;
}
process_verdict_results(&ticket, &results, vi).await;
}
#[cfg(test)]
mod tests {
use super::*;
use crate::board::DEFAULT_TICKET_PHASE;
use crate::util::test::TicketBuilder;
use crate::util::test::{expect_ticket, expect_ticket_status, init_test_stores};
use crate::workspace::test_ws_named;
#[tokio::test]
async fn guard_phase_mismatch_rejected() {
init_test_stores().await;
let ws = test_ws_named("/tmp/test", "test");
let ticket_id = TicketBuilder::new(board(), ws)
.title("Test")
.create()
.await
.expect("create_ticket");
board()
.transition_to(
&ticket_id,
Some(TicketPhase::Backlog),
TicketPhase::InDevelopment,
None,
)
.await
.expect("transition_to");
let ticket = board()
.get_ticket(&ticket_id)
.await
.expect("get_ticket")
.expect("ticket exists");
assert!(
!guard_phase_and_circuit_breaker(&ticket, TicketPhase::InReview, "test_label").await,
"guard_phase_and_circuit_breaker must reject a phase mismatch"
);
assert!(
guard_phase_and_circuit_breaker(&ticket, TicketPhase::InDevelopment, "test_label")
.await,
"guard_phase_and_circuit_breaker must pass when phase matches and comments are below threshold"
);
}
#[allow(clippy::too_many_lines)]
#[tokio::test]
async fn circuit_breaker_moves_other_ready_for_development_tickets_to_planning() {
init_management_test_stores().await;
let ws_a = test_ws_named("/ws_a", "ws_a");
let ws_b = test_ws_named("/ws_b", "ws_b");
let trip_id = TicketBuilder::new(board(), ws_a.clone())
.title("Trip Ticket")
.phase(TicketPhase::ReadyForDevelopment)
.create()
.await
.expect("create_ticket A");
let victim_id = TicketBuilder::new(board(), ws_a)
.title("Victim Ticket")
.phase(TicketPhase::ReadyForDevelopment)
.create()
.await
.expect("create_ticket B");
let other_ws_id = TicketBuilder::new(board(), ws_b)
.title("Other Workspace Ticket")
.phase(TicketPhase::ReadyForDevelopment)
.create()
.await
.expect("create_ticket C");
board()
.add_comment(&trip_id, SYSTEM_ROLE, "Some comment")
.await
.expect("add_comment to A");
let ticket_a = board()
.get_ticket(&trip_id)
.await
.expect("get_ticket A")
.expect("ticket A exists");
let tripped = run_circuit_breaker(
&ticket_a,
TicketPhase::ReadyForDevelopment,
0, <[TicketComment]>::len, |count| format!("Breaker tripped at {count}"),
"test",
)
.await;
assert!(tripped, "circuit breaker should have tripped");
{
let ticket_a = board()
.get_ticket(&trip_id)
.await
.expect("get_ticket A")
.expect("ticket A exists");
assert_eq!(
ticket_a.status,
TicketPhase::Failed,
"tripped ticket A should be Failed"
);
}
{
let ticket_b = board()
.get_ticket(&victim_id)
.await
.expect("get_ticket B")
.expect("ticket B exists");
assert_eq!(
ticket_b.status,
TicketPhase::Planning,
"other ReadyForDevelopment ticket B in same workspace should be Planning"
);
}
{
let ticket_c = board()
.get_ticket(&other_ws_id)
.await
.expect("get_ticket C")
.expect("ticket C exists");
assert_eq!(
ticket_c.status,
TicketPhase::ReadyForDevelopment,
"ticket C in different workspace must not be moved"
);
}
{
let comments = board()
.get_comments(&victim_id)
.await
.expect("get_comments for B");
let comment = comments
.iter()
.find(|c| c.role == SYSTEM_ROLE)
.expect("ticket B should have a system comment");
assert!(
comment.content.contains(&trip_id),
"comment should contain the tripped ticket's ID"
);
assert!(
comment
.content
.contains("Moved to planning due to circuit breaker trip on"),
"comment should start with expected format"
);
assert!(
comment.content.contains(
"Re-advance to ReadyForDevelopment after Manager resolves the failure"
),
"comment should end with expected format"
);
}
}
#[tokio::test]
async fn record_verdict_comments_counts() {
init_test_stores().await;
let ws = test_ws_named("/tmp/test", "test");
let ticket_id = TicketBuilder::new(board(), ws)
.title("Test")
.create()
.await
.expect("create_ticket");
let passing_verdict = crate::Verdict {
score: REVIEW_QA_THRESHOLD, critique: None,
issues_detected: vec![],
};
let results = vec![ParallelVerdict {
response: "Looks good.".into(),
verdict: Some(passing_verdict),
}];
record_verdict_comments(
&ticket_id,
&results,
Role::Reviewer.as_str(),
VerdictFilter::FailingOnly,
)
.await;
let comments = board()
.get_comments(&ticket_id)
.await
.expect("get_comments");
assert_eq!(
comments.len(),
0,
"passing verdicts with FailingOnly filter should produce 0 comments"
);
let failing = crate::Verdict {
score: 3, critique: Some("Missing error handling.".into()),
issues_detected: vec!["No timeout check".into()],
};
let results = vec![ParallelVerdict {
response: "Has issues.".into(),
verdict: Some(failing),
}];
record_verdict_comments(
&ticket_id,
&results,
Role::Reviewer.as_str(),
VerdictFilter::FailingOnly,
)
.await;
let comments = board()
.get_comments(&ticket_id)
.await
.expect("get_comments");
assert_eq!(
comments.len(),
1,
"failing verdict should create one comment"
);
assert_eq!(comments[0].role, "reviewer_1");
let pass = crate::Verdict {
score: 10,
critique: Some("Excellent analysis.".into()),
issues_detected: vec![],
};
let fail = crate::Verdict {
score: 4,
critique: Some("Needs more research.".into()),
issues_detected: vec!["Missing citations".into()],
};
let results = vec![
ParallelVerdict {
response: "Agent 1 response.".into(),
verdict: Some(pass),
},
ParallelVerdict {
response: "Agent 2 response.".into(),
verdict: Some(fail),
},
];
record_verdict_comments(
&ticket_id,
&results,
Role::Analyst.as_str(),
VerdictFilter::All,
)
.await;
let comments = board()
.get_comments(&ticket_id)
.await
.expect("get_comments");
assert_eq!(
comments.len(),
3,
"All filter should write both verdicts (total 3)"
);
}
async fn init_management_test_stores() {
init_test_stores().await;
let _ = crate::workspace::init_global().await;
let _ = crate::manager_queue::init_global();
}
async fn create_test_workspace(name: &str, path: &str) -> crate::Workspace {
let now = crate::turso::now();
crate::workspace::store()
.conn
.execute(
"INSERT INTO workspaces (name, path, created_at, updated_at, paused) \
VALUES (?1, ?2, ?3, ?4, ?5)",
turso::params![name, path, now.clone(), now, 0],
)
.await
.expect("insert test workspace");
test_ws_named(path, name)
}
async fn setup_ticket_to_done_test(suffix: &str) -> crate::Workspace {
init_management_test_stores().await;
let ws_name = format!("ws_{suffix}");
let ws_path = format!("/tmp/test_{suffix}");
create_test_workspace(&ws_name, &ws_path).await
}
#[tokio::test]
async fn transition_ticket_to_done_buffer_and_notify() {
let ws = setup_ticket_to_done_test("drains_buffer").await;
let first_id = TicketBuilder::new(board(), ws.clone())
.title("Ticket A")
.phase(TicketPhase::QaPassed)
.create()
.await
.expect("create ticket A");
let second_id = TicketBuilder::new(board(), ws)
.title("Ticket B")
.phase(TicketPhase::QaPassed)
.create()
.await
.expect("create ticket B");
let ticket_a = board()
.get_ticket(&first_id)
.await
.expect("get_ticket")
.expect("ticket A exists");
transition_ticket_to_done(
&ticket_a,
TicketPhase::QaPassed,
"Test — ticket A done, B still active",
)
.await;
let intermediate = crate::ticket_buffer::drain("ws_drains_buffer");
assert!(
!intermediate.is_empty(),
"After first QaPassed → Done with other active tickets: \
should have buffered the notification (got empty buffer)",
);
let ticket_b = board()
.get_ticket(&second_id)
.await
.expect("get_ticket")
.expect("ticket B exists");
transition_ticket_to_done(
&ticket_b,
TicketPhase::QaPassed,
"Test — ticket B done, last ticket",
)
.await;
for (id, label) in [(&first_id, "A"), (&second_id, "B")] {
let t = board()
.get_ticket(id)
.await
.expect("get_ticket")
.unwrap_or_else(|| panic!("ticket {label} exists"));
assert_eq!(t.status, TicketPhase::Done, "Ticket {label} should be Done");
}
let drained = crate::ticket_buffer::drain("ws_drains_buffer");
assert!(
drained.is_empty(),
"Buffer should be empty after last ticket's Notify drains it",
);
}
#[tokio::test]
async fn transition_never_pauses_workspace() {
struct Case {
name: &'static str,
ws_suffix: &'static str,
ws_path: &'static str,
source: TicketPhase,
target: TicketPhase,
policy: NotifyPolicy,
}
let cases = [
Case {
name: "Failed with Buffer",
ws_suffix: "ws_no_pause_on_fail_test",
ws_path: "/tmp/test_ws_no_pause_on_fail",
source: DEFAULT_TICKET_PHASE,
target: TicketPhase::Failed,
policy: NotifyPolicy::Buffer,
},
Case {
name: "non-failure with Notify",
ws_suffix: "ws_no_pause_test",
ws_path: "/tmp/test_ws_no_pause",
source: TicketPhase::Backlog,
target: TicketPhase::Analysis,
policy: NotifyPolicy::Notify,
},
];
init_management_test_stores().await;
for case in &cases {
let ws = create_test_workspace(case.ws_suffix, case.ws_path).await;
let ticket_id = TicketBuilder::new(board(), ws)
.title("Test Ticket")
.create()
.await
.expect("create_ticket");
let ticket = board()
.get_ticket(&ticket_id)
.await
.expect("get_ticket")
.expect("ticket exists");
transition_ticket(&ticket, case.source, case.target, case.policy, None)
.await
.expect("transition_ticket");
let ws = crate::workspace::get_by_name(case.ws_suffix)
.await
.expect("get_by_name")
.expect("workspace exists");
assert!(
!ws.paused,
"case {}: workspace should NOT be paused after transition to {:?}",
case.name, case.target,
);
}
}
#[tokio::test]
async fn notify_ticket_failed_transition_does_not_panic() {
let ws = setup_ticket_to_done_test("failed_notify_test").await;
let ticket_id = TicketBuilder::new(board(), ws)
.title("Failed Notify Test")
.create()
.await
.expect("create_ticket");
let _ = board()
.add_comment(&ticket_id, SYSTEM_ROLE, "❌ Test failure detail")
.await;
let ticket = board()
.get_ticket(&ticket_id)
.await
.expect("get_ticket")
.expect("ticket exists");
transition_ticket(
&ticket,
DEFAULT_TICKET_PHASE,
TicketPhase::Failed,
NotifyPolicy::Notify,
None,
)
.await
.expect("transition to Failed");
}
#[tokio::test]
async fn notify_ticket_non_failed_transition_does_not_panic() {
let ws = setup_ticket_to_done_test("non_failed_notify_test").await;
let ticket_id = TicketBuilder::new(board(), ws)
.title("Non-Failed Notify Test")
.phase(TicketPhase::Backlog)
.create()
.await
.expect("create_ticket");
let ticket = board()
.get_ticket(&ticket_id)
.await
.expect("get_ticket")
.expect("ticket exists");
transition_ticket(
&ticket,
TicketPhase::Backlog,
TicketPhase::Analysis,
NotifyPolicy::Notify,
None,
)
.await
.expect("transition to Analysis");
}
#[tokio::test]
async fn sanitation_breaker_counts_failures() {
init_management_test_stores().await;
let ws = test_ws_named("/tmp/test", "san_breaker_test");
let ticket_id = TicketBuilder::new(board(), ws)
.title("Sanitation Breaker Test")
.phase(TicketPhase::InSanitation)
.create()
.await
.expect("create ticket");
for _ in 0..2 {
let _ = board()
.add_comment(
&ticket_id,
SYSTEM_ROLE,
&format!("{SANITATION_FAILED_PREFIX} — garbage files: 1"),
)
.await;
}
let ticket = board()
.get_ticket(&ticket_id)
.await
.expect("get_ticket")
.expect("ticket exists");
assert!(
!run_circuit_breaker(
&ticket,
TicketPhase::InSanitation,
SANITATION_CIRCUIT_BREAKER_THRESHOLD,
count_sanitation_failures,
sanitation_breaker_comment,
"Sanitation",
)
.await,
"Should NOT trip with 2 failures (threshold: 3)"
);
let _ = board()
.add_comment(
&ticket_id,
SYSTEM_ROLE,
&format!("{SANITATION_FAILED_PREFIX} — garbage files: 1"),
)
.await;
let _ = board()
.add_comment(
&ticket_id,
SYSTEM_ROLE,
&format!("{SANITATION_FAILED_PREFIX} — garbage files: 1"),
)
.await;
let ticket = board()
.get_ticket(&ticket_id)
.await
.expect("get_ticket")
.expect("ticket exists");
let tripped = run_circuit_breaker(
&ticket,
TicketPhase::InSanitation,
SANITATION_CIRCUIT_BREAKER_THRESHOLD,
count_sanitation_failures,
sanitation_breaker_comment,
"Sanitation",
)
.await;
assert!(tripped, "Should trip with 4 failures (threshold: 3, 4 > 3)");
let status = board()
.get_ticket_status(&ticket_id)
.await
.expect("get_ticket_status")
.expect("ticket exists");
assert_eq!(
status,
TicketPhase::Failed,
"Circuit breaker should transition to Failed"
);
}
fn pass_verdict() -> crate::Verdict {
crate::Verdict {
score: REVIEW_QA_THRESHOLD,
critique: Some("Good work.".into()),
issues_detected: vec![],
}
}
fn fail_verdict() -> crate::Verdict {
crate::Verdict {
score: 3,
critique: Some("Missing error handling.".into()),
issues_detected: vec!["No timeout check".into()],
}
}
#[tokio::test]
#[allow(clippy::too_many_lines)]
async fn process_verdict_results_cases() {
struct Case {
name: &'static str,
ws_suffix: &'static str,
title: &'static str,
phase: TicketPhase,
results: Vec<ParallelVerdict>,
vi: VerifierInfo,
expected_status: TicketPhase,
expected_pipeline_reservation: bool,
}
init_management_test_stores().await;
let cases = vec![
Case {
name: "all failed -> Failed",
ws_suffix: "vp_all_fail",
title: "VP All Failed",
phase: TicketPhase::InReview,
results: vec![
ParallelVerdict {
response: String::new(),
verdict: None,
},
ParallelVerdict {
response: String::new(),
verdict: None,
},
ParallelVerdict {
response: String::new(),
verdict: None,
},
],
vi: REVIEWER_VI,
expected_status: TicketPhase::Failed,
expected_pipeline_reservation: false,
},
Case {
name: "any failed -> bounce-back with pipeline reservation",
ws_suffix: "vp_any_fail",
title: "VP Any Failed",
phase: TicketPhase::InReview,
results: vec![
ParallelVerdict {
response: "Good.".into(),
verdict: Some(pass_verdict()),
},
ParallelVerdict {
response: "Issues found.".into(),
verdict: Some(fail_verdict()),
},
ParallelVerdict {
response: "Looks fine.".into(),
verdict: Some(pass_verdict()),
},
],
vi: REVIEWER_VI,
expected_status: TicketPhase::ReadyForDevelopment,
expected_pipeline_reservation: true,
},
Case {
name: "all passed -> Reviewed",
ws_suffix: "vp_all_pass",
title: "VP All Pass",
phase: TicketPhase::InReview,
results: vec![
ParallelVerdict {
response: "Good.".into(),
verdict: Some(pass_verdict()),
},
ParallelVerdict {
response: "Fine.".into(),
verdict: Some(pass_verdict()),
},
ParallelVerdict {
response: "OK.".into(),
verdict: Some(pass_verdict()),
},
],
vi: REVIEWER_VI,
expected_status: TicketPhase::Reviewed,
expected_pipeline_reservation: false,
},
Case {
name: "all passed (QA) -> QaPassed",
ws_suffix: "vp_qa_pass",
title: "VP QA Pass",
phase: TicketPhase::InQa,
results: vec![
ParallelVerdict {
response: "QA pass.".into(),
verdict: Some(pass_verdict()),
},
ParallelVerdict {
response: "OK.".into(),
verdict: Some(pass_verdict()),
},
ParallelVerdict {
response: "Good.".into(),
verdict: Some(pass_verdict()),
},
],
vi: QA_VI,
expected_status: TicketPhase::QaPassed,
expected_pipeline_reservation: false,
},
];
for case in &cases {
let ws = test_ws_named("/tmp/test", case.ws_suffix);
let ticket_id = TicketBuilder::new(board(), ws)
.title(case.title)
.phase(case.phase)
.create()
.await
.expect("create_ticket");
let ticket = expect_ticket(board(), &ticket_id).await;
process_verdict_results(&ticket, &case.results, case.vi).await;
let ticket = expect_ticket(board(), &ticket_id).await;
assert_eq!(
ticket.status, case.expected_status,
"case {}: expected status {:?}, got {:?}",
case.name, case.expected_status, ticket.status,
);
assert_eq!(
ticket.pipeline_reservation, case.expected_pipeline_reservation,
"case {}: expected pipeline_reservation={}, got {}",
case.name, case.expected_pipeline_reservation, ticket.pipeline_reservation,
);
}
}
#[tokio::test]
async fn circuit_breaker_comment_boundary() {
struct Case {
name: &'static str,
ws_suffix: &'static str,
title: &'static str,
comment_count: usize,
expected_trip: bool,
expected_status: TicketPhase,
}
init_management_test_stores().await;
let cases = [
Case {
name: "> threshold trips",
ws_suffix: "cb_thresh",
title: "CB Threshold",
comment_count: CIRCUIT_BREAKER_COMMENT_THRESHOLD + 1,
expected_trip: true,
expected_status: TicketPhase::Failed,
},
Case {
name: "= threshold does not trip",
ws_suffix: "cb_no_trip",
title: "CB No Trip",
comment_count: CIRCUIT_BREAKER_COMMENT_THRESHOLD,
expected_trip: false,
expected_status: TicketPhase::InReview,
},
];
for case in &cases {
let ws = test_ws_named("/tmp/test", case.ws_suffix);
let ticket_id = TicketBuilder::new(board(), ws)
.title(case.title)
.phase(TicketPhase::InReview)
.create()
.await
.expect("create_ticket");
for i in 0..case.comment_count {
board()
.add_comment(&ticket_id, "user", &format!("Comment {i}"))
.await
.expect("add_comment");
}
let ticket = expect_ticket(board(), &ticket_id).await;
let tripped = run_circuit_breaker(
&ticket,
TicketPhase::InReview,
CIRCUIT_BREAKER_COMMENT_THRESHOLD,
<[TicketComment]>::len,
general_breaker_comment,
"test",
)
.await;
assert_eq!(
tripped, case.expected_trip,
"case {}: expected trip={}, got tripped={}",
case.name, case.expected_trip, tripped,
);
let status = expect_ticket_status(board(), &ticket_id).await;
assert_eq!(
status, case.expected_status,
"case {}: expected status {:?}, got {:?}",
case.name, case.expected_status, status,
);
}
}
#[tokio::test]
async fn circuit_breaker_guard_prevents_retrip() {
init_management_test_stores().await;
let ws = test_ws_named("/tmp/test", "cb_guard");
let ticket_id = TicketBuilder::new(board(), ws)
.title("CB Guard")
.phase(TicketPhase::InReview)
.create()
.await
.expect("create_ticket");
for i in 0..=CIRCUIT_BREAKER_COMMENT_THRESHOLD {
board()
.add_comment(&ticket_id, "user", &format!("Comment {i}"))
.await
.expect("add_comment");
}
let ticket = board()
.get_ticket(&ticket_id)
.await
.expect("get_ticket")
.expect("ticket exists");
let tripped = run_circuit_breaker(
&ticket,
TicketPhase::InReview,
CIRCUIT_BREAKER_COMMENT_THRESHOLD,
<[TicketComment]>::len,
general_breaker_comment,
"test",
)
.await;
assert!(tripped, "breaker should trip");
let status = board()
.get_ticket_status(&ticket_id)
.await
.expect("get_ticket_status")
.expect("ticket exists");
assert_eq!(
status,
TicketPhase::Failed,
"ticket should be Failed after trip"
);
let comments = board()
.get_comments(&ticket_id)
.await
.expect("get_comments");
let has_marker = comments
.iter()
.any(|c| c.content.to_lowercase().contains("circuit breaker"));
assert!(
has_marker,
"trip comment must contain circuit breaker marker"
);
let ticket = board()
.get_ticket(&ticket_id)
.await
.expect("get_ticket")
.expect("ticket exists");
let guarded = guard_phase_and_circuit_breaker(&ticket, TicketPhase::InReview, "test").await;
assert!(
!guarded,
"phase guard must reject re-trip (ticket is now Failed)"
);
let status = board()
.get_ticket_status(&ticket_id)
.await
.expect("get_ticket_status")
.expect("ticket exists");
assert_eq!(status, TicketPhase::Failed, "ticket must remain Failed");
}
#[tokio::test]
#[allow(clippy::too_many_lines)]
async fn handle_analyst_verdicts_cases() {
struct Case {
name: &'static str,
ws_suffix: &'static str,
title: &'static str,
results: Vec<ParallelVerdict>,
expected_comment_substring: &'static str,
}
init_management_test_stores().await;
let cases = vec![
Case {
name: "all pass -> Planning with LGTM",
ws_suffix: "an_all_pass",
title: "Analyst All Pass",
results: vec![
ParallelVerdict {
response: "Analysis A".into(),
verdict: Some(crate::Verdict {
score: 10,
critique: Some("Great analysis.".into()),
issues_detected: vec![],
}),
},
ParallelVerdict {
response: "Analysis B".into(),
verdict: Some(crate::Verdict {
score: 9,
critique: Some("Solid work.".into()),
issues_detected: vec![],
}),
},
ParallelVerdict {
response: "Analysis C".into(),
verdict: Some(crate::Verdict {
score: 8,
critique: Some("Good analysis.".into()),
issues_detected: vec![],
}),
},
],
expected_comment_substring: "All LGTM",
},
Case {
name: "partial fail -> Planning with blockers",
ws_suffix: "an_partial",
title: "Analyst Partial Fail",
results: vec![
ParallelVerdict {
response: "Analysis A".into(),
verdict: Some(crate::Verdict {
score: 10,
critique: Some("Great.".into()),
issues_detected: vec![],
}),
},
ParallelVerdict {
response: "Analysis B".into(),
verdict: Some(crate::Verdict {
score: 3,
critique: Some("Poor analysis.".into()),
issues_detected: vec!["Missing data".into()],
}),
},
ParallelVerdict {
response: "Analysis C".into(),
verdict: Some(crate::Verdict {
score: 8,
critique: Some("Decent.".into()),
issues_detected: vec!["Minor issue".into()],
}),
},
],
expected_comment_substring: "blockers",
},
Case {
name: "no verdicts -> Planning with no analysis",
ws_suffix: "an_no_v",
title: "Analyst No Verdicts",
results: vec![
ParallelVerdict {
response: String::new(),
verdict: None,
},
ParallelVerdict {
response: String::new(),
verdict: None,
},
ParallelVerdict {
response: String::new(),
verdict: None,
},
],
expected_comment_substring: "no analysis",
},
];
for case in &cases {
let ws = test_ws_named("/tmp/test", case.ws_suffix);
let ticket_id = TicketBuilder::new(board(), ws)
.title(case.title)
.phase(TicketPhase::Analysis)
.create()
.await
.expect("create_ticket");
let ticket = expect_ticket(board(), &ticket_id).await;
handle_analyst_verdicts(&ticket, &case.results).await;
let status = expect_ticket_status(board(), &ticket_id).await;
assert_eq!(
status,
TicketPhase::Planning,
"case {}: expected Planning, got {:?}",
case.name,
status,
);
let comments = board()
.get_comments(&ticket_id)
.await
.expect("get_comments");
assert_eq!(
comments.len(),
4,
"case {}: expected 4 comments (3 per-analyst + 1 system summary), got {}",
case.name,
comments.len(),
);
let system = comments.iter().find(|c| c.role == SYSTEM_ROLE);
assert!(
system.is_some(),
"case {}: system summary comment should exist",
case.name,
);
assert!(
system
.unwrap()
.content
.contains(case.expected_comment_substring),
"case {}: system comment should contain {:?}, got: {}",
case.name,
case.expected_comment_substring,
system.unwrap().content,
);
}
}
#[tokio::test]
async fn handle_qa_passed_no_git_to_done() {
init_management_test_stores().await;
let ws = test_ws_named("/nonexistent/mahbot-test-qa-no-git", "qa_no_git");
let ticket_id = TicketBuilder::new(board(), ws.clone())
.title("QA No Git")
.phase(TicketPhase::QaPassed)
.create()
.await
.expect("create_ticket");
let ticket = board()
.get_ticket(&ticket_id)
.await
.expect("get_ticket")
.expect("ticket exists");
handle_qa_passed(ticket, ws).await;
let status = board()
.get_ticket_status(&ticket_id)
.await
.expect("get_ticket_status")
.expect("ticket exists");
assert_eq!(
status,
TicketPhase::Done,
"QA passed should eventually transition to Done"
);
}
#[tokio::test]
async fn handle_qa_passed_untracked_files_to_insanitation() {
if !crate::diff_parse::git_is_installed().await {
eprintln!("git not installed — skipping git-dependent test");
return;
}
init_management_test_stores().await;
let (_dir, repo_path) = crate::util::test::init_temp_repo();
std::fs::write(repo_path.join("untracked.txt"), b"garbage").expect("write untracked file");
let ws = test_ws_named(repo_path.to_str().unwrap(), "qa_untracked");
let ticket_id = TicketBuilder::new(board(), ws.clone())
.title("QA Untracked")
.phase(TicketPhase::QaPassed)
.create()
.await
.expect("create_ticket");
let ticket = board()
.get_ticket(&ticket_id)
.await
.expect("get_ticket")
.expect("ticket exists");
handle_qa_passed(ticket, ws).await;
let status = board()
.get_ticket_status(&ticket_id)
.await
.expect("get_ticket_status")
.expect("ticket exists");
assert_eq!(
status,
TicketPhase::InSanitation,
"QA passed with untracked files should transition to InSanitation"
);
let ticket = board()
.get_ticket(&ticket_id)
.await
.expect("get_ticket")
.expect("ticket exists");
let expected_key =
crate::session::ticket_session_key(&ticket_id, crate::Role::Sanitation.as_str());
assert_eq!(
ticket.assigned_to.as_deref(),
Some(expected_key.as_str()),
"assigned_to should be set to sanitation session key"
);
}
}