pub mod breakers;
pub mod fixloop;
pub mod git;
pub mod providers;
use std::collections::{BTreeMap, BTreeSet, VecDeque};
use std::path::{Path, PathBuf};
use std::sync::Mutex;
use std::time::{Duration, Instant};
use octl_core::plan::{self, Acceptance, Chunk, Plan, Tier};
use serde::Serialize;
use crate::error::CliError;
use crate::floor::{
self, evaluate_floor, BaselineSnapshot, CheckRun, FloorInputs, FloorVerdict, RunSnapshot,
};
use crate::harness::{
CancelToken, Check as HarnessCheck, ChunkOutcome, ChunkRequest, CodeHarness, Usage,
};
use crate::output::{self, OutputFormat, OutputSpec};
use crate::pipeline::{
route_proposal, Action, ChunkState, ChunkStatus, Coordinator, CoordinatorProposal, Decider,
DeciderVerdict, DecisionContext, DecisionEnvelope, DecisionTier, DecisionTrigger, Finding,
FindingVerdict, Severity,
};
use breakers::{failure_fingerprint, ResourceBudget, ResourceMeter};
use fixloop::{next_tier, FixLoopConfig};
use git::MergeOutcome;
use providers::{
SpecContext, SpecProvider, VerifyContext, VerifyDisposition, VerifyJudgment, VerifyProvider,
};
#[derive(Debug, thiserror::Error)]
pub enum PipelineError {
#[error("git error: {0}")]
Git(String),
#[error("setup error: {0}")]
Setup(String),
#[error("spec stage failed: {0}")]
Spec(String),
#[error("plan invalid: {0}")]
PlanInvalid(String),
#[error("verify stage failed: {0}")]
Verify(String),
#[error("floor capture error: {0}")]
Floor(String),
#[error("harness error: {0}")]
Harness(String),
#[error("io error: {0}")]
Io(String),
}
impl PipelineError {
fn stage(stage: &str, message: impl Into<String>) -> Self {
match stage {
"spec" => PipelineError::Spec(message.into()),
_ => PipelineError::Verify(message.into()),
}
}
fn with_note(self, note: impl AsRef<str>) -> Self {
let n = note.as_ref();
match self {
PipelineError::Git(m) => PipelineError::Git(format!("{m} — {n}")),
PipelineError::Setup(m) => PipelineError::Setup(format!("{m} — {n}")),
PipelineError::Spec(m) => PipelineError::Spec(format!("{m} — {n}")),
PipelineError::PlanInvalid(m) => PipelineError::PlanInvalid(format!("{m} — {n}")),
PipelineError::Verify(m) => PipelineError::Verify(format!("{m} — {n}")),
PipelineError::Floor(m) => PipelineError::Floor(format!("{m} — {n}")),
PipelineError::Harness(m) => PipelineError::Harness(format!("{m} — {n}")),
PipelineError::Io(m) => PipelineError::Io(format!("{m} — {n}")),
}
}
fn code(&self) -> &'static str {
match self {
PipelineError::Git(_) => "git_error",
PipelineError::Setup(_) => "setup_error",
PipelineError::Spec(_) => "spec_failed",
PipelineError::PlanInvalid(_) => "plan_invalid",
PipelineError::Verify(_) => "verify_failed",
PipelineError::Floor(_) => "floor_error",
PipelineError::Harness(_) => "harness_error",
PipelineError::Io(_) => "io_error",
}
}
}
impl From<floor::FloorError> for PipelineError {
fn from(e: floor::FloorError) -> Self {
PipelineError::Floor(e.to_string())
}
}
impl From<PipelineError> for CliError {
fn from(e: PipelineError) -> Self {
let code = e.code();
match e {
PipelineError::PlanInvalid(_) | PipelineError::Setup(_) => {
CliError::user(code, e.to_string())
}
_ => CliError::system(code, e.to_string()),
}
}
}
#[derive(Debug)]
pub struct PipelineFailure {
pub error: PipelineError,
pub report: Option<Box<PipelineReport>>,
}
impl std::fmt::Display for PipelineFailure {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
self.error.fmt(f)
}
}
impl std::error::Error for PipelineFailure {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
Some(&self.error)
}
}
impl From<PipelineError> for PipelineFailure {
fn from(error: PipelineError) -> Self {
PipelineFailure {
error,
report: None,
}
}
}
pub struct PipelineConfig {
pub repo: PathBuf,
pub intent: String,
pub source_branch: String,
pub files: Vec<PathBuf>,
pub slug: Option<String>,
pub test_cmd: String,
pub clippy_cmd: String,
pub workdir: PathBuf,
pub file_scope_slack: usize,
pub keep: bool,
pub chunk_timeout: Option<Duration>,
pub max_build_concurrency: usize,
pub fix_loop: FixLoopConfig,
pub budget: ResourceBudget,
}
#[allow(clippy::trivially_copy_pass_by_ref)]
fn is_false(b: &bool) -> bool {
!*b
}
#[derive(Debug, Clone, Serialize)]
pub struct ChunkReport {
pub id: String,
pub title: String,
pub tier: String,
pub outcome: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub floor_passed: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub floor: Option<FloorVerdict>,
pub merged: bool,
#[serde(skip_serializing_if = "Option::is_none")]
pub commit: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub merge_commit: Option<String>,
#[serde(skip_serializing_if = "is_false", default)]
pub replayed: bool,
#[serde(skip_serializing_if = "Option::is_none")]
pub reason: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub branch_preserved: Option<String>,
}
#[derive(Debug, Clone, Serialize)]
pub struct VerifyReport {
pub acceptance_checks_passed: bool,
pub judged_passed: bool,
pub passed: bool,
pub summary: String,
pub findings: Vec<String>,
}
#[derive(Debug, Clone, Serialize)]
pub struct PipelineReport {
pub slug: String,
pub source_branch: String,
pub integration_branch: String,
pub intent_rev: u32,
pub plan_rev: u32,
pub chunk_count: usize,
pub chunks: Vec<ChunkReport>,
#[serde(skip_serializing_if = "Option::is_none")]
pub verify: Option<VerifyReport>,
#[serde(skip_serializing_if = "Option::is_none")]
pub feature_floor: Option<FloorVerdict>,
pub merged: bool,
#[serde(skip_serializing_if = "Option::is_none")]
pub final_commit: Option<String>,
pub status: String,
pub decisions: Vec<DecisionEnvelope>,
pub recode_count: u32,
pub promote_count: u32,
pub respec_count: u32,
#[serde(skip_serializing_if = "Option::is_none")]
pub circuit_breaker: Option<String>,
pub resources: ResourceMeter,
#[serde(skip_serializing_if = "Option::is_none")]
pub failure: Option<String>,
}
#[must_use]
pub fn slugify(intent: &str) -> String {
let seed = intent.lines().find(|l| !l.trim().is_empty()).unwrap_or("");
let mut slug = String::new();
let mut prev_hyphen = false;
for ch in seed.chars() {
if ch.is_ascii_alphanumeric() {
slug.push(ch.to_ascii_lowercase());
prev_hyphen = false;
} else if !prev_hyphen && !slug.is_empty() {
slug.push('-');
prev_hyphen = true;
}
}
let slug = slug.trim_matches('-');
let slug: String = slug.chars().take(48).collect();
let slug = slug.trim_matches('-').to_string();
if slug.is_empty() {
"feature".to_string()
} else {
slug
}
}
pub fn resolve_intent(raw: &str) -> Result<String, PipelineError> {
let text = if let Some(path) = raw.strip_prefix('@') {
std::fs::read_to_string(path)
.map_err(|e| PipelineError::Setup(format!("could not read intent file {path}: {e}")))?
} else if Path::new(raw).is_file() {
std::fs::read_to_string(raw)
.map_err(|e| PipelineError::Setup(format!("could not read intent file {raw}: {e}")))?
} else {
raw.to_string()
};
if text.trim().is_empty() {
return Err(PipelineError::Setup("intent is empty".to_string()));
}
Ok(text)
}
fn topo_order(chunks: &[Chunk]) -> Vec<usize> {
use std::collections::HashMap;
let index: HashMap<&str, usize> = chunks
.iter()
.enumerate()
.map(|(i, c)| (c.id.as_str(), i))
.collect();
let mut done = vec![false; chunks.len()];
let mut order = Vec::with_capacity(chunks.len());
while order.len() < chunks.len() {
let mut progressed = false;
for (i, c) in chunks.iter().enumerate() {
if done[i] {
continue;
}
let ready = c
.deps
.iter()
.all(|d| index.get(d.as_str()).is_some_and(|&j| done[j]));
if ready {
done[i] = true;
order.push(i);
progressed = true;
}
}
if !progressed {
for (i, _) in chunks.iter().enumerate() {
if !done[i] {
order.push(i);
done[i] = true;
}
}
}
}
order
}
fn to_harness_check(i: usize, c: &plan::Check) -> HarnessCheck {
HarnessCheck {
id: format!("chk-{i}"),
desc: c.desc.clone(),
run: c.run.clone(),
timeout: None,
}
}
fn capture_snapshot(cfg: &PipelineConfig, dir: &Path) -> Result<RunSnapshot, PipelineError> {
let alloc = |what| {
tempfile::TempDir::new().map_err(move |e| {
PipelineError::from(floor::FloorError::Capture {
what,
message: format!("could not allocate a floor target dir: {e}"),
})
})
};
let test_target_dir = alloc("tests")?;
let clippy_target_dir = alloc("clippy")?;
let meta = if dir.join("Cargo.toml").exists() {
let m = floor::metadata::load(dir)?;
floor::metadata::reject_forged_harness(&m)?;
Some(m)
} else {
None
};
let mut tests =
floor::runner::capture_test_snapshot(&cfg.test_cmd, dir, test_target_dir.path())?;
if let Some(meta) = &meta {
floor::metadata::verify_enumeration(meta, &tests.targets)?;
floor::runner::capture_doctests(dir, test_target_dir.path(), meta, &mut tests)?;
}
let clippy =
floor::runner::capture_clippy_snapshot(&cfg.clippy_cmd, dir, clippy_target_dir.path())?;
Ok(RunSnapshot {
tests,
clippy,
coverage: None,
})
}
fn envelope(
actor: &str,
tier: DecisionTier,
reason: impl Into<String>,
inputs: Vec<String>,
model: impl Into<String>,
prompt_version: impl Into<String>,
) -> DecisionEnvelope {
DecisionEnvelope {
actor: actor.to_string(),
input_artifacts: inputs,
reason: reason.into(),
decision_tier: tier,
model: model.into(),
prompt_version: prompt_version.into(),
}
}
pub trait TierHarness {
fn harness(&self, tier: Tier) -> &dyn CodeHarness;
fn next_tier(&self, tier: Tier) -> Option<Tier> {
next_tier(tier)
}
}
pub struct SingleTierHarness<'a>(pub &'a dyn CodeHarness);
impl TierHarness for SingleTierHarness<'_> {
fn harness(&self, _tier: Tier) -> &dyn CodeHarness {
self.0
}
fn next_tier(&self, _tier: Tier) -> Option<Tier> {
None
}
}
struct LiveTierHarness {
code: crate::harness::claude::ClaudeHarness,
mid: crate::harness::claude::ClaudeHarness,
high: crate::harness::claude::ClaudeHarness,
}
impl TierHarness for LiveTierHarness {
fn harness(&self, tier: Tier) -> &dyn CodeHarness {
match tier {
Tier::Code => &self.code,
Tier::Mid => &self.mid,
Tier::High => &self.high,
}
}
}
struct LiveCoordinator;
impl Coordinator for LiveCoordinator {
fn coordinate(&self, _ctx: &DecisionContext) -> Vec<CoordinatorProposal> {
Vec::new()
}
fn model(&self) -> String {
"coordinator".to_string()
}
fn prompt_version(&self) -> String {
"v1".to_string()
}
}
static LIVE_COORDINATOR: LiveCoordinator = LiveCoordinator;
struct LiveDecider {
model: String,
}
impl Decider for LiveDecider {
fn decide_consequential(
&self,
_ctx: &DecisionContext,
proposed: &CoordinatorProposal,
) -> DeciderVerdict {
DeciderVerdict {
action: proposed.action.clone(),
reason: proposed.reason.clone(),
input_artifacts: proposed.input_artifacts.clone(),
}
}
fn model(&self) -> String {
self.model.clone()
}
fn prompt_version(&self) -> String {
"v1".to_string()
}
}
fn live_decision_ctx(run: &Run, plan: &Plan, trigger: DecisionTrigger) -> DecisionContext {
let chunks = plan
.chunks
.iter()
.map(|c| {
let status = match run.chunk_status.get(&c.id) {
Some(LiveChunkStatus::Merged) => ChunkStatus::AwaitingVerify,
_ => ChunkStatus::Pending,
};
let tier = run.chunk_tier.get(&c.id).copied().unwrap_or(c.tier);
(c.id.clone(), ChunkState { status, tier })
})
.collect();
DecisionContext {
run_id: format!("pipeline-{}", run.slug),
plan_rev: plan.plan_rev,
intent_rev: plan.intent_rev,
chunks,
trigger,
}
}
struct Run<'a> {
cfg: &'a PipelineConfig,
coordinator: &'a dyn Coordinator,
decider: &'a dyn Decider,
repo: PathBuf,
slug: String,
integration_branch: String,
integration_wt: PathBuf,
fork_commit: String,
decisions: Vec<DecisionEnvelope>,
chunk_reports: Vec<ChunkReport>,
feature_floor: Option<FloorVerdict>,
code_block_status: Option<&'static str>,
rollback_conflict: Option<String>,
chunk_status: BTreeMap<String, LiveChunkStatus>,
chunk_tier: BTreeMap<String, Tier>,
chunk_promotions: BTreeMap<String, u32>,
chunk_provenance: BTreeMap<String, ChunkProvenance>,
merge_seq: u64,
chunk_recode_total: BTreeMap<(u32, String, &'static str), u32>,
preserved: Vec<(PathBuf, String)>,
recode_count: u32,
promote_count: u32,
respec_count: u32,
circuit_breaker: Option<String>,
meter: ResourceMeter,
started: Instant,
merged_to_source: bool,
}
#[derive(Debug, Clone)]
struct ChunkProvenance {
base: String,
commit: String,
order: u64,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum LiveChunkStatus {
Pending,
Merged,
}
impl Drop for Run<'_> {
fn drop(&mut self) {
teardown(self);
}
}
pub fn run_pipeline(
cfg: &PipelineConfig,
spec: &dyn SpecProvider,
code: &dyn CodeHarness,
verify: &dyn VerifyProvider,
) -> Result<PipelineReport, PipelineFailure> {
let resolver = SingleTierHarness(code);
let decider = crate::pipeline::ScriptedDecider::confirming();
run_pipeline_tiered(cfg, spec, &resolver, verify, &decider)
}
pub fn run_pipeline_tiered(
cfg: &PipelineConfig,
spec: &dyn SpecProvider,
harnesses: &dyn TierHarness,
verify: &dyn VerifyProvider,
decider: &dyn Decider,
) -> Result<PipelineReport, PipelineFailure> {
let repo = git::toplevel(&cfg.repo)?;
if !git::branch_exists(&repo, &cfg.source_branch) {
return Err(PipelineError::Setup(format!(
"source `{}` is not a local branch (tags, remotes, and HEAD are rejected)",
cfg.source_branch
))
.into());
}
let source_commit = git::resolve_commit(&repo, &cfg.source_branch)?;
let slug = cfg
.slug
.as_deref()
.map_or_else(|| slugify(&cfg.intent), slugify);
let integration_branch = format!("feat/{slug}");
if git::branch_exists(&repo, &integration_branch) {
return Err(PipelineError::Setup(format!(
"integration branch `{integration_branch}` already exists; refusing to reuse it"
))
.into());
}
std::fs::create_dir_all(&cfg.workdir).map_err(|e| {
PipelineError::Io(format!(
"could not create workdir {}: {e}",
cfg.workdir.display()
))
})?;
std::fs::write(cfg.workdir.join("intent.md"), &cfg.intent)
.map_err(|e| PipelineError::Io(format!("could not write intent.md: {e}")))?;
git::create_branch(&repo, &integration_branch, &source_commit)?;
let integration_wt = cfg.workdir.join("integration");
let mut run = Run {
cfg,
coordinator: &LIVE_COORDINATOR,
decider,
repo: repo.clone(),
slug: slug.clone(),
integration_branch: integration_branch.clone(),
integration_wt: integration_wt.clone(),
fork_commit: source_commit.clone(),
decisions: Vec::new(),
chunk_reports: Vec::new(),
feature_floor: None,
code_block_status: None,
rollback_conflict: None,
chunk_status: BTreeMap::new(),
chunk_tier: BTreeMap::new(),
chunk_promotions: BTreeMap::new(),
chunk_provenance: BTreeMap::new(),
merge_seq: 0,
chunk_recode_total: BTreeMap::new(),
preserved: Vec::new(),
recode_count: 0,
promote_count: 0,
respec_count: 0,
circuit_breaker: None,
meter: ResourceMeter::new(),
started: Instant::now(),
merged_to_source: false,
};
git::worktree_add(&repo, &integration_wt, &integration_branch)?;
verify_capture_ref(&integration_wt, &run.fork_commit)?;
let baseline_snapshot = capture_snapshot(cfg, &integration_wt)?;
let baseline = BaselineSnapshot::new(
format!("{integration_branch}@fork"),
run.fork_commit.clone(),
floor::runner::rustc_version(&integration_wt),
baseline_snapshot,
);
let mut plan =
produce_and_validate_plan(&mut run, spec, &baseline.to_plan_baseline(), 1, None)?;
gate_plan_baseline(&baseline, &plan)?;
git::restore_to(&run.integration_wt, &run.fork_commit)?;
write_plan(&run, &plan)?;
run.decisions.push(envelope(
"spec",
DecisionTier::Decider,
format!("produced plan with {} chunk(s)", plan.chunks.len()),
vec![
format!("intent_rev:1"),
format!("baseline:{}", baseline.r#ref),
],
spec.model(),
spec.prompt_version(),
));
run.chunk_status = plan
.chunks
.iter()
.map(|c| (c.id.clone(), LiveChunkStatus::Pending))
.collect();
run.chunk_tier = plan.chunks.iter().map(|c| (c.id.clone(), c.tier)).collect();
let mut pending_findings: BTreeMap<String, Vec<String>> = BTreeMap::new();
let mut pending_prior_diff: BTreeMap<String, String> = BTreeMap::new();
let body: Result<PipelineReport, PipelineError> = (|| {
let mut fix_iter = 0u32;
let outcome = loop {
refresh_storage(&mut run);
if let Some(msg) = resource_breach(&run) {
run.circuit_breaker = Some(msg);
break LoopExit::Terminal {
verify: None,
status: "circuit_breaker",
};
}
run_code_stage(
&mut run,
&plan,
harnesses,
&baseline,
&pending_findings,
&pending_prior_diff,
)?;
pending_findings.clear();
pending_prior_diff.clear();
if run.circuit_breaker.is_some() {
break LoopExit::Terminal {
verify: None,
status: "circuit_breaker",
};
}
if !all_merged(&run, &plan) {
let status = run.code_block_status.unwrap_or("chunk_failed");
break LoopExit::Terminal {
verify: None,
status,
};
}
let feat_tip = git::head(&run.integration_wt)?;
let (verify_report, disposition) = run_verify_stage(&mut run, &plan, verify)?;
git::restore_to(&run.integration_wt, &feat_tip)?;
if let Some(msg) = resource_breach(&run) {
run.circuit_breaker = Some(msg);
break LoopExit::Terminal {
verify: Some(verify_report),
status: "circuit_breaker",
};
}
if verify_report.passed {
break LoopExit::Converged {
verify: verify_report,
feat_tip,
};
}
if fix_iter >= run.cfg.fix_loop.max_fix_iterations {
let status = if run.cfg.fix_loop.max_fix_iterations == 0 {
"verify_failed"
} else {
run.circuit_breaker = Some(format!(
"verify still failing after {} fix iteration(s)",
run.cfg.fix_loop.max_fix_iterations
));
"circuit_breaker"
};
break LoopExit::Terminal {
verify: Some(verify_report),
status,
};
}
fix_iter += 1;
match disposition {
VerifyDisposition::Fix | VerifyDisposition::FixChunks { .. } => {
let targets = resolve_fix_targets(&disposition, &plan, &run);
if targets.is_empty() {
break LoopExit::Terminal {
verify: Some(verify_report),
status: "verify_failed",
};
}
let seeds: BTreeSet<String> = targets.iter().cloned().collect();
let affected = dependent_closure(&plan, &seeds);
let keep: BTreeSet<String> = run
.chunk_status
.iter()
.filter(|(id, s)| {
**s == LiveChunkStatus::Merged && !affected.contains(id.as_str())
})
.map(|(id, _)| id.clone())
.collect();
let mut captured_diffs: BTreeMap<String, String> = BTreeMap::new();
for id in &targets {
if let Some(prov) = run.chunk_provenance.get(id) {
if let Ok(d) = git::diff(&run.integration_wt, &prov.base, &prov.commit)
{
captured_diffs.insert(id.clone(), d);
}
}
}
match rebuild_integration(&mut run, &keep)? {
RebuildOutcome::Rebuilt => {}
RebuildOutcome::Conflict { chunk_id } => {
run.rollback_conflict = Some(chunk_id);
break LoopExit::Terminal {
verify: Some(verify_report),
status: "rollback_conflict",
};
}
}
run.chunk_reports
.retain(|r| !affected.contains(r.id.as_str()));
for id in &targets {
record_recode_decision(
&mut run,
&plan,
id,
&verify_report.findings,
"verify FIX",
);
pending_findings.insert(id.clone(), verify_report.findings.clone());
if let Some(d) = captured_diffs.remove(id) {
pending_prior_diff.insert(id.clone(), d);
}
}
for id in &affected {
run.chunk_status
.insert(id.clone(), LiveChunkStatus::Pending);
}
}
VerifyDisposition::SpecFlaw { reason, chunk_ids } => {
if run.respec_count >= run.cfg.fix_loop.max_respec {
let status = if run.cfg.fix_loop.max_respec == 0 {
"verify_failed"
} else {
run.circuit_breaker = Some(format!(
"re-spec budget exhausted after {} re-spec(s)",
run.cfg.fix_loop.max_respec
));
"circuit_breaker"
};
break LoopExit::Terminal {
verify: Some(verify_report),
status,
};
}
match trigger_re_spec(
&mut run,
spec,
&plan,
&reason,
&chunk_ids,
&verify_report.findings,
&baseline,
)? {
ReSpecOutcome::Replanned(new_plan) => plan = *new_plan,
ReSpecOutcome::Escalated => {
break LoopExit::Terminal {
verify: Some(verify_report),
status: "escalated",
};
}
ReSpecOutcome::RollbackConflict { chunk_id } => {
run.rollback_conflict = Some(chunk_id);
break LoopExit::Terminal {
verify: Some(verify_report),
status: "rollback_conflict",
};
}
}
}
}
};
let (verify_report, feat_tip) = match outcome {
LoopExit::Terminal { verify, status } => {
return Ok(finalize(&run, &plan, verify, false, None, status));
}
LoopExit::Converged { verify, feat_tip } => (verify, feat_tip),
};
let declared: Vec<PathBuf> = union_declared_files(&plan);
let feature_floor = evaluate_feature_floor(&run, &plan, &baseline, &declared, &feat_tip)?;
run.feature_floor = Some(feature_floor.clone());
if !feature_floor.passed() {
return Ok(finalize(
&run,
&plan,
Some(verify_report),
false,
None,
"floor_blocked",
));
}
let converge_ctx = live_decision_ctx(
&run,
&plan,
DecisionTrigger::VerifyReport {
report_id: format!("verify-plan-v{}", plan.plan_rev),
findings: Vec::new(),
},
);
let (converge_action, converge_env) = route_proposal(
run.coordinator,
run.decider,
&converge_ctx,
CoordinatorProposal {
action: Action::DeclareConverged,
reason: "declared converged: verify passed and the feature floor is green"
.to_string(),
input_artifacts: vec![
format!("feat:{feat_tip}"),
format!("source:{source_commit}"),
],
},
);
run.decisions.push(converge_env);
if !matches!(converge_action, Action::DeclareConverged) {
return Ok(finalize(
&run,
&plan,
Some(verify_report),
false,
None,
"escalated",
));
}
match merge_feature_to_source(&run, &feat_tip)? {
MergeOutcome::Merged { commit } => {
run.merged_to_source = true;
run.decisions.push(envelope(
"supervisor",
DecisionTier::Coordinator,
format!("merged {feat_tip} into {}", run.cfg.source_branch),
vec![
format!("feat:{feat_tip}"),
format!("source:{source_commit}"),
],
"supervisor",
"v1",
));
Ok(finalize(
&run,
&plan,
Some(verify_report),
true,
Some(commit),
"merged",
))
}
MergeOutcome::Conflict { details } => {
let mut vr = verify_report;
vr.summary = format!("{} (source merge conflicted: {details})", vr.summary);
Ok(finalize(
&run,
&plan,
Some(vr),
false,
None,
"merge_conflict",
))
}
}
})();
body.map_err(|error| {
let mut report = finalize(&run, &plan, None, false, None, "pipeline_error");
report.failure = Some(error.to_string());
PipelineFailure {
error,
report: Some(Box::new(report)),
}
})
}
enum LoopExit {
Terminal {
verify: Option<VerifyReport>,
status: &'static str,
},
Converged {
verify: VerifyReport,
feat_tip: String,
},
}
fn all_merged(run: &Run, plan: &Plan) -> bool {
plan.chunks
.iter()
.all(|c| run.chunk_status.get(&c.id) == Some(&LiveChunkStatus::Merged))
}
fn resolve_fix_targets(disp: &VerifyDisposition, plan: &Plan, run: &Run) -> Vec<String> {
let is_merged = |id: &str| run.chunk_status.get(id) == Some(&LiveChunkStatus::Merged);
let all_merged: Vec<String> = plan
.chunks
.iter()
.filter(|c| is_merged(&c.id))
.map(|c| c.id.clone())
.collect();
match disp {
VerifyDisposition::FixChunks { chunk_ids } => {
plan.chunks
.iter()
.map(|c| c.id.clone())
.filter(|id| is_merged(id) && chunk_ids.iter().any(|c| c == id))
.collect()
}
_ => all_merged,
}
}
fn dependent_closure(plan: &Plan, seeds: &BTreeSet<String>) -> BTreeSet<String> {
let mut affected = seeds.clone();
loop {
let mut grew = false;
for c in &plan.chunks {
if affected.contains(&c.id) {
continue;
}
if c.deps.iter().any(|d| affected.contains(d)) {
affected.insert(c.id.clone());
grew = true;
}
}
if !grew {
break;
}
}
affected
}
fn record_recode_decision(
run: &mut Run,
plan: &Plan,
chunk_id: &str,
findings: &[String],
source: &str,
) {
let action = Action::ReCodeChunk {
chunk_id: chunk_id.to_string(),
findings: findings
.iter()
.enumerate()
.map(|(i, f)| Finding {
id: format!("{chunk_id}-f{i}"),
summary: f.clone(),
verdict: FindingVerdict::Fix,
severity: Severity::Medium,
})
.collect(),
};
let ctx = live_decision_ctx(
run,
plan,
DecisionTrigger::ChunkCommitted {
chunk_id: chunk_id.to_string(),
},
);
let (_, env) = route_proposal(
run.coordinator,
run.decider,
&ctx,
CoordinatorProposal {
action,
reason: format!("re-code chunk {chunk_id} ({source})"),
input_artifacts: vec![format!("chunk:{chunk_id}")],
},
);
run.recode_count = run.recode_count.saturating_add(1);
run.decisions.push(env);
}
enum ReSpecOutcome {
Replanned(Box<Plan>),
Escalated,
RollbackConflict { chunk_id: String },
}
fn promotion_target(
run: &Run,
harnesses: &dyn TierHarness,
chunk_id: &str,
current_tier: Tier,
) -> Option<Tier> {
let used = run.chunk_promotions.get(chunk_id).copied().unwrap_or(0);
if used >= run.cfg.fix_loop.max_promotions {
return None;
}
harnesses.next_tier(current_tier)
}
fn promote_chunk(run: &mut Run, plan: &Plan, chunk_id: &str, current_tier: Tier, promoted: Tier) {
let ctx = live_decision_ctx(
run,
plan,
DecisionTrigger::ChunkCommitted {
chunk_id: chunk_id.to_string(),
},
);
let (_, env) = route_proposal(
run.coordinator,
run.decider,
&ctx,
CoordinatorProposal {
action: Action::PromoteTier {
chunk_id: chunk_id.to_string(),
tier: promoted,
},
reason: format!(
"promote chunk {chunk_id} {} → {} (repeat-fail)",
current_tier.wire_name(),
promoted.wire_name()
),
input_artifacts: vec![format!("chunk:{chunk_id}")],
},
);
run.decisions.push(env);
run.chunk_tier.insert(chunk_id.to_string(), promoted);
let used = run
.chunk_promotions
.entry(chunk_id.to_string())
.or_insert(0);
*used = used.saturating_add(1);
run.promote_count = run.promote_count.saturating_add(1);
}
fn trigger_re_spec(
run: &mut Run,
spec: &dyn SpecProvider,
old_plan: &Plan,
reason: &str,
forced: &[String],
findings: &[String],
baseline: &BaselineSnapshot,
) -> Result<ReSpecOutcome, PipelineError> {
let new_rev = old_plan.plan_rev.saturating_add(1);
let trigger_findings: Vec<Finding> = findings
.iter()
.enumerate()
.map(|(i, f)| Finding {
id: format!("respec-f{i}"),
summary: f.clone(),
verdict: FindingVerdict::SpecFlaw,
severity: Severity::High,
})
.collect();
let ctx = live_decision_ctx(
run,
old_plan,
DecisionTrigger::VerifyReport {
report_id: format!("respec-v{new_rev}"),
findings: trigger_findings,
},
);
let (action, env) = route_proposal(
run.coordinator,
run.decider,
&ctx,
CoordinatorProposal {
action: Action::TriggerReSpec {
reason: reason.to_string(),
chunk_ids: forced.to_vec(),
},
reason: format!("re-spec to plan.v{new_rev}: {reason}"),
input_artifacts: vec![format!("plan:{}", old_plan.plan_rev)],
},
);
run.decisions.push(env);
let (reason, forced): (String, Vec<String>) = match action {
Action::TriggerReSpec { reason, chunk_ids } => (reason, chunk_ids),
_ => return Ok(ReSpecOutcome::Escalated),
};
let feat_tip = git::head(&run.integration_wt)?;
let old_raw = serde_json::to_value(old_plan)
.map_err(|e| PipelineError::Io(format!("could not serialize prior plan: {e}")))?;
let new_plan = produce_and_validate_plan(
run,
spec,
&baseline.to_plan_baseline(),
new_rev,
Some((&old_raw, reason.as_str())),
)?;
gate_plan_baseline(baseline, &new_plan)?;
git::restore_to(&run.integration_wt, &feat_tip)?;
let merged: BTreeSet<String> = run
.chunk_status
.iter()
.filter(|(_, s)| **s == LiveChunkStatus::Merged)
.map(|(id, _)| id.clone())
.collect();
let diff = fixloop::dag_diff(old_plan, &new_plan, &merged, &forced);
let mut status = BTreeMap::new();
for id in &diff.kept_done {
status.insert(id.clone(), LiveChunkStatus::Merged);
}
for id in &diff.revert_to_pending {
status.insert(id.clone(), LiveChunkStatus::Pending);
}
let keep: BTreeSet<&str> = diff.kept_done.iter().map(String::as_str).collect();
let mut new_tier = BTreeMap::new();
let mut new_promotions = BTreeMap::new();
for c in &new_plan.chunks {
if keep.contains(c.id.as_str()) {
let tier = run.chunk_tier.get(&c.id).copied().unwrap_or(c.tier);
new_tier.insert(c.id.clone(), tier);
if let Some(&n) = run.chunk_promotions.get(&c.id) {
new_promotions.insert(c.id.clone(), n);
}
} else {
new_tier.insert(c.id.clone(), c.tier);
}
}
let kept_ids: BTreeSet<String> = diff.kept_done.iter().cloned().collect();
if let RebuildOutcome::Conflict { chunk_id } = rebuild_integration(run, &kept_ids)? {
return Ok(ReSpecOutcome::RollbackConflict { chunk_id });
}
run.chunk_status = status;
run.chunk_tier = new_tier;
run.chunk_promotions = new_promotions;
run.chunk_recode_total
.retain(|(rev, _, _), _| *rev >= new_rev);
run.chunk_reports.retain(|r| keep.contains(r.id.as_str()));
run.respec_count = run.respec_count.saturating_add(1);
write_plan(run, &new_plan)?;
run.decisions.push(envelope(
"spec",
DecisionTier::Decider,
format!(
"re-spec plan.v{new_rev}: {} chunk(s) revert to pending, {} kept done",
diff.revert_to_pending.len(),
diff.kept_done.len()
),
vec![format!("plan:{new_rev}"), format!("intent_rev:1")],
spec.model(),
spec.prompt_version(),
));
Ok(ReSpecOutcome::Replanned(Box::new(new_plan)))
}
fn write_plan(run: &Run, plan: &Plan) -> Result<(), PipelineError> {
let plan_json = serde_json::to_string_pretty(plan)
.map_err(|e| PipelineError::Io(format!("could not serialize plan.json: {e}")))?;
std::fs::write(run.cfg.workdir.join("plan.json"), &plan_json)
.map_err(|e| PipelineError::Io(format!("could not write plan.json: {e}")))?;
std::fs::write(
run.cfg
.workdir
.join(format!("plan.v{}.json", plan.plan_rev)),
&plan_json,
)
.map_err(|e| PipelineError::Io(format!("could not write plan revision: {e}")))?;
Ok(())
}
const MAX_PLAN_ATTEMPTS: u32 = 2;
const INVALID_PLAN_FILE: &str = "plan.invalid.json";
fn produce_and_validate_plan(
run: &mut Run,
spec: &dyn SpecProvider,
baseline: &plan::Baseline,
plan_rev: u32,
respec: Option<(&serde_json::Value, &str)>,
) -> Result<Plan, PipelineError> {
let ctx = SpecContext {
intent: &run.cfg.intent,
slug: &run.slug,
source_branch: &run.cfg.source_branch,
integration_branch: &run.integration_branch,
files: &run.cfg.files,
worktree: &run.integration_wt,
baseline,
};
let mut last_raw: Option<serde_json::Value> = None;
let mut last_err: Option<String> = None;
for attempt in 0..MAX_PLAN_ATTEMPTS {
let produced = match (attempt, &last_raw, &last_err) {
(0, _, _) => match respec {
Some((prev, reason)) => spec.respec_plan(&ctx, prev, reason),
None => spec.produce_plan(&ctx),
},
(_, Some(invalid), Some(err)) => spec.repair_plan(&ctx, invalid, err),
_ => spec.produce_plan(&ctx),
};
run.meter.record_agent_run(None);
let raw = match produced {
Ok(raw) => raw,
Err(e) => {
let _ = persist_invalid_plan(run, last_raw.as_ref());
return Err(e);
}
};
let normalized = normalize_plan(raw.clone(), run, baseline, plan_rev);
match plan::parse_and_validate_plan(&normalized) {
Ok(p) => return Ok(p),
Err(e) => {
last_err = Some(e.to_string());
last_raw = Some(raw);
}
}
}
let last_err = last_err.unwrap_or_else(|| "no plan produced".to_string());
let persisted = persist_invalid_plan(run, last_raw.as_ref());
Err(PipelineError::PlanInvalid(format!(
"spec produced an invalid plan after {MAX_PLAN_ATTEMPTS} attempt(s): {last_err}{persisted}"
)))
}
fn gate_plan_baseline(live: &BaselineSnapshot, plan: &Plan) -> Result<(), PipelineError> {
live.verify_plan_baseline(&plan.baseline).map_err(|e| {
PipelineError::PlanInvalid(format!(
"plan baseline does not match the live baseline: {e}"
))
})
}
fn verify_capture_ref(worktree: &Path, expected_oid: &str) -> Result<(), PipelineError> {
let head = git::head(worktree)?;
if head != expected_oid {
return Err(PipelineError::Setup(format!(
"capture worktree HEAD {head} != expected fork OID {expected_oid}; refusing to capture a baseline under an unverified OID"
)));
}
if !git::is_clean(worktree)? {
return Err(PipelineError::Setup(format!(
"capture worktree {} has uncommitted changes; refusing to capture a baseline whose files are not the pinned OID {expected_oid}",
worktree.display()
)));
}
Ok(())
}
fn persist_invalid_plan(run: &Run, raw: Option<&serde_json::Value>) -> String {
let Some(raw) = raw else {
return String::new();
};
let path = run.cfg.workdir.join(INVALID_PLAN_FILE);
let body = serde_json::to_string_pretty(raw).unwrap_or_else(|_| raw.to_string());
match std::fs::write(&path, body) {
Ok(()) => format!(" (raw invalid plan written to {})", path.display()),
Err(e) => format!(" (could not persist invalid plan: {e})"),
}
}
fn normalize_plan(
raw: serde_json::Value,
run: &Run,
baseline: &plan::Baseline,
plan_rev: u32,
) -> serde_json::Value {
use serde_json::json;
let mut obj = match raw {
serde_json::Value::Object(m) => m,
other => return other,
};
obj.insert(
"schema_version".to_string(),
json!(plan::PLAN_SCHEMA_VERSION),
);
obj.insert("plan_rev".to_string(), json!(plan_rev));
obj.insert("intent_rev".to_string(), json!(1));
obj.insert(
"feature".to_string(),
json!({
"slug": run.slug,
"source_branch": run.cfg.source_branch,
"integration_branch": run.integration_branch,
}),
);
obj.insert(
"baseline".to_string(),
json!({
"ref": baseline.r#ref,
"commit_oid": baseline.commit_oid,
"toolchain": baseline.toolchain,
"test_passlist_hash": baseline.test_passlist_hash,
"clippy_warnings_hash": baseline.clippy_warnings_hash,
"enumerated_targets_hash": baseline.enumerated_targets_hash,
}),
);
serde_json::Value::Object(obj)
}
fn union_declared_files(plan: &Plan) -> Vec<PathBuf> {
let mut seen = std::collections::BTreeSet::new();
for chunk in &plan.chunks {
for f in &chunk.files_touched {
seen.insert(PathBuf::from(f));
}
}
seen.into_iter().collect()
}
fn resource_breach(run: &Run) -> Option<String> {
run.meter.breach(&run.cfg.budget, run.started.elapsed())
}
fn refresh_storage(run: &mut Run) {
if run.cfg.budget.max_storage_bytes.is_some() {
let bytes = breakers::dir_size_bytes(&run.cfg.workdir);
run.meter.observe_storage_bytes(bytes);
}
}
type ReplayResult = (BTreeMap<String, ChunkProvenance>, Vec<(String, String)>);
enum RebuildOutcome {
Rebuilt,
Conflict { chunk_id: String },
}
enum Replayed {
Done(ReplayResult),
Conflict(String),
}
fn provenance_ref(slug: &str, chunk_id: &str) -> String {
format!("refs/pipeline/prov/{slug}/{chunk_id}")
}
fn provenance_ref_prefix(slug: &str) -> String {
format!("refs/pipeline/prov/{slug}/")
}
fn authored_commit_for<'b>(run: &'b Run, id: &str, prov: &'b ChunkProvenance) -> &'b str {
run.chunk_reports
.iter()
.find(|r| r.id == id)
.and_then(|r| r.commit.as_deref())
.unwrap_or(&prov.commit)
}
fn pin_provenance_refs(run: &Run, kept: &[(String, ChunkProvenance)]) {
for (id, prov) in kept {
let oid = authored_commit_for(run, id, prov);
let _ = git::update_ref(&run.repo, &provenance_ref(&run.slug, id), oid);
}
}
fn rebuild_integration(
run: &mut Run,
keep: &BTreeSet<String>,
) -> Result<RebuildOutcome, PipelineError> {
let missing: Vec<&String> = keep
.iter()
.filter(|id| !run.chunk_provenance.contains_key(id.as_str()))
.collect();
if !missing.is_empty() {
return Err(PipelineError::Git(format!(
"provenance rollback refused: kept chunk(s) lack merge provenance: {missing:?}"
)));
}
let mut kept: Vec<(String, ChunkProvenance)> = run
.chunk_provenance
.iter()
.filter(|(id, _)| keep.contains(id.as_str()))
.map(|(id, p)| (id.clone(), p.clone()))
.collect();
kept.sort_by_key(|(_, p)| p.order);
pin_provenance_refs(run, &kept);
let original_tip = git::head(&run.integration_wt)?;
let replay = || -> Result<Replayed, PipelineError> {
git::restore_to(&run.integration_wt, &run.fork_commit)?;
let mut new_prov: BTreeMap<String, ChunkProvenance> = BTreeMap::new();
let mut report_updates: Vec<(String, String)> = Vec::new();
for (id, prov) in &kept {
let base = git::head(&run.integration_wt)?;
match git::cherry_pick(&run.integration_wt, &prov.base, &prov.commit)? {
MergeOutcome::Merged { commit } => {
report_updates.push((id.clone(), commit.clone()));
new_prov.insert(
id.clone(),
ChunkProvenance {
base,
commit,
order: prov.order,
},
);
}
MergeOutcome::Conflict { .. } => return Ok(Replayed::Conflict(id.clone())),
}
}
Ok(Replayed::Done((new_prov, report_updates)))
};
match replay() {
Ok(Replayed::Done((new_prov, report_updates))) => {
for (id, commit) in report_updates {
for r in &mut run.chunk_reports {
if r.id == id {
r.merge_commit = Some(commit.clone());
r.replayed = true;
}
}
}
run.chunk_provenance = new_prov;
Ok(RebuildOutcome::Rebuilt)
}
Ok(Replayed::Conflict(chunk_id)) => {
restore_intact(&run.integration_wt, &original_tip)?;
Ok(RebuildOutcome::Conflict { chunk_id })
}
Err(e) => {
if let Err(restore_err) = restore_intact(&run.integration_wt, &original_tip) {
return Err(PipelineError::Git(format!(
"provenance rollback failed ({e}) AND restoring the integration branch to its intact tip {original_tip} also failed ({restore_err}); the branch may be in an unknown state"
)));
}
Err(e)
}
}
}
fn restore_intact(worktree: &Path, original_tip: &str) -> Result<(), PipelineError> {
git::restore_to(worktree, original_tip)?;
let head = git::head(worktree)?;
if head != original_tip {
return Err(PipelineError::Git(format!(
"rollback restore did not land: HEAD is {head}, expected {original_tip}"
)));
}
if !git::is_clean(worktree)? {
return Err(PipelineError::Git(
"rollback restore left the integration worktree dirty".to_string(),
));
}
Ok(())
}
fn run_code_stage(
run: &mut Run,
plan: &Plan,
harnesses: &dyn TierHarness,
baseline: &BaselineSnapshot,
pending_findings: &BTreeMap<String, Vec<String>>,
pending_prior_diff: &BTreeMap<String, String>,
) -> Result<(), PipelineError> {
let waves = ready_waves(plan, &run.chunk_status);
for wave in &waves {
let pending: Vec<usize> = wave
.iter()
.copied()
.filter(|&i| run.chunk_status.get(&plan.chunks[i].id) != Some(&LiveChunkStatus::Merged))
.collect();
if pending.is_empty() {
continue;
}
let k = effective_build_concurrency(run, pending.len());
if k <= 1 {
for &idx in &pending {
process_chunk_sequential(
run,
plan,
idx,
harnesses,
baseline,
pending_findings,
pending_prior_diff,
)?;
if run.code_block_status.is_some() {
return Ok(());
}
}
} else {
run_wave_concurrent(
run,
plan,
&pending,
harnesses,
baseline,
k,
pending_findings,
pending_prior_diff,
)?;
if run.code_block_status.is_some() {
return Ok(());
}
}
}
Ok(())
}
fn effective_build_concurrency(run: &Run, wave_len: usize) -> usize {
let mut k = run.cfg.max_build_concurrency.min(wave_len).max(1);
if let Some(cap) = run.cfg.budget.max_processes {
let used = run.meter.processes;
let remaining = (cap as usize).saturating_sub(used as usize).max(1);
k = k.min(remaining);
}
k
}
#[allow(clippy::too_many_arguments)]
fn process_chunk_sequential(
run: &mut Run,
plan: &Plan,
idx: usize,
harnesses: &dyn TierHarness,
baseline: &BaselineSnapshot,
pending_findings: &BTreeMap<String, Vec<String>>,
pending_prior_diff: &BTreeMap<String, String>,
) -> Result<(), PipelineError> {
let chunk = &plan.chunks[idx];
let verify_seed: Vec<String> = pending_findings.get(&chunk.id).cloned().unwrap_or_default();
let mut findings: Vec<String> = verify_seed.clone();
let mut prior_diff: Option<String> = pending_prior_diff.get(&chunk.id).cloned();
let mut recode = 1u32;
let mut seq = 1u32;
loop {
let current_tier = run.chunk_tier.get(&chunk.id).copied().unwrap_or(chunk.tier);
match attempt_chunk(
run,
plan,
chunk,
harnesses,
current_tier,
baseline,
seq,
&findings,
prior_diff.as_deref(),
)? {
ChunkAttempt::Merged {
verdict,
base,
commit,
merge_commit,
} => {
run.decisions.push(envelope(
"supervisor",
DecisionTier::Coordinator,
format!("chunk {} floor green — merged", chunk.id),
vec![format!("chunk:{}", chunk.id), format!("commit:{commit}")],
"supervisor",
"v1",
));
run.merge_seq += 1;
run.chunk_provenance.insert(
chunk.id.clone(),
ChunkProvenance {
base,
commit: commit.clone(),
order: run.merge_seq,
},
);
upsert_chunk_report(
run,
ChunkReport {
id: chunk.id.clone(),
title: chunk.title.clone(),
tier: current_tier.wire_name().to_string(),
outcome: "committed".to_string(),
floor_passed: Some(true),
floor: Some(verdict),
merged: true,
commit: Some(commit),
merge_commit: Some(merge_commit),
replayed: false,
reason: None,
branch_preserved: None,
},
);
run.chunk_status
.insert(chunk.id.clone(), LiveChunkStatus::Merged);
refresh_storage(run);
if let Some(msg) = resource_breach(run) {
run.circuit_breaker = Some(msg);
run.code_block_status = Some("circuit_breaker");
return Ok(());
}
break;
}
ChunkAttempt::Blocked {
outcome,
diff: attempt_diff,
status,
reason,
findings: attempt_findings,
floor,
floor_passed,
recodable,
wt,
branch,
} => {
let fp = failure_fingerprint(
&chunk.id,
current_tier.wire_name(),
status,
&attempt_findings,
);
let recurrence = run.meter.record_failure(&fp);
refresh_storage(run);
if let Some(msg) = run
.cfg
.budget
.identical_failure_breach(recurrence)
.or_else(|| resource_breach(run))
{
run.circuit_breaker = Some(msg.clone());
run.code_block_status = Some("circuit_breaker");
run.decisions.push(envelope(
"supervisor",
DecisionTier::Coordinator,
format!(
"chunk {} stopped by circuit-breaker — preserved, not merged ({msg})",
chunk.id
),
vec![format!("chunk:{}", chunk.id)],
"supervisor",
"v1",
));
push_blocked_chunk(
run,
chunk,
outcome,
floor,
floor_passed,
reason,
&wt,
&branch,
);
return Ok(());
}
let recode_key = (plan.plan_rev, chunk.id.clone(), current_tier.wire_name());
let recode_total = run
.chunk_recode_total
.get(&recode_key)
.copied()
.unwrap_or(0);
if recodable
&& recode <= run.cfg.fix_loop.max_recode_per_chunk
&& recode_total < run.cfg.fix_loop.max_recode_per_chunk
{
record_recode_decision(
run,
plan,
&chunk.id,
&attempt_findings,
"floor re-code",
);
*run.chunk_recode_total.entry(recode_key).or_insert(0) += 1;
let _ = git::worktree_remove(&run.repo, &wt);
let _ = git::delete_branch(&run.repo, &branch, true);
if attempt_diff.is_some() {
prior_diff = attempt_diff;
}
findings = verify_seed
.iter()
.cloned()
.chain(attempt_findings)
.collect();
recode += 1;
seq += 1;
continue;
}
if let Some(promoted) = recodable
.then(|| promotion_target(run, harnesses, &chunk.id, current_tier))
.flatten()
{
promote_chunk(run, plan, &chunk.id, current_tier, promoted);
let _ = git::worktree_remove(&run.repo, &wt);
let _ = git::delete_branch(&run.repo, &branch, true);
if attempt_diff.is_some() {
prior_diff = attempt_diff;
}
findings = verify_seed
.iter()
.cloned()
.chain(attempt_findings)
.collect();
recode = 1;
seq += 1;
continue;
}
let promotions = run.chunk_promotions.get(&chunk.id).copied().unwrap_or(0);
if recodable && (run.cfg.fix_loop.max_recode_per_chunk > 0 || promotions > 0) {
let cumulative_recodes: u32 = run
.chunk_recode_total
.iter()
.filter(|((rev, id, _tier), _)| *rev == plan.plan_rev && id == &chunk.id)
.map(|(_, n)| *n)
.sum();
run.circuit_breaker = Some(format!(
"chunk {} still blocked after {seq} attempt(s) this visit ({cumulative_recodes} cumulative floor re-code(s) across visits, {promotions} promotion(s)): {reason}",
chunk.id,
));
run.code_block_status = Some("circuit_breaker");
} else {
run.code_block_status = Some(status);
}
run.decisions.push(envelope(
"supervisor",
DecisionTier::Coordinator,
format!(
"chunk {} blocked — preserved, not merged ({reason})",
chunk.id
),
vec![format!("chunk:{}", chunk.id)],
"supervisor",
"v1",
));
push_blocked_chunk(
run,
chunk,
outcome,
floor,
floor_passed,
reason,
&wt,
&branch,
);
return Ok(());
}
}
}
Ok(())
}
fn ready_waves(plan: &Plan, status: &BTreeMap<String, LiveChunkStatus>) -> Vec<Vec<usize>> {
let is_merged = |id: &str| status.get(id) == Some(&LiveChunkStatus::Merged);
let mut remaining: Vec<usize> = plan
.chunks
.iter()
.enumerate()
.filter(|(_, c)| !is_merged(&c.id))
.map(|(i, _)| i)
.collect();
let mut done: BTreeSet<String> = plan
.chunks
.iter()
.filter(|c| is_merged(&c.id))
.map(|c| c.id.clone())
.collect();
let mut waves: Vec<Vec<usize>> = Vec::new();
while !remaining.is_empty() {
let wave: Vec<usize> = remaining
.iter()
.copied()
.filter(|&i| plan.chunks[i].deps.iter().all(|d| done.contains(d)))
.collect();
if wave.is_empty() {
waves.push(remaining.clone());
break;
}
for &i in &wave {
done.insert(plan.chunks[i].id.clone());
}
remaining.retain(|i| !wave.contains(i));
waves.push(wave);
}
waves
}
struct WaveBuildResult {
idx: usize,
tier: Tier,
usages: Vec<Option<Usage>>,
recode_findings: Vec<Vec<String>>,
outcome: WaveBuildOutcome,
}
struct WaveArtifact {
wt: PathBuf,
branch: String,
initial_tip: Option<String>,
}
enum WaveBuildOutcome {
Built {
verdict: FloorVerdict,
base: String,
commit: String,
wt: PathBuf,
branch: String,
},
Blocked {
outcome: &'static str,
status: &'static str,
reason: String,
floor: Option<FloorVerdict>,
floor_passed: Option<bool>,
recodable: bool,
wt: PathBuf,
branch: String,
},
}
enum WaveJob {
Done(WaveBuildResult),
Error(PipelineError),
Panicked(String),
}
fn panic_message(payload: &(dyn std::any::Any + Send)) -> String {
payload
.downcast_ref::<&str>()
.map(|s| (*s).to_string())
.or_else(|| payload.downcast_ref::<String>().cloned())
.unwrap_or_else(|| "unknown panic payload".to_string())
}
#[allow(clippy::too_many_arguments)]
fn build_chunk_in_wave(
cfg: &PipelineConfig,
repo: &Path,
slug: &str,
wave_base: &str,
plan_rev: u32,
idx: usize,
chunk: &Chunk,
tier: Tier,
harness: &dyn CodeHarness,
baseline: &BaselineSnapshot,
verify_seed: &[String],
seed_prior_diff: Option<&str>,
recode_budget: u32,
git_lock: &Mutex<()>,
artifacts: &Mutex<BTreeMap<usize, WaveArtifact>>,
) -> Result<WaveBuildResult, PipelineError> {
let mut findings: Vec<String> = verify_seed.to_vec();
let mut prior_diff: Option<String> = seed_prior_diff.map(str::to_string);
let mut seq = 1u32;
let mut recode = 1u32;
let mut usages: Vec<Option<Usage>> = Vec::new();
let mut recode_findings: Vec<Vec<String>> = Vec::new();
loop {
{
let (wt, branch) = chunk_attempt_names(cfg, slug, &chunk.id, seq);
let initial_tip = if git::branch_exists(repo, &branch) {
git::resolve_commit(repo, &branch).ok()
} else {
None
};
artifacts
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.insert(
idx,
WaveArtifact {
wt,
branch,
initial_tip,
},
);
}
let (built, usage) = build_and_gate(
cfg,
repo,
slug,
wave_base,
plan_rev,
chunk,
harness,
baseline,
seq,
&findings,
prior_diff.as_deref(),
Some(git_lock),
)?;
usages.push(usage);
match built {
BuildAttempt::Built {
verdict,
base,
commit,
wt,
branch,
} => {
artifacts
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.remove(&idx);
return Ok(WaveBuildResult {
idx,
tier,
usages,
recode_findings,
outcome: WaveBuildOutcome::Built {
verdict,
base,
commit,
wt,
branch,
},
});
}
BuildAttempt::Blocked {
outcome,
diff,
status,
reason,
findings: attempt_findings,
floor,
floor_passed,
recodable,
wt,
branch,
} => {
if recodable && recode <= recode_budget {
{
let _g = git_lock
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let _ = git::worktree_remove(repo, &wt);
let _ = git::delete_branch(repo, &branch, true);
}
if diff.is_some() {
prior_diff = diff;
}
recode_findings.push(attempt_findings.clone());
findings = verify_seed
.iter()
.cloned()
.chain(attempt_findings)
.collect();
recode += 1;
seq += 1;
continue;
}
drop(attempt_findings);
artifacts
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.remove(&idx);
return Ok(WaveBuildResult {
idx,
tier,
usages,
recode_findings,
outcome: WaveBuildOutcome::Blocked {
outcome,
status,
reason,
floor,
floor_passed,
recodable,
wt,
branch,
},
});
}
}
}
}
#[allow(clippy::too_many_arguments)]
fn run_wave_concurrent(
run: &mut Run,
plan: &Plan,
pending: &[usize],
harnesses: &dyn TierHarness,
baseline: &BaselineSnapshot,
k: usize,
pending_findings: &BTreeMap<String, Vec<String>>,
pending_prior_diff: &BTreeMap<String, String>,
) -> Result<(), PipelineError> {
let wave_base = git::head(&run.integration_wt)?;
let plan_rev = plan.plan_rev;
let max_recode = run.cfg.fix_loop.max_recode_per_chunk;
struct Job<'a> {
idx: usize,
chunk: &'a Chunk,
tier: Tier,
harness: &'a dyn CodeHarness,
verify_seed: Vec<String>,
prior_diff: Option<String>,
recode_budget: u32,
}
let jobs: Vec<Job> = pending
.iter()
.map(|&idx| {
let chunk = &plan.chunks[idx];
let tier = run.chunk_tier.get(&chunk.id).copied().unwrap_or(chunk.tier);
let spent = run
.chunk_recode_total
.get(&(plan_rev, chunk.id.clone(), tier.wire_name()))
.copied()
.unwrap_or(0);
Job {
idx,
chunk,
tier,
harness: harnesses.harness(tier),
verify_seed: pending_findings.get(&chunk.id).cloned().unwrap_or_default(),
prior_diff: pending_prior_diff.get(&chunk.id).cloned(),
recode_budget: max_recode.saturating_sub(spent),
}
})
.collect();
let cfg = run.cfg;
let repo = run.repo.clone();
let slug = run.slug.clone();
let git_lock: Mutex<()> = Mutex::new(());
let queue: Mutex<VecDeque<usize>> = Mutex::new((0..jobs.len()).collect());
let results: Mutex<BTreeMap<usize, WaveJob>> = Mutex::new(BTreeMap::new());
let artifacts: Mutex<BTreeMap<usize, WaveArtifact>> = Mutex::new(BTreeMap::new());
std::thread::scope(|scope| {
for _ in 0..k {
scope.spawn(|| loop {
let next = {
let mut q = queue
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
q.pop_front()
};
let Some(job_i) = next else { break };
let job = &jobs[job_i];
let res = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
build_chunk_in_wave(
cfg,
&repo,
&slug,
&wave_base,
plan_rev,
job.idx,
job.chunk,
job.tier,
job.harness,
baseline,
&job.verify_seed,
job.prior_diff.as_deref(),
job.recode_budget,
&git_lock,
&artifacts,
)
}));
let outcome = match res {
Ok(Ok(r)) => WaveJob::Done(r),
Ok(Err(e)) => WaveJob::Error(e),
Err(panic) => WaveJob::Panicked(panic_message(panic.as_ref())),
};
results
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.insert(job.idx, outcome);
});
}
});
let mut results = results
.into_inner()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let artifacts = artifacts
.into_inner()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let mut built: Vec<WaveBuildResult> = Vec::new();
let mut blocked: Vec<WaveBuildResult> = Vec::new();
let mut hard_error: Option<PipelineError> = None;
let mut panicked: Option<(String, String)> = None;
for &idx in pending {
let r = match results
.remove(&idx)
.expect("every wave job records a result")
{
WaveJob::Done(r) => r,
WaveJob::Error(e) => {
if let Some(artifact) = artifacts.get(&idx) {
audit_terminal_worker_artifact(
run,
&plan.chunks[idx],
&wave_base,
artifact,
"terminal wave worker hard-errored after committing — branch preserved, contents unaudited (floor never gated this attempt)",
);
}
if hard_error.is_none() {
hard_error = Some(e);
}
continue;
}
WaveJob::Panicked(msg) => {
if let Some(artifact) = artifacts.get(&idx) {
audit_terminal_worker_artifact(
run,
&plan.chunks[idx],
&wave_base,
artifact,
"terminal wave worker panicked after committing — branch preserved, contents unaudited (floor never gated this attempt)",
);
}
if panicked.is_none() {
panicked = Some((plan.chunks[idx].id.clone(), msg));
}
continue;
}
};
for u in &r.usages {
run.meter.record_agent_run(u.as_ref());
}
let chunk = &plan.chunks[idx];
let recode_key = (plan_rev, chunk.id.clone(), r.tier.wire_name());
for f in &r.recode_findings {
record_recode_decision(run, plan, &chunk.id, f, "floor re-code");
*run.chunk_recode_total
.entry(recode_key.clone())
.or_insert(0) += 1;
}
match r.outcome {
WaveBuildOutcome::Built { .. } => built.push(r),
WaveBuildOutcome::Blocked { .. } => blocked.push(r),
}
}
if let Some(e) = hard_error {
for r in blocked.iter().chain(built.iter()) {
preserve_wave_build(run, &plan.chunks[r.idx], r);
}
return match panicked {
Some((chunk_id, msg)) => Err(e.with_note(format!(
"a sibling build thread also panicked on chunk {chunk_id}: {msg}"
))),
None => Err(e),
};
}
if let Some((chunk_id, msg)) = panicked {
for r in blocked.iter().chain(built.iter()) {
preserve_wave_build(run, &plan.chunks[r.idx], r);
}
run.circuit_breaker = Some(format!(
"wave build thread panicked on chunk {chunk_id} — sibling builds preserved, stage stopped: {msg}"
));
run.code_block_status = Some("circuit_breaker");
run.decisions.push(envelope(
"supervisor",
DecisionTier::Coordinator,
format!(
"chunk {chunk_id} build thread panicked — sibling builds preserved, stage stopped: {msg}"
),
vec![format!("chunk:{chunk_id}")],
"supervisor",
"v1",
));
return Ok(());
}
for r in &blocked {
preserve_wave_build(run, &plan.chunks[r.idx], r);
}
refresh_storage(run);
if let Some(msg) = resource_breach(run) {
run.circuit_breaker = Some(msg.clone());
run.code_block_status = Some("circuit_breaker");
run.decisions.push(envelope(
"supervisor",
DecisionTier::Coordinator,
format!("wave stopped by circuit-breaker before merge — builds preserved ({msg})"),
vec![format!("plan_rev:{plan_rev}")],
"supervisor",
"v1",
));
for r in &built {
preserve_wave_build(run, &plan.chunks[r.idx], r);
}
return Ok(());
}
let mut pending_merges: VecDeque<WaveBuildResult> = built.into();
while let Some(r) = pending_merges.pop_front() {
let idx = r.idx;
let tier = r.tier;
let (base, commit, wt, branch) = match r.outcome {
WaveBuildOutcome::Built {
verdict: _,
base,
commit,
wt,
branch,
} => (base, commit, wt, branch),
WaveBuildOutcome::Blocked { .. } => unreachable!("split above keeps only Built here"),
};
let chunk = &plan.chunks[idx];
let pre_tip = git::head(&run.integration_wt)?;
match git::merge_no_ff(
&run.integration_wt,
&commit,
&format!("pipeline: merge chunk {}", chunk.id),
)? {
MergeOutcome::Merged {
commit: merge_commit,
} => {
let post_tip = git::head(&run.integration_wt)?;
let changed = floor::git::changed_files(&run.integration_wt, &pre_tip, &post_tip)?;
let verdict2 = gate_chunk(
run.cfg,
&run.repo,
chunk,
&run.integration_wt,
&pre_tip,
&changed,
baseline,
)?;
if verdict2.passed() {
finalize_wave_merge(
run,
chunk,
tier,
verdict2,
base,
commit,
merge_commit,
&wt,
&branch,
);
refresh_storage(run);
if let Some(msg) = resource_breach(run) {
run.circuit_breaker = Some(msg);
run.code_block_status = Some("circuit_breaker");
preserve_pending_merges(run, plan, &pending_merges);
return Ok(());
}
} else {
git::restore_to(&run.integration_wt, &pre_tip)?;
let _ = git::worktree_remove(&run.repo, &wt);
let _ = git::delete_branch(&run.repo, &branch, true);
run.decisions.push(envelope(
"supervisor",
DecisionTier::Coordinator,
format!(
"chunk {} floor regressed at merge — rebase&fix off moved tip ({})",
chunk.id,
fixloop::floor_findings(&verdict2).join("; ")
),
vec![format!("chunk:{}", chunk.id)],
"supervisor",
"v1",
));
process_chunk_sequential(
run,
plan,
idx,
harnesses,
baseline,
pending_findings,
pending_prior_diff,
)?;
if run.code_block_status.is_some() {
preserve_pending_merges(run, plan, &pending_merges);
return Ok(());
}
}
}
MergeOutcome::Conflict { details } => {
let _ = git::worktree_remove(&run.repo, &wt);
let _ = git::delete_branch(&run.repo, &branch, true);
run.decisions.push(envelope(
"supervisor",
DecisionTier::Coordinator,
format!(
"chunk {} merge conflict — rebase&fix off moved tip: {details}",
chunk.id
),
vec![format!("chunk:{}", chunk.id)],
"supervisor",
"v1",
));
process_chunk_sequential(
run,
plan,
idx,
harnesses,
baseline,
pending_findings,
pending_prior_diff,
)?;
if run.code_block_status.is_some() {
preserve_pending_merges(run, plan, &pending_merges);
return Ok(());
}
}
}
}
for r in blocked {
let idx = r.idx;
let chunk_id = plan.chunks[idx].id.clone();
if let WaveBuildOutcome::Blocked {
recodable: false,
status,
..
} = &r.outcome
{
run.code_block_status = Some(status);
run.decisions.push(envelope(
"supervisor",
DecisionTier::Coordinator,
format!(
"chunk {chunk_id} blocked in wave (not re-codable) — preserved, not merged"
),
vec![format!("chunk:{chunk_id}")],
"supervisor",
"v1",
));
return Ok(());
}
reconcile_preserved_wave_build(run, &plan.chunks[idx], &r);
run.decisions.push(envelope(
"supervisor",
DecisionTier::Coordinator,
format!(
"chunk {chunk_id} exhausted the wave re-code budget — re-queued to the sequential drain for tier promotion"
),
vec![format!("chunk:{chunk_id}")],
"supervisor",
"v1",
));
process_chunk_sequential(
run,
plan,
idx,
harnesses,
baseline,
pending_findings,
pending_prior_diff,
)?;
if run.code_block_status.is_some() {
return Ok(());
}
}
Ok(())
}
#[allow(clippy::too_many_arguments)]
fn finalize_wave_merge(
run: &mut Run,
chunk: &Chunk,
tier: Tier,
verdict: FloorVerdict,
base: String,
commit: String,
merge_commit: String,
wt: &Path,
branch: &str,
) {
run.decisions.push(envelope(
"supervisor",
DecisionTier::Coordinator,
format!("chunk {} floor green — merged (wave)", chunk.id),
vec![format!("chunk:{}", chunk.id), format!("commit:{commit}")],
"supervisor",
"v1",
));
run.merge_seq += 1;
run.chunk_provenance.insert(
chunk.id.clone(),
ChunkProvenance {
base,
commit: commit.clone(),
order: run.merge_seq,
},
);
upsert_chunk_report(
run,
ChunkReport {
id: chunk.id.clone(),
title: chunk.title.clone(),
tier: tier.wire_name().to_string(),
outcome: "committed".to_string(),
floor_passed: Some(true),
floor: Some(verdict),
merged: true,
commit: Some(commit),
merge_commit: Some(merge_commit),
replayed: false,
reason: None,
branch_preserved: None,
},
);
run.chunk_status
.insert(chunk.id.clone(), LiveChunkStatus::Merged);
let _ = git::worktree_remove(&run.repo, wt);
let _ = git::delete_branch(&run.repo, branch, true);
}
fn preserve_pending_merges(run: &mut Run, plan: &Plan, pending: &VecDeque<WaveBuildResult>) {
for r in pending {
preserve_wave_build(run, &plan.chunks[r.idx], r);
}
}
fn preserve_wave_build(run: &mut Run, chunk: &Chunk, r: &WaveBuildResult) {
let (outcome, floor, floor_passed, reason, wt, branch) = match &r.outcome {
WaveBuildOutcome::Blocked {
outcome,
floor,
floor_passed,
reason,
wt,
branch,
..
} => (
*outcome,
floor.clone(),
*floor_passed,
reason.clone(),
wt.as_path(),
branch.as_str(),
),
WaveBuildOutcome::Built {
verdict,
wt,
branch,
..
} => (
"committed",
Some(verdict.clone()),
Some(true),
"wave build preserved before merge".to_string(),
wt.as_path(),
branch.as_str(),
),
};
push_blocked_chunk(run, chunk, outcome, floor, floor_passed, reason, wt, branch);
}
fn audit_terminal_worker_artifact(
run: &mut Run,
chunk: &Chunk,
base: &str,
artifact: &WaveArtifact,
reason: &str,
) {
if !git::branch_exists(&run.repo, &artifact.branch) {
return;
}
let commit = git::resolve_commit(&run.repo, &artifact.branch).ok();
if commit.is_some() && commit == artifact.initial_tip {
return;
}
let ahead = git::commits_ahead_of(&run.repo, base, &artifact.branch).unwrap_or(1);
if ahead == 0 {
return;
}
run.preserved
.push((artifact.wt.clone(), artifact.branch.clone()));
upsert_chunk_report(
run,
ChunkReport {
id: chunk.id.clone(),
title: chunk.title.clone(),
tier: chunk.tier.wire_name().to_string(),
outcome: "committed".to_string(),
floor_passed: None,
floor: None,
merged: false,
commit,
merge_commit: None,
replayed: false,
reason: Some(reason.to_string()),
branch_preserved: Some(artifact.branch.clone()),
},
);
}
fn reconcile_preserved_wave_build(run: &mut Run, chunk: &Chunk, r: &WaveBuildResult) {
let (wt, branch) = match &r.outcome {
WaveBuildOutcome::Blocked { wt, branch, .. }
| WaveBuildOutcome::Built { wt, branch, .. } => (wt.clone(), branch.clone()),
};
let _ = git::worktree_remove(&run.repo, &wt);
let _ = git::delete_branch(&run.repo, &branch, true);
run.preserved
.retain(|(w, b)| w.as_path() != wt.as_path() || b.as_str() != branch.as_str());
run.chunk_reports.retain(|rep| rep.id != chunk.id);
}
enum ChunkAttempt {
Merged {
verdict: FloorVerdict,
base: String,
commit: String,
merge_commit: String,
},
Blocked {
outcome: &'static str,
diff: Option<String>,
status: &'static str,
reason: String,
findings: Vec<String>,
floor: Option<FloorVerdict>,
floor_passed: Option<bool>,
recodable: bool,
wt: PathBuf,
branch: String,
},
}
#[allow(clippy::too_many_arguments)]
fn attempt_chunk(
run: &mut Run,
plan: &Plan,
chunk: &Chunk,
harnesses: &dyn TierHarness,
current_tier: Tier,
baseline: &BaselineSnapshot,
seq: u32,
findings: &[String],
prior_diff: Option<&str>,
) -> Result<ChunkAttempt, PipelineError> {
let base_commit = git::head(&run.integration_wt)?;
let (built, usage) = build_and_gate(
run.cfg,
&run.repo,
&run.slug,
&base_commit,
plan.plan_rev,
chunk,
harnesses.harness(current_tier),
baseline,
seq,
findings,
prior_diff,
None,
)?;
run.meter.record_agent_run(usage.as_ref());
let (verdict, base, commit, chunk_wt, chunk_branch) = match built {
BuildAttempt::Blocked {
outcome,
diff,
status,
reason,
findings,
floor,
floor_passed,
recodable,
wt,
branch,
} => {
return Ok(ChunkAttempt::Blocked {
outcome,
diff,
status,
reason,
findings,
floor,
floor_passed,
recodable,
wt,
branch,
})
}
BuildAttempt::Built {
verdict,
base,
commit,
wt,
branch,
} => (verdict, base, commit, wt, branch),
};
match git::merge_no_ff(
&run.integration_wt,
&commit,
&format!("pipeline: merge chunk {}", chunk.id),
)? {
MergeOutcome::Merged {
commit: merge_commit,
} => {
let _ = git::worktree_remove(&run.repo, &chunk_wt);
let _ = git::delete_branch(&run.repo, &chunk_branch, true);
Ok(ChunkAttempt::Merged {
verdict,
base,
commit,
merge_commit,
})
}
MergeOutcome::Conflict { details } => Ok(ChunkAttempt::Blocked {
outcome: "committed",
diff: None,
status: "chunk_merge_conflict",
reason: format!("chunk merge conflict: {details}"),
findings: vec![format!("chunk merge conflict: {details}")],
floor: Some(verdict),
floor_passed: Some(true),
recodable: false,
wt: chunk_wt,
branch: chunk_branch,
}),
}
}
fn chunk_attempt_names(
cfg: &PipelineConfig,
slug: &str,
chunk_id: &str,
seq: u32,
) -> (PathBuf, String) {
let suffix = if seq == 1 {
String::new()
} else {
format!("-a{seq}")
};
let branch = format!("{slug}/chunk-{chunk_id}{suffix}");
let wt = cfg.workdir.join(format!("chunk-{chunk_id}{suffix}"));
(wt, branch)
}
#[allow(clippy::too_many_arguments)]
fn build_and_gate(
cfg: &PipelineConfig,
repo: &Path,
slug: &str,
base_commit: &str,
plan_rev: u32,
chunk: &Chunk,
harness: &dyn CodeHarness,
baseline: &BaselineSnapshot,
seq: u32,
findings: &[String],
prior_diff: Option<&str>,
git_lock: Option<&Mutex<()>>,
) -> Result<(BuildAttempt, Option<Usage>), PipelineError> {
let (chunk_wt, chunk_branch) = chunk_attempt_names(cfg, slug, &chunk.id, seq);
{
let _guard = git_lock.map(|m| m.lock().unwrap_or_else(std::sync::PoisonError::into_inner));
git::worktree_add_new_branch(repo, &chunk_wt, &chunk_branch, base_commit)?;
}
let checks: Vec<HarnessCheck> = chunk
.checks
.iter()
.enumerate()
.map(|(i, c)| to_harness_check(i, c))
.collect();
let req = ChunkRequest {
run_id: format!("pipeline-{slug}"),
chunk_id: chunk.id.clone(),
attempt_id: format!("a{seq}"),
worktree_path: chunk_wt.clone(),
base_commit: base_commit.to_string(),
plan_rev: plan_rev.to_string(),
brief: fixloop::rebrief(&chunk.brief, findings, prior_diff),
checks,
files: chunk.files_touched.iter().map(PathBuf::from).collect(),
timeout: cfg.chunk_timeout,
};
let cancel = CancelToken::new();
let result = harness
.run_chunk(&req, &cancel)
.map_err(|e| PipelineError::Harness(e.to_string()))?;
let usage = result.usage.clone();
let harness_block = |outcome: &'static str, reason: String| BuildAttempt::Blocked {
outcome,
diff: None,
status: "chunk_failed",
findings: vec![reason.clone()],
reason,
floor: None,
floor_passed: None,
recodable: true,
wt: chunk_wt.clone(),
branch: chunk_branch.clone(),
};
let commit = match &result.outcome {
ChunkOutcome::Committed { commit } => commit.clone(),
ChunkOutcome::NoChange => {
return Ok((
harness_block("no_change", "chunk produced no commit".to_string()),
usage,
))
}
ChunkOutcome::Failed { reason } => {
return Ok((harness_block("failed", reason.clone()), usage))
}
ChunkOutcome::Timeout => {
return Ok((
harness_block("timeout", "chunk timed out".to_string()),
usage,
))
}
ChunkOutcome::Cancelled => {
return Ok((
harness_block("cancelled", "chunk cancelled".to_string()),
usage,
))
}
};
let head = git::head(&chunk_wt)?;
if head != commit {
return Ok((
harness_block(
"failed",
format!("harness reported commit {commit} but worktree HEAD is {head}"),
),
usage,
));
}
if head == base_commit {
return Ok((
harness_block(
"no_change",
"harness reported a commit but HEAD did not advance".to_string(),
),
usage,
));
}
if !git::is_ancestor(&chunk_wt, base_commit, &head)? {
return Ok((
harness_block(
"failed",
format!(
"chunk commit {head} is not a descendant of its base {base_commit} (history rewritten)"
),
),
usage,
));
}
if git::range_has_merge(&chunk_wt, base_commit, &head)? {
return Ok((
harness_block(
"failed",
format!(
"chunk history {base_commit}..{head} contains a merge commit; the provenance rollback replays a linear range and cannot cherry-pick a merge — the chunk must be a linear sequence of commits"
),
),
usage,
));
}
if !git::is_clean(&chunk_wt)? {
return Ok((
harness_block(
"failed",
"chunk worktree has uncommitted changes after the commit".to_string(),
),
usage,
));
}
let changed = floor::git::changed_files(&chunk_wt, base_commit, &head)?;
if changed.is_empty() {
return Ok((
harness_block("no_change", "committed chunk has an empty diff".to_string()),
usage,
));
}
let verdict = gate_chunk(cfg, repo, chunk, &chunk_wt, base_commit, &changed, baseline)?;
if !verdict.passed() {
let diff = git::diff(&chunk_wt, base_commit, &head).ok();
return Ok((
BuildAttempt::Blocked {
outcome: "committed",
diff,
status: "chunk_floor_blocked",
reason: "floor gate failed".to_string(),
findings: fixloop::floor_findings(&verdict),
floor: Some(verdict),
floor_passed: Some(false),
recodable: true,
wt: chunk_wt,
branch: chunk_branch,
},
usage,
));
}
Ok((
BuildAttempt::Built {
verdict,
base: base_commit.to_string(),
commit: head,
wt: chunk_wt,
branch: chunk_branch,
},
usage,
))
}
enum BuildAttempt {
Built {
verdict: FloorVerdict,
base: String,
commit: String,
wt: PathBuf,
branch: String,
},
Blocked {
outcome: &'static str,
diff: Option<String>,
status: &'static str,
reason: String,
findings: Vec<String>,
floor: Option<FloorVerdict>,
floor_passed: Option<bool>,
recodable: bool,
wt: PathBuf,
branch: String,
},
}
fn upsert_chunk_report(run: &mut Run, report: ChunkReport) {
run.chunk_reports.retain(|r| r.id != report.id);
run.chunk_reports.push(report);
}
#[allow(clippy::too_many_arguments)]
fn push_blocked_chunk(
run: &mut Run,
chunk: &Chunk,
outcome: &str,
floor: Option<FloorVerdict>,
floor_passed: Option<bool>,
reason: String,
chunk_wt: &Path,
chunk_branch: &str,
) {
run.preserved
.push((chunk_wt.to_path_buf(), chunk_branch.to_string()));
upsert_chunk_report(
run,
ChunkReport {
id: chunk.id.clone(),
title: chunk.title.clone(),
tier: chunk.tier.wire_name().to_string(),
outcome: outcome.to_string(),
floor_passed,
floor,
merged: false,
commit: None,
merge_commit: None,
replayed: false,
reason: Some(reason),
branch_preserved: Some(chunk_branch.to_string()),
},
);
}
fn gate_chunk(
cfg: &PipelineConfig,
repo: &Path,
chunk: &Chunk,
chunk_wt: &Path,
base_commit: &str,
changed: &[PathBuf],
baseline: &BaselineSnapshot,
) -> Result<FloorVerdict, PipelineError> {
let check_results: Vec<CheckRun> = floor::runner::run_checks(&chunk.checks, chunk_wt);
let current = capture_snapshot(cfg, chunk_wt)?;
let declared: Vec<PathBuf> = chunk.files_touched.iter().map(PathBuf::from).collect();
let baseline_assertions = floor::runner::assertion_counts_at_ref(repo, base_commit, &declared)?;
let current_assertions = floor::runner::assertion_counts_on_disk(chunk_wt, &declared);
let inputs = FloorInputs {
baseline: &baseline.snapshot,
current: ¤t,
check_results: &check_results,
declared_files: &declared,
changed_files: changed,
baseline_assertions: &baseline_assertions,
current_assertions: ¤t_assertions,
file_scope_slack: cfg.file_scope_slack,
};
Ok(evaluate_floor(&inputs))
}
fn run_verify_stage(
run: &mut Run,
plan: &Plan,
verify: &dyn VerifyProvider,
) -> Result<(VerifyReport, VerifyDisposition), PipelineError> {
let acceptance_checks: Vec<plan::Check> = plan
.acceptance
.iter()
.filter_map(acceptance_to_check)
.collect();
let acceptance_results = floor::runner::run_checks(&acceptance_checks, &run.integration_wt);
let acceptance_checks_passed = acceptance_results.iter().all(|r| r.passed);
let judgment: VerifyJudgment = verify.verify(&VerifyContext {
intent: &run.cfg.intent,
plan,
worktree: &run.integration_wt,
acceptance_results: &acceptance_results,
})?;
run.meter.record_agent_run(None);
run.decisions.push(envelope(
"verify",
DecisionTier::Decider,
format!(
"acceptance checks {}, judge {}",
if acceptance_checks_passed {
"passed"
} else {
"FAILED"
},
if judgment.passed { "passed" } else { "FAILED" }
),
vec![format!("plan:{}", plan.plan_rev)],
verify.model(),
verify.prompt_version(),
));
let passed = acceptance_checks_passed && judgment.passed;
let disposition = if judgment.passed {
VerifyDisposition::Fix
} else {
judgment.disposition.clone()
};
let mut findings = judgment.findings;
for r in acceptance_results.iter().filter(|r| !r.passed) {
findings.push(format!("acceptance check failed: {} (`{}`)", r.desc, r.run));
}
if !passed && findings.is_empty() {
findings.push(format!(
"verify failed without specific findings: {}. Review the implementation against the intent and correct it.",
summary_or_default(&judgment.summary)
));
}
Ok((
VerifyReport {
acceptance_checks_passed,
judged_passed: judgment.passed,
passed,
summary: judgment.summary,
findings,
},
disposition,
))
}
fn summary_or_default(summary: &str) -> &str {
if summary.trim().is_empty() {
"(no summary)"
} else {
summary
}
}
fn acceptance_to_check(a: &Acceptance) -> Option<plan::Check> {
match a {
Acceptance::Check {
desc,
run,
cwd,
expect_exit,
} => Some(plan::Check {
desc: desc.clone(),
run: run.clone(),
cwd: cwd.clone(),
expect_exit: *expect_exit,
extra: serde_json::Map::new(),
}),
Acceptance::Assertion { .. } => None,
}
}
fn evaluate_feature_floor(
run: &Run,
plan: &Plan,
baseline: &BaselineSnapshot,
declared: &[PathBuf],
feat_tip: &str,
) -> Result<FloorVerdict, PipelineError> {
let acceptance_checks: Vec<plan::Check> = plan
.acceptance
.iter()
.filter_map(acceptance_to_check)
.collect();
let check_results = floor::runner::run_checks(&acceptance_checks, &run.integration_wt);
let current = capture_snapshot(run.cfg, &run.integration_wt)?;
let changed = floor::git::changed_files(&run.integration_wt, &run.fork_commit, feat_tip)?;
let baseline_assertions =
floor::runner::assertion_counts_at_ref(&run.repo, &run.fork_commit, declared)?;
let current_assertions = floor::runner::assertion_counts_on_disk(&run.integration_wt, declared);
let inputs = FloorInputs {
baseline: &baseline.snapshot,
current: ¤t,
check_results: &check_results,
declared_files: declared,
changed_files: &changed,
baseline_assertions: &baseline_assertions,
current_assertions: ¤t_assertions,
file_scope_slack: run.cfg.file_scope_slack,
};
Ok(evaluate_floor(&inputs))
}
fn merge_feature_to_source(run: &Run, feat_tip: &str) -> Result<MergeOutcome, PipelineError> {
let message = format!(
"pipeline: merge {} into {}",
run.integration_branch, run.cfg.source_branch
);
if let Some(src_wt) = git::worktree_for_branch(&run.repo, &run.cfg.source_branch)? {
if !git::is_clean(&src_wt)? {
return Err(PipelineError::Setup(format!(
"source branch `{}` worktree {} is dirty; cannot merge",
run.cfg.source_branch,
src_wt.display()
)));
}
git::merge_no_ff(&src_wt, feat_tip, &message)
} else {
let src_wt = run.cfg.workdir.join("source-merge");
git::worktree_add(&run.repo, &src_wt, &run.cfg.source_branch)?;
let out = git::merge_no_ff(&src_wt, feat_tip, &message);
let _ = git::worktree_remove(&run.repo, &src_wt);
out
}
}
fn finalize(
run: &Run,
plan: &Plan,
verify: Option<VerifyReport>,
merged: bool,
final_commit: Option<String>,
status: &str,
) -> PipelineReport {
let failure = match status {
"merged" => None,
"chunk_floor_blocked" => {
Some("a chunk failed the deterministic floor; the feature was not merged".to_string())
}
"chunk_merge_conflict" => {
Some("a chunk floor-passed but conflicted merging into the integration branch".to_string())
}
"chunk_failed" => {
Some("a chunk failed to produce a mergeable commit (harness failure / no change / timeout)".to_string())
}
"verify_failed" => {
Some("verify judged the product does not match intent (or an acceptance check failed); not merged".to_string())
}
"floor_blocked" => Some("the feature floor regressed at the tip; not merged".to_string()),
"rollback_conflict" => Some(format!(
"the provenance rollback could not cleanly replay kept chunk{} onto the rebuilt integration branch; not merged — the integration branch was restored intact and its work preserved",
run.rollback_conflict
.as_ref()
.map(|c| format!(" `{c}`"))
.unwrap_or_default()
)),
"escalated" => Some(
"the decider escalated a consequential decision (declined to converge or re-spec) and handed the feature up; not merged — see the decision log for the reason".to_string(),
),
"merge_conflict" => {
Some("the feature floor was green but the source branch moved and the merge conflicted".to_string())
}
"circuit_breaker" => Some(
run.circuit_breaker
.clone()
.unwrap_or_else(|| "a circuit-breaker stopped the fix loop; not merged".to_string()),
),
other => Some(other.to_string()),
};
PipelineReport {
slug: run.slug.clone(),
source_branch: run.cfg.source_branch.clone(),
integration_branch: run.integration_branch.clone(),
intent_rev: 1,
plan_rev: plan.plan_rev,
chunk_count: plan.chunks.len(),
chunks: run.chunk_reports.clone(),
verify,
feature_floor: run.feature_floor.clone(),
merged,
final_commit,
status: status.to_string(),
decisions: run.decisions.clone(),
recode_count: run.recode_count,
promote_count: run.promote_count,
respec_count: run.respec_count,
circuit_breaker: run.circuit_breaker.clone(),
resources: run.meter.clone(),
failure,
}
}
fn teardown(run: &Run) {
if run.cfg.keep {
return;
}
let _ = git::worktree_remove(&run.repo, &run.integration_wt);
let safe_to_delete = run.merged_to_source
|| git::commits_ahead_of(&run.repo, &run.cfg.source_branch, &run.integration_branch)
.is_ok_and(|n| n == 0);
if safe_to_delete {
let branch_deleted =
git::delete_branch(&run.repo, &run.integration_branch, run.merged_to_source).is_ok();
if branch_deleted {
if let Ok(refs) = git::refs_under(&run.repo, &provenance_ref_prefix(&run.slug)) {
for r in &refs {
let _ = git::delete_ref(&run.repo, r);
}
}
}
}
}
pub struct PipelineRunConfig {
pub intent: String,
pub source_branch: String,
pub files: Vec<PathBuf>,
pub slug: Option<String>,
pub repo: Option<PathBuf>,
pub test_cmd: Option<String>,
pub clippy_cmd: Option<String>,
pub workdir: Option<PathBuf>,
pub file_scope_slack: usize,
pub keep: bool,
pub chunk_timeout_secs: Option<u64>,
pub max_build_concurrency: Option<usize>,
pub max_recode_per_chunk: Option<u32>,
pub max_fix_iterations: Option<u32>,
pub max_respec: Option<u32>,
pub max_promotions: Option<u32>,
pub max_cost_usd: Option<f64>,
pub max_total_tokens: Option<u64>,
pub max_wall_time_secs: Option<u64>,
pub max_processes: Option<u32>,
pub max_storage_mb: Option<u64>,
pub max_identical_failures: Option<u32>,
}
fn resolve_u64_ceiling(user: Option<u64>, default: Option<u64>) -> Option<u64> {
match user {
Some(0) => None,
Some(v) => Some(v),
None => default,
}
}
fn resolve_u32_ceiling(user: Option<u32>, default: Option<u32>) -> Option<u32> {
match user {
Some(0) => None,
Some(v) => Some(v),
None => default,
}
}
fn resolve_f64_ceiling(user: Option<f64>, default: Option<f64>) -> Option<f64> {
match user {
Some(v) if v.is_finite() && v > 0.0 => Some(v),
Some(_) => None,
None => default,
}
}
pub fn cmd_run(
cfg: &PipelineRunConfig,
spec: &OutputSpec,
warnings: &[String],
) -> Result<(), CliError> {
let intent = resolve_intent(&cfg.intent)?;
let repo = cfg
.repo
.clone()
.unwrap_or_else(|| std::env::current_dir().unwrap_or_else(|_| PathBuf::from(".")));
let slug_preview = cfg.slug.clone().unwrap_or_else(|| slugify(&intent));
let workdir = cfg.workdir.clone().unwrap_or_else(|| {
std::env::temp_dir()
.join("octl-pipeline")
.join(&slug_preview)
});
let pcfg = PipelineConfig {
repo,
intent,
source_branch: cfg.source_branch.clone(),
files: cfg.files.clone(),
slug: cfg.slug.clone(),
test_cmd: cfg
.test_cmd
.clone()
.unwrap_or_else(|| "cargo test".to_string()),
clippy_cmd: cfg
.clippy_cmd
.clone()
.unwrap_or_else(|| "cargo clippy".to_string()),
workdir,
file_scope_slack: cfg.file_scope_slack,
keep: cfg.keep,
chunk_timeout: cfg.chunk_timeout_secs.map(Duration::from_secs),
max_build_concurrency: cfg.max_build_concurrency.filter(|&n| n > 1).unwrap_or(1),
fix_loop: {
let d = FixLoopConfig::live_default();
FixLoopConfig {
max_recode_per_chunk: cfg.max_recode_per_chunk.unwrap_or(d.max_recode_per_chunk),
max_fix_iterations: cfg.max_fix_iterations.unwrap_or(d.max_fix_iterations),
max_respec: cfg.max_respec.unwrap_or(d.max_respec),
max_promotions: cfg.max_promotions.unwrap_or(d.max_promotions),
}
},
budget: {
let d = ResourceBudget::live_default();
ResourceBudget {
max_cost_usd: resolve_f64_ceiling(cfg.max_cost_usd, d.max_cost_usd),
max_total_tokens: resolve_u64_ceiling(cfg.max_total_tokens, d.max_total_tokens),
max_wall_time: resolve_u64_ceiling(
cfg.max_wall_time_secs,
d.max_wall_time.map(|w| w.as_secs()),
)
.map(Duration::from_secs),
max_processes: resolve_u32_ceiling(cfg.max_processes, d.max_processes),
max_storage_bytes: resolve_u64_ceiling(
cfg.max_storage_mb,
d.max_storage_bytes.map(|b| b / (1024 * 1024)),
)
.map(|mb| mb.saturating_mul(1024 * 1024)),
max_identical_failures: resolve_u32_ceiling(
cfg.max_identical_failures,
d.max_identical_failures,
),
}
},
};
let spec_provider = providers::ClaudeSpecProvider;
let verify_provider = providers::ClaudeVerifyProvider;
use crate::harness::claude::ClaudeHarness;
let harnesses = LiveTierHarness {
code: ClaudeHarness::deepseek("flash"),
mid: ClaudeHarness::deepseek("pro"),
high: ClaudeHarness::claude(Some("opus".to_string())),
};
let decider = LiveDecider {
model: verify_provider.model(),
};
let report = match run_pipeline_tiered(
&pcfg,
&spec_provider,
&harnesses,
&verify_provider,
&decider,
) {
Ok(report) => report,
Err(PipelineFailure { error, report }) => {
if let Some(report) = report {
match spec.format {
OutputFormat::Json | OutputFormat::Jsonl => {
if let Err(e) = output::emit_envelope(&report, spec, warnings) {
eprintln!("warning: could not render failure report: {}", e.message);
}
}
OutputFormat::Text => {
print_report(&report);
output::emit_text_warnings(warnings);
}
}
}
return Err(error.into());
}
};
match spec.format {
OutputFormat::Json | OutputFormat::Jsonl => output::emit_envelope(&report, spec, warnings)?,
OutputFormat::Text => {
print_report(&report);
output::emit_text_warnings(warnings);
}
}
Ok(())
}
fn human_bytes(bytes: u64) -> String {
const UNITS: [&str; 5] = ["B", "KiB", "MiB", "GiB", "TiB"];
let mut v = bytes as f64;
let mut unit = 0;
while v >= 1024.0 && unit < UNITS.len() - 1 {
v /= 1024.0;
unit += 1;
}
if unit == 0 {
format!("{bytes} B")
} else {
format!("{v:.1} {}", UNITS[unit])
}
}
fn print_report(r: &PipelineReport) {
println!("pipeline {} — {}", r.slug, r.status);
println!(
" source: {} integration: {}",
r.source_branch, r.integration_branch
);
println!(" chunks: {}", r.chunk_count);
for c in &r.chunks {
let floor = match c.floor_passed {
Some(true) => "floor:green",
Some(false) => "floor:BLOCKED",
None => "floor:-",
};
let merged = if c.merged { "merged" } else { "not-merged" };
println!(
" [{}] {} — {} {} {}",
c.id, c.title, c.outcome, floor, merged
);
if let Some(reason) = &c.reason {
println!(" reason: {}", output::escape_one_line(reason));
}
if let Some(branch) = &c.branch_preserved {
println!(" preserved: {}", output::escape_one_line(branch));
}
}
if let Some(v) = &r.verify {
println!(
" verify: {} (acceptance-checks: {}, judge: {}) — {}",
if v.passed { "passed" } else { "FAILED" },
v.acceptance_checks_passed,
v.judged_passed,
output::escape_one_line(&v.summary)
);
}
if r.recode_count > 0 || r.respec_count > 0 || r.promote_count > 0 {
println!(
" fix loop: {} re-code(s), {} promotion(s), {} re-spec(s) → plan.v{}",
r.recode_count, r.promote_count, r.respec_count, r.plan_rev
);
}
match (&r.merged, &r.final_commit) {
(true, Some(commit)) => println!(" merged → {} @ {}", r.source_branch, commit),
_ => println!(" merged: no"),
}
{
let res = &r.resources;
println!(
" resources: {} token(s), ${:.4}, {} agent invocation(s), {} scratch storage",
res.total_tokens,
res.cost_usd,
res.processes,
human_bytes(res.storage_bytes)
);
}
if let Some(cb) = &r.circuit_breaker {
println!(" circuit-breaker: {}", output::escape_one_line(cb));
}
if let Some(f) = &r.failure {
println!(" failure: {}", output::escape_one_line(f));
}
for d in &r.decisions {
println!(
" decision[{}] {}: {}",
match d.decision_tier {
DecisionTier::Coordinator => "coordinator",
DecisionTier::Decider => "decider",
},
d.actor,
output::escape_one_line(&d.reason)
);
}
}
#[cfg(test)]
mod tests;