use std::collections::{BTreeMap, BTreeSet};
use std::num::{NonZeroU32, NonZeroU64};
use std::sync::mpsc::{self, Receiver, Sender};
use std::time::{Duration, Instant};
use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
use crate::agentgraph::{self, Interrupted, TurnAddress};
use crate::channel::{ChannelState, Command, CommandOutcome, Surface};
use crate::edits::{self, Frontier};
use crate::error::{Error, Result};
use crate::event::{Envelope, Labels};
use crate::executor::{
CancelMode, CancellationToken, DispatchHandle, DispatchOutcome, DispatchRequest, Executor,
WorkspaceSpec,
};
use crate::graph::{self, Graph, GraphState, Landing, NodeStatus};
use crate::journal::{self, Journal};
use crate::ledger::{self, LaunchRecord, OwnershipLock, RunPaths};
use crate::plan::{Node, NodeKind};
use crate::projection::{self, RunState};
use crate::rules::ExecutorRules;
use crate::sys;
pub const NODE_GRAPH_ENV: &str = "ONEPIPELINE_NODE_GRAPH";
pub const DEFAULT_NODE_GRAPH: &str = "graphs/node-scope.yaml";
pub const EXECUTOR_RULES_ENV: &str = "ONEPIPELINE_EXECUTOR_RULES";
pub const NODE_VALIDATOR_ENV: &str = "ONEPIPELINE_NODE_VALIDATOR";
pub const ENVELOPE_REVIEWER_ENV: &str = "ONEPIPELINE_ENVELOPE_REVIEWER";
pub const PROJECT_DIR_ENV: &str = "ONEPIPELINE_PROJECT_DIR";
pub const DRIVE_VERB: &str = "drive-run";
pub const STALL_AFTER_ENV: &str = "ONEPIPELINE_STALL_AFTER_SECONDS";
pub const DEFAULT_STALL_AFTER_SECONDS: u64 = 2_400;
pub const BOUNDARY_ATTEMPTS_ENV: &str = "ONEPIPELINE_BOUNDARY_ATTEMPTS";
pub const BOUNDARY_BACKOFF_ENV: &str = "ONEPIPELINE_BOUNDARY_BACKOFF_SECONDS";
pub const DEFAULT_BOUNDARY_ATTEMPTS: u32 = 3;
pub const DEFAULT_BOUNDARY_BACKOFF_SECONDS: u64 = 5;
const BOUNDARY_BACKOFF_CEILING: Duration = Duration::from_secs(120);
pub const PUBLICATION_ATTEMPTS_ENV: &str = "ONEPIPELINE_PUBLICATION_ATTEMPTS";
pub const DEFAULT_PUBLICATION_ATTEMPTS: NonZeroU32 = NonZeroU32::new(3).unwrap();
pub const MERGE_PATH_READS_ENV: &str = "ONEPIPELINE_MERGE_PATH_READS";
pub const MERGE_PATH_BACKOFF_ENV: &str = "ONEPIPELINE_MERGE_PATH_BACKOFF_SECONDS";
pub const DEFAULT_MERGE_PATH_READS: NonZeroU32 = NonZeroU32::new(3).unwrap();
pub const DEFAULT_MERGE_PATH_BACKOFF_SECONDS: u64 = 5;
pub const CANCEL_GRACE_ENV: &str = "ONEPIPELINE_CANCEL_GRACE_SECONDS";
pub const DEFAULT_CANCEL_GRACE_SECONDS: u64 = 300;
pub const CANCEL_INPUT: &str = "Stop this task now. Do not start any new work, and do not begin \
another file, command, or tool call. Commit anything you have not \
committed yet, then end your turn.";
const DRAIN_WINDOW: Duration = Duration::from_millis(25);
const TEARDOWN_TICK: Duration = Duration::from_millis(25);
const CHANNEL_POLL: Duration = Duration::from_millis(200);
pub(crate) const UPSTREAM_EVERY: Duration = Duration::from_millis(500);
pub const RUN_RESULT_SCHEMA_VERSION: u32 = 5;
fn readable_run_result_version<'de, D: serde::Deserializer<'de>>(
reader: D,
) -> std::result::Result<u32, D::Error> {
let found = u32::deserialize(reader)?;
if found != RUN_RESULT_SCHEMA_VERSION {
return Err(serde::de::Error::custom(format!(
"run result schema_version {found}, and this build reads \
{RUN_RESULT_SCHEMA_VERSION}"
)));
}
Ok(found)
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(into = "RunResultWire", from = "RunResultWire")]
pub struct RunResult {
pub run_id: String,
pub state: GraphState,
pub nodes: Vec<NodeResult>,
}
impl RunResult {
pub fn ok(&self) -> bool {
self.state == GraphState::Complete
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
struct RunResultWire {
#[serde(deserialize_with = "readable_run_result_version")]
schema_version: u32,
run_id: String,
state: GraphState,
ok: bool,
nodes: Vec<NodeResult>,
}
impl From<RunResult> for RunResultWire {
fn from(result: RunResult) -> Self {
Self {
schema_version: RUN_RESULT_SCHEMA_VERSION,
ok: result.ok(),
run_id: result.run_id,
state: result.state,
nodes: result.nodes,
}
}
}
impl From<RunResultWire> for RunResult {
fn from(wire: RunResultWire) -> Self {
Self {
run_id: wire.run_id,
state: wire.state,
nodes: wire.nodes,
}
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct NodeResult {
pub id: String,
pub status: NodeStatus,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub outcome: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub landing: Option<Landing>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub action: Option<String>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub unblocks: Vec<String>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub blocked_by: Vec<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub branch: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub change_url: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cause: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub head: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub superseded_by: Option<String>,
}
#[derive(Debug, Clone, PartialEq)]
pub struct Settlement {
pub node: String,
pub status: NodeStatus,
pub outcome: Option<String>,
pub landing: Option<Landing>,
pub detail: Option<String>,
pub branch: Option<String>,
pub change_url: Option<String>,
pub cause: Option<String>,
pub head: Option<String>,
pub completed_steps: Vec<String>,
}
impl Settlement {
pub fn plain(node: &str, status: NodeStatus, outcome: Option<&str>) -> Self {
Self {
node: node.to_string(),
status,
outcome: outcome.map(str::to_string),
landing: None,
detail: None,
branch: None,
change_url: None,
cause: None,
head: None,
completed_steps: Vec::new(),
}
}
}
pub(crate) enum Message {
Event(Box<Envelope>),
Redispatched(Box<Redispatch>),
Cancelling(Box<Cancelling>),
BodyNotDrafted(Box<UndraftedBody>),
CriterionChecked(Box<CriterionChecked>),
Settled(Box<Settlement>),
}
pub(crate) struct CriterionChecked {
pub node: crate::graph::NodeRef,
pub check: crate::criteria::Checkable,
pub answer: crate::criteria::Answer,
}
pub(crate) struct UndraftedBody {
pub node: String,
pub ending: crate::lifecycle::Undrafted,
}
pub(crate) struct Cancelling {
pub node: String,
pub phase: CancelPhase,
pub detail: String,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum CancelPhase {
Interrupted,
Killed,
}
impl CancelPhase {
fn kind(self) -> &'static str {
match self {
Self::Interrupted => "dispatch-interrupted",
Self::Killed => "dispatch-killed",
}
}
}
pub(crate) struct Redispatch {
pub node: String,
pub attempt: NonZeroU32,
pub attempts: NonZeroU32,
pub reason: String,
}
struct Dispatch {
node: Node,
cancel: CancellationToken,
started: Instant,
last_progress: Instant,
reported_quiet: bool,
control: Option<TurnAddress>,
}
impl Dispatch {
fn live(&self) -> edits::LiveDispatch {
edits::LiveDispatch {
graph_run: self.control.as_ref().map(|at| at.run().to_string()),
running_for_seconds: self.started.elapsed().as_secs(),
}
}
}
pub fn claim(paths: &RunPaths) -> Result<OwnershipLock> {
OwnershipLock::acquire(paths, "drive")
}
pub fn drive_holding(paths: &RunPaths, lock: OwnershipLock) -> Result<GraphState> {
let launch: LaunchRecord = ledger::read_json(&paths.launch())?;
if launch.node_graph.is_empty() {
return Err(Error::Invalid(format!(
"launch record for run '{}' has no resolved node graph",
paths.run
)));
}
let mut journal = Journal::open(paths);
let mut state = projection::fold(&journal::read(&paths.journal()));
report_unreadable_records(paths, &state);
let outcome = converge(paths, &mut journal, &mut state, &launch)?;
record_result(paths, &state, outcome)?;
lock.release();
Ok(outcome)
}
fn converge(
paths: &RunPaths,
journal: &mut Journal,
state: &mut RunState,
launch: &LaunchRecord,
) -> Result<GraphState> {
let writeback = crate::taskgraph::Store::resolve()
.ok()
.and_then(|store| crate::writeback::Writeback::start(store.binary(), paths, launch));
let channel = ChannelState::new(paths);
let rules = executor_rules()?;
let (tx, rx): (Sender<Message>, Receiver<Message>) = mpsc::channel();
let mut in_flight: BTreeMap<String, Dispatch> = BTreeMap::new();
let stall_after = Duration::from_secs(stall_after_seconds());
let mut upstreams = crate::crossdag::Observer::of_run(paths, state);
let mut releases = crate::release::Watch::of_run(paths);
let mut announced_ready: BTreeSet<String> = BTreeSet::new();
let mut held: BTreeMap<DecisionRef, Decision> = state
.decisions_pending
.iter()
.map(|(reference, pending)| {
(
DecisionRef::of_wire(reference),
Decision {
reference: DecisionRef::of_wire(reference),
kind: pending.kind.clone(),
unblocks: pending.unblocks.clone(),
},
)
})
.collect();
let mut holding: BTreeMap<String, Vec<HoldReason>> = state
.holds
.iter()
.filter_map(|(node, reasons)| {
let read: Option<Vec<HoldReason>> =
reasons.iter().map(HoldReason::of_payload).collect();
read.map(|reasons| (node.clone(), reasons))
})
.collect();
let mut derived: Option<BTreeMap<String, NodeStatus>> = None;
let mut read_upstreams: Option<Instant> = None;
let mut took_up_releases;
let mut unpublished = true;
let mut channel_seen = channel.fingerprint();
let mut upstream_seen = upstreams.marks(&state.graph);
let mut upstream_looked = Instant::now();
loop {
crate::loopstats::pass();
let mut moved = false;
if reconcile_edits(paths, journal, state, &channel, launch, &mut in_flight)? {
derived = None;
unpublished = true;
moved = true;
}
let has_upstreams = !crate::crossdag::edges(&state.graph).is_empty();
if has_upstreams && due(read_upstreams, UPSTREAM_EVERY) {
read_upstreams = Some(Instant::now());
let resolved = upstreams.resolve(&state.graph, paths, journal)?;
if resolved != state.cross_dag {
state.cross_dag = resolved;
derived = None;
unpublished = true;
}
}
let statuses = statuses_of(&mut derived, state);
announce_ready(paths, journal, &statuses, &mut announced_ready)?;
let decisions = decisions_now(state, &statuses, &channel);
report_decisions(paths, journal, &decisions, &mut held)?;
let mut paused = paused_by(&decisions);
let watching = crate::release::watching(
state,
&statuses,
&in_flight.keys().cloned().collect::<BTreeSet<String>>(),
);
releases.refresh(paths, state, &watching);
let held_for_release = releases.held(&watching);
releases.report(paths, journal, &held_for_release, &watching)?;
releases.relay_releases(
paths,
journal,
state,
&statuses,
launch.filters.vcs.as_ref(),
)?;
took_up_releases = Some(Instant::now());
let awaiting_release: BTreeMap<String, Vec<String>> = held_for_release
.iter()
.map(|node| (node.clone(), releases.awaited_deps(node)))
.collect();
paused.extend(held_for_release);
if adopt_releases(paths, journal, state, &statuses, &mut releases, &in_flight)? {
derived = None;
unpublished = true;
moved = true;
}
let statuses = statuses_of(&mut derived, state);
if start_ready(
paths,
journal,
state,
&statuses,
&rules,
launch,
&tx,
&mut in_flight,
&paused,
&releases,
)? {
derived = None;
unpublished = true;
moved = true;
}
let statuses = statuses_of(&mut derived, state);
if unpublished {
if let Some(writeback) = &writeback {
writeback.publish(paths, launch, state, &statuses);
}
unpublished = false;
}
report_holds(
paths,
journal,
&holds_now(state, &statuses, &in_flight, &decisions, &awaiting_release),
&mut holding,
)?;
if let Some(writeback) = &writeback {
report_unprojected(paths, journal, writeback)?;
}
if in_flight.is_empty() {
if graph::is_terminal(&statuses) {
break;
}
if !any_node_can_still_move(&statuses) {
break;
}
}
let next = [
if has_upstreams {
until_due(read_upstreams, UPSTREAM_EVERY)
} else {
Duration::MAX
},
if releases.names_a_release_dependency() || releases.relays_anything(state) {
until_due(took_up_releases, releases.take_up_every())
} else {
Duration::MAX
},
next_quiet(&in_flight, stall_after),
];
let deadline = if moved {
Duration::ZERO
} else {
next.into_iter().min().unwrap_or(Duration::MAX)
};
let arrived = {
let mut outside = || {
if releases.take_up_answers() {
return true;
}
if writeback
.as_ref()
.is_some_and(crate::writeback::Writeback::has_unprojected)
{
return true;
}
if !has_upstreams || upstream_looked.elapsed() < UPSTREAM_EVERY {
return false;
}
upstream_looked = Instant::now();
let now = upstreams.marks(&state.graph);
if now == upstream_seen {
return false;
}
upstream_seen = now;
true
};
wait_for_work(
paths,
&rx,
&channel,
&mut channel_seen,
deadline,
&mut outside,
)?
};
let Some(arrived) = arrived else {
break;
};
for message in arrived {
match message {
Message::Event(envelope) => {
if let Some(node) = envelope.labels.node.clone() {
if let Some(dispatch) = in_flight.get_mut(&node) {
if projection::evidences_progress(&envelope) {
dispatch.last_progress = Instant::now();
dispatch.reported_quiet = false;
}
if let Some(address) = addressed_by(&envelope) {
dispatch.control = Some(address);
}
}
}
journal.relay(&envelope)?;
}
Message::Redispatched(again) => journal.emit(
journal::PipelineKind::NodeDispatched,
journal::labels(&paths.run, Some(&again.node)),
journal::payload(&[
("attempt", json!(again.attempt)),
("attempts", json!(again.attempts)),
("reason", json!(bounded(&again.reason))),
]),
)?,
Message::Cancelling(step) => raise(paths, journal, cancelling_surface(&step))?,
Message::BodyNotDrafted(undrafted) => journal.emit(
journal::PipelineKind::BodyNotDrafted,
journal::labels(&paths.run, Some(&undrafted.node)),
journal::payload(&[
("ending", json!(undrafted.ending.ending())),
("detail", json!(undrafted.ending.why())),
]),
)?,
Message::CriterionChecked(checked) => {
journal.emit(
journal::PipelineKind::CriterionChecked,
journal::labels(&paths.run, Some(checked.node.as_str())),
criterion_payload(&checked),
)?;
if let crate::criteria::Answer::Mismatch { holds } = &checked.answer {
raise(paths, journal, criterion_finding(&checked, holds))?;
}
}
Message::Settled(settlement) => {
in_flight.remove(&settlement.node);
settle(paths, journal, &settlement)?;
*state = projection::fold(&journal::read(&paths.journal()));
derived = None;
unpublished = true;
}
}
}
watch_for_quiet(paths, journal, stall_after, &mut in_flight)?;
}
let final_statuses = statuses_of(&mut derived, state);
crate::loopstats::flush(paths)?;
if let Some(writeback) = &writeback {
writeback.publish(paths, launch, state, &final_statuses);
writeback.wait_briefly();
report_unprojected(paths, journal, writeback)?;
}
Ok(graph::state_of(&final_statuses))
}
pub(crate) fn due(last: Option<Instant>, every: Duration) -> bool {
last.is_none_or(|last| last.elapsed() >= every)
}
fn until_due(last: Option<Instant>, every: Duration) -> Duration {
last.map_or(Duration::ZERO, |last| every.saturating_sub(last.elapsed()))
}
fn next_quiet(in_flight: &BTreeMap<String, Dispatch>, stall_after: Duration) -> Duration {
in_flight
.values()
.filter(|dispatch| !dispatch.reported_quiet)
.map(|dispatch| stall_after.saturating_sub(dispatch.last_progress.elapsed()))
.min()
.unwrap_or(Duration::MAX)
}
fn statuses_of(
cache: &mut Option<BTreeMap<String, NodeStatus>>,
state: &RunState,
) -> BTreeMap<String, NodeStatus> {
cache.get_or_insert_with(|| state.statuses()).clone()
}
fn wait_for_work(
paths: &RunPaths,
rx: &Receiver<Message>,
channel: &ChannelState,
seen: &mut crate::channel::Fingerprint,
deadline: Duration,
outside: &mut dyn FnMut() -> bool,
) -> Result<Option<Vec<Message>>> {
let waiting_since = Instant::now();
loop {
crate::loopstats::flush(paths)?;
let left = deadline.saturating_sub(waiting_since.elapsed());
match rx.recv_timeout(CHANNEL_POLL.min(left)) {
Ok(message) => {
let mut batch = vec![message];
let drain_started = Instant::now();
while drain_started.elapsed() < DRAIN_WINDOW {
let Ok(message) = rx.try_recv() else {
break;
};
batch.push(message);
}
return Ok(Some(batch));
}
Err(mpsc::RecvTimeoutError::Timeout) => {}
Err(mpsc::RecvTimeoutError::Disconnected) => return Ok(None),
}
let now = channel.fingerprint();
if now != *seen {
*seen = now;
return Ok(Some(Vec::new()));
}
if outside() {
return Ok(Some(Vec::new()));
}
if waiting_since.elapsed() >= deadline {
return Ok(Some(Vec::new()));
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
enum HoldReason {
Dependencies { blocking: Vec<String> },
Concurrency {
ahead: Vec<String>,
limit: usize,
},
Decision { reference: DecisionRef },
Release { awaiting: Vec<String> },
}
impl HoldReason {
fn payload(&self) -> Value {
match self {
Self::Dependencies { blocking } => {
json!({ "kind": "dependencies", "blocking": blocking })
}
Self::Concurrency { ahead, limit } => {
json!({ "kind": "concurrency", "ahead": ahead, "limit": limit })
}
Self::Decision { reference } => {
json!({ "kind": "decision", "reference": reference.as_wire() })
}
Self::Release { awaiting } => json!({ "kind": "release", "awaiting": awaiting }),
}
}
fn of_payload(entry: &Value) -> Option<Self> {
let ids = |key: &str| -> Option<Vec<String>> {
entry
.get(key)?
.as_array()?
.iter()
.map(|id| id.as_str().map(str::to_string))
.collect()
};
match entry.get("kind")?.as_str()? {
"dependencies" => Some(Self::Dependencies {
blocking: ids("blocking")?,
}),
"concurrency" => Some(Self::Concurrency {
ahead: ids("ahead")?,
limit: usize::try_from(entry.get("limit")?.as_u64()?).ok()?,
}),
"decision" => Some(Self::Decision {
reference: DecisionRef::of_wire(entry.get("reference")?.as_str()?),
}),
"release" => Some(Self::Release {
awaiting: ids("awaiting")?,
}),
_ => None,
}
}
}
fn holds_now(
state: &RunState,
statuses: &BTreeMap<String, NodeStatus>,
in_flight: &BTreeMap<String, Dispatch>,
decisions: &BTreeMap<DecisionRef, Decision>,
awaiting_release: &BTreeMap<String, Vec<String>>,
) -> BTreeMap<String, Vec<HoldReason>> {
let concurrency = state.graph.concurrency as usize;
let ahead: Vec<String> = in_flight.keys().cloned().collect();
let mut holds: BTreeMap<String, Vec<HoldReason>> = BTreeMap::new();
for node in state.graph.iter() {
if in_flight.contains_key(&node.id) {
continue;
}
let status = statuses
.get(&node.id)
.copied()
.unwrap_or(NodeStatus::Pending);
if !matches!(
status,
NodeStatus::Pending
| NodeStatus::Ready
| NodeStatus::Blocked
| NodeStatus::CompleteDraft
) {
continue;
}
let mut reasons: Vec<HoldReason> = Vec::new();
let blocking = unsettled_deps(state, statuses, node);
if !blocking.is_empty() {
reasons.push(HoldReason::Dependencies { blocking });
}
if status == NodeStatus::Ready
&& node.kind != NodeKind::Human
&& in_flight.len() >= concurrency
{
reasons.push(HoldReason::Concurrency {
ahead: ahead.clone(),
limit: concurrency,
});
}
for decision in decisions.values() {
if decision.unblocks.contains(&node.id) {
reasons.push(HoldReason::Decision {
reference: decision.reference.clone(),
});
}
}
if let Some(awaiting) = awaiting_release.get(&node.id) {
reasons.push(HoldReason::Release {
awaiting: awaiting.clone(),
});
}
if !reasons.is_empty() {
holds.insert(node.id.clone(), reasons);
}
}
holds
}
fn unsettled_deps(
state: &RunState,
statuses: &BTreeMap<String, NodeStatus>,
node: &Node,
) -> Vec<String> {
node.deps
.iter()
.filter(|dep| {
let status = if crate::crossdag::is_reference(dep) {
state.cross_dag.get(*dep).copied()
} else if state.graph.contains(dep) {
statuses.get(*dep).copied()
} else {
return false;
};
status != Some(NodeStatus::Done)
})
.cloned()
.collect()
}
fn report_holds(
paths: &RunPaths,
journal: &mut Journal,
holds: &BTreeMap<String, Vec<HoldReason>>,
reported: &mut BTreeMap<String, Vec<HoldReason>>,
) -> Result<()> {
for (node, reasons) in holds {
if reported.get(node) == Some(reasons) {
continue;
}
journal.emit(
journal::PipelineKind::NodeHeld,
journal::labels(&paths.run, Some(node)),
journal::payload(&[(
"reasons",
Value::Array(reasons.iter().map(HoldReason::payload).collect()),
)]),
)?;
}
let cleared: Vec<(String, Vec<HoldReason>)> = reported
.iter()
.filter(|(node, _)| !holds.contains_key(*node))
.map(|(node, reasons)| (node.clone(), reasons.clone()))
.collect();
for (node, released) in cleared {
journal.emit(
journal::PipelineKind::NodeUnheld,
journal::labels(&paths.run, Some(&node)),
journal::payload(&[(
"released",
Value::Array(released.iter().map(HoldReason::payload).collect()),
)]),
)?;
}
*reported = holds.clone();
Ok(())
}
fn report_unreadable_records(paths: &RunPaths, state: &RunState) {
if state.strict && !journal::has_unreadable_lines(&paths.journal()) {
return;
}
eprintln!(
"onepipeline: run '{}' has a journal record this build cannot read; the graph \
it is driving may be missing a committed edit.",
paths.run
);
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct Decision {
reference: DecisionRef,
kind: String,
unblocks: Vec<String>,
}
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
enum DecisionRef {
Attestation(String),
Surface(u64),
}
impl DecisionRef {
fn as_wire(&self) -> String {
match self {
Self::Attestation(node) => node.clone(),
Self::Surface(id) => format!("surface:{id}"),
}
}
fn of_wire(reference: &str) -> Self {
reference
.strip_prefix("surface:")
.and_then(|id| id.parse().ok())
.map_or_else(|| Self::Attestation(reference.to_string()), Self::Surface)
}
}
fn decisions_now(
state: &RunState,
statuses: &BTreeMap<String, NodeStatus>,
channel: &ChannelState,
) -> BTreeMap<DecisionRef, Decision> {
let mut decisions = BTreeMap::new();
for (id, status) in statuses {
if *status != NodeStatus::Waiting {
continue;
}
decisions.insert(
DecisionRef::Attestation(id.clone()),
Decision {
reference: DecisionRef::Attestation(id.clone()),
kind: "attestation".to_string(),
unblocks: descendants(&state.graph, std::slice::from_ref(id)),
},
);
}
let queue = channel.queue();
for surface in queue.waiting.iter().chain(queue.pending.iter()) {
if !surface.blocking || surface.abandoned {
continue;
}
let reference = DecisionRef::Surface(surface.id);
let unblocks = surface
.workstream
.clone()
.map(|node| descendants(&state.graph, std::slice::from_ref(&node)))
.unwrap_or_default();
decisions.insert(
reference.clone(),
Decision {
reference,
kind: surface.kind.clone(),
unblocks,
},
);
}
decisions
}
fn descendants(graph: &Graph, roots: &[String]) -> Vec<String> {
let mut seen: BTreeSet<String> = BTreeSet::new();
let mut pending: Vec<String> = roots.to_vec();
while let Some(current) = pending.pop() {
for dependent in graph.dependents_of(¤t) {
if seen.insert(dependent.clone()) {
pending.push(dependent);
}
}
}
graph
.ids()
.filter(|id| seen.contains(*id))
.cloned()
.collect()
}
fn report_decisions(
paths: &RunPaths,
journal: &mut Journal,
decisions: &BTreeMap<DecisionRef, Decision>,
held: &mut BTreeMap<DecisionRef, Decision>,
) -> Result<()> {
for (reference, decision) in decisions {
if held.get(reference) == Some(decision) {
continue;
}
journal.emit(
journal::PipelineKind::DecisionPending,
journal::labels(&paths.run, Some(&decision.reference.as_wire())),
journal::payload(&[
("reference", json!(decision.reference.as_wire())),
("kind", json!(decision.kind)),
("unblocks", json!(decision.unblocks)),
]),
)?;
}
let cleared: Vec<Decision> = held
.iter()
.filter(|(reference, _)| !decisions.contains_key(*reference))
.map(|(_, decision)| decision.clone())
.collect();
for decision in cleared {
journal.emit(
journal::PipelineKind::DecisionCleared,
journal::labels(&paths.run, Some(&decision.reference.as_wire())),
journal::payload(&[
("reference", json!(decision.reference.as_wire())),
("kind", json!(decision.kind)),
("released", json!(decision.unblocks)),
]),
)?;
}
*held = decisions.clone();
Ok(())
}
fn paused_by(decisions: &BTreeMap<DecisionRef, Decision>) -> BTreeSet<String> {
decisions
.values()
.flat_map(|decision| decision.unblocks.iter().cloned())
.collect()
}
fn announce_ready(
paths: &RunPaths,
journal: &mut Journal,
statuses: &BTreeMap<String, NodeStatus>,
announced: &mut BTreeSet<String>,
) -> Result<()> {
announced.retain(|id| statuses.get(id).copied() == Some(NodeStatus::Ready));
let fresh: Vec<String> = statuses
.iter()
.filter(|(_, status)| **status == NodeStatus::Ready)
.map(|(id, _)| id.clone())
.filter(|id| !announced.contains(id))
.collect();
for id in fresh {
journal.emit(
journal::PipelineKind::NodeReady,
journal::labels(&paths.run, Some(&id)),
journal::payload(&[]),
)?;
announced.insert(id);
}
Ok(())
}
fn addressed_by(envelope: &Envelope) -> Option<TurnAddress> {
if envelope.source != crate::event::Source::Agentgraph {
return None;
}
TurnAddress::of(
envelope.labels.run_id.as_deref()?,
envelope.labels.extra.get("member")?.as_str()?,
)
}
fn any_node_can_still_move(statuses: &BTreeMap<String, NodeStatus>) -> bool {
statuses.values().any(|status| {
matches!(
status,
NodeStatus::Ready | NodeStatus::Running | NodeStatus::CompleteDraft
)
})
}
fn reconcile_edits(
paths: &RunPaths,
journal: &mut Journal,
state: &mut RunState,
channel: &ChannelState,
launch: &LaunchRecord,
in_flight: &mut BTreeMap<String, Dispatch>,
) -> Result<bool> {
let mut changed = false;
for envelope in channel.claim_commands()? {
let author = envelope.author;
let mut applied = true;
let mut reason = None;
for command in &envelope.commands {
let compiled = crate::channel::allows(author, command).and_then(|()| {
compile_and_deliver(paths, state, author, command, launch, in_flight)
});
match compiled {
Ok(operations) => {
for target in cancelled_by(command) {
if let Some(dispatch) = in_flight.get(&target) {
dispatch.cancel.cancel();
}
}
journal.emit(
journal::PipelineKind::EditCommitted,
journal::labels(&paths.run, None),
journal::payload(&[
("author", json!(author)),
("command", json!(command)),
("operations", json!(operations)),
]),
)?;
record_operation_facts(paths, journal, author, &operations)?;
if author == crate::channel::Author::Monitor {
if let Some(surface) = monitor_edit(command) {
raise(paths, journal, surface)?;
}
}
*state = projection::fold(&journal::read(&paths.journal()));
changed = true;
}
Err(error) => {
applied = false;
reason = Some(error.to_string());
record_rejection(paths, journal, author, command, &error)?;
break;
}
}
}
channel.answer_commands(&CommandOutcome {
id: envelope.id,
applied,
reason,
})?;
}
Ok(changed)
}
fn compile_and_deliver(
paths: &RunPaths,
state: &RunState,
author: crate::channel::Author,
command: &Command,
launch: &LaunchRecord,
in_flight: &BTreeMap<String, Dispatch>,
) -> Result<Vec<edits::Operation>> {
let frontier = Frontier {
in_flight: in_flight
.iter()
.map(|(id, dispatch)| (id.clone(), dispatch.live()))
.collect(),
node_validator: launch.node_validator().map(str::to_owned),
..state.frontier()
};
let mut candidate = state.graph.clone();
let operations = edits::compile(&mut candidate, &frontier, author, command)?;
let Command::Note {
id,
addressee,
text,
criterion,
deliver,
persist,
} = command
else {
return Ok(operations);
};
deliver_manager_note(
paths,
&Offered {
id,
addressee: *addressee,
text,
criterion: criterion.as_ref(),
reach: crate::note::Reach::of(id, *deliver, *persist)?,
dispatchable: frontier.recorded.get(id) != Some(&NodeStatus::Done),
},
in_flight
.get(id)
.and_then(|dispatch| dispatch.control.clone())
.as_ref(),
)
}
pub(crate) fn record_rejection(
paths: &RunPaths,
journal: &mut Journal,
author: crate::channel::Author,
command: &Command,
error: &Error,
) -> Result<()> {
journal.emit(
journal::PipelineKind::EditRejected,
journal::labels(&paths.run, None),
journal::payload(&[
("author", json!(author)),
("command", json!(command)),
("reason", json!(error.to_string())),
]),
)?;
raise(
paths,
journal,
Surface {
id: 0,
kind: "edit-rejected".into(),
message: format!("reconciler: rejected — {error}"),
source: crate::channel::source::RECONCILER.into(),
blocking: false,
queued_at: sys::now_millis(),
abandoned: false,
asker: None,
workstream: None,
},
)
}
pub(crate) struct Offered<'a> {
pub id: &'a str,
pub addressee: crate::note::Addressee,
pub text: &'a crate::note::NoteText,
pub criterion: Option<&'a crate::note::Criterion>,
pub reach: crate::note::Reach,
pub dispatchable: bool,
}
pub(crate) fn deliver_manager_note(
paths: &RunPaths,
offered: &Offered<'_>,
live: Option<&TurnAddress>,
) -> Result<Vec<edits::Operation>> {
let Offered {
id,
addressee,
text,
criterion,
reach,
dispatchable,
} = *offered;
let note = crate::note::of(addressee, text, criterion)
.map_err(|refused| Error::Refused(format!("note: node '{id}': {refused}")))?;
let recorded = |reached| {
Ok(vec![edits::Operation::NoteDelivered {
node: id.to_string(),
addressee,
text: text.clone(),
criterion: criterion.cloned(),
reached,
}])
};
if !reach.attempts_a_live_turn() {
return match dispatchable {
true => recorded(crate::note::Reached::Carried),
false => Err(nowhere_to_carry(id)),
};
}
let attempted = match live.cloned().or_else(|| last_turn_address(paths, id)) {
Some(address) => agentgraph::note(&address, ¬e).map_err(|why| why.to_string()),
None => Err(
"no dispatch of this node has reported a member yet, so there is no \
conversation to hand it to"
.to_string(),
),
};
match attempted {
Ok(accepted) => recorded(crate::note::Reached::from(&accepted)),
Err(_) if reach.composes_forward() && dispatchable => {
recorded(crate::note::Reached::Carried)
}
Err(why) if reach.composes_forward() => Err(crate::note::reaches_nobody(
id,
&format!(
"{why}; and it has settled done, so no dispatch of it will take the note \
either"
),
)),
Err(why) => Err(crate::note::reaches_nobody(
id,
&format!("{why}; and `persist: false` composes it into no dispatch"),
)),
}
}
fn nowhere_to_carry(id: &str) -> Error {
crate::note::reaches_nobody(
id,
"it has settled done, so no dispatch of it will ever take the note and \
`deliver: next` asks for no live delivery",
)
}
fn last_turn_address(paths: &RunPaths, node: &str) -> Option<TurnAddress> {
journal::read(&paths.journal())
.into_iter()
.rev()
.filter(|envelope| envelope.labels.node.as_deref() == Some(node))
.find_map(|envelope| addressed_by(&envelope))
}
fn deliver_note(
journal: &mut Journal,
id: &str,
note: &str,
in_flight: &BTreeMap<String, Dispatch>,
) -> Result<edits::Delivery> {
let Some(address) = in_flight
.get(id)
.and_then(|dispatch| dispatch.control.clone())
else {
return Ok(edits::Delivery::Deferred);
};
let interrupt = agentgraph::interrupt(&address, note);
for event in interrupt.events {
let mut event = event;
if event.labels.node.is_none() {
event.labels.node = Some(id.to_string());
}
journal.relay(&event)?;
}
match interrupt.outcome {
Interrupted::Delivered => Ok(edits::Delivery::Live),
Interrupted::NoTurn(_) => Ok(edits::Delivery::Deferred),
Interrupted::Failed(reason) => Err(Error::Refused(format!(
"delivering the arrival note to node '{id}' failed: {reason}"
))),
}
}
fn adopt_releases(
paths: &RunPaths,
journal: &mut Journal,
state: &mut RunState,
statuses: &BTreeMap<String, NodeStatus>,
releases: &mut crate::release::Watch,
in_flight: &BTreeMap<String, Dispatch>,
) -> Result<bool> {
let told: Vec<Node> = in_flight
.values()
.map(|dispatch| dispatch.node.clone())
.chain(
state
.graph
.iter()
.filter(|node| statuses.get(&node.id) == Some(&NodeStatus::CompleteDraft))
.cloned(),
)
.collect();
let ready = releases.ready_to_adopt(&told);
if ready.is_empty() {
return Ok(false);
}
for (node, released) in ready {
let note = crate::release::arrival_note(&released);
let delivery = match deliver_note(journal, &node, ¬e, in_flight) {
Ok(delivery) => delivery,
Err(error) => {
eprintln!(
"onepipeline: the release note for node '{node}' was not delivered: {error}"
);
continue;
}
};
releases.adopted(&node);
journal.emit(
journal::PipelineKind::ReleaseAdopted,
journal::labels(&paths.run, Some(&node)),
journal::payload(&[
("node", json!(node)),
(
"delivery",
json!(match delivery {
edits::Delivery::Live => "live",
edits::Delivery::Deferred => "next",
}),
),
(
"versions",
json!(released
.iter()
.map(crate::release::Released::payload)
.collect::<Vec<_>>()),
),
]),
)?;
}
*state = projection::fold(&journal::read(&paths.journal()));
Ok(true)
}
fn cancelled_by(command: &Command) -> Vec<String> {
match command {
Command::Drop { id, .. } | Command::Retry { id, .. } | Command::Cancel { id, .. } => {
vec![id.clone()]
}
_ => Vec::new(),
}
}
#[allow(
clippy::too_many_arguments,
reason = "the reconcile loop's borrowed state, which cannot be bundled without \
taking one mutable borrow where three independent ones are needed"
)]
fn start_ready(
paths: &RunPaths,
journal: &mut Journal,
state: &mut RunState,
statuses: &BTreeMap<String, NodeStatus>,
rules: &ExecutorRules,
launch: &LaunchRecord,
tx: &Sender<Message>,
in_flight: &mut BTreeMap<String, Dispatch>,
paused: &BTreeSet<String>,
releases: &crate::release::Watch,
) -> Result<bool> {
let concurrency = state.graph.concurrency as usize;
let actionable: Vec<Node> = state
.graph
.iter()
.filter(|node| match statuses.get(&node.id) {
Some(NodeStatus::Ready) => true,
Some(NodeStatus::Waiting) => !state.recorded.contains_key(&node.id),
_ => false,
})
.filter(|node| !in_flight.contains_key(&node.id))
.filter(|node| !paused.contains(&node.id))
.cloned()
.collect();
let mut settled_here = false;
for node in actionable {
if node.kind != NodeKind::Human && in_flight.len() >= concurrency {
break;
}
if node.expects_no_diff {
settle(
paths,
journal,
&Settlement::plain(&node.id, NodeStatus::Done, Some(NO_CHANGES)),
)?;
settled_here = true;
continue;
}
if node.kind == NodeKind::Human {
settle(
paths,
journal,
&Settlement::plain(&node.id, NodeStatus::Waiting, None),
)?;
settled_here = true;
continue;
}
let cancel = CancellationToken::new();
journal.emit(
journal::PipelineKind::NodeDispatched,
journal::labels(&paths.run, Some(&node.id)),
journal::payload(&[("persona", json!(node.persona)), ("attempt", json!(1))]),
)?;
let references = releases.references(&node);
spawn(
paths,
rules,
launch,
&node,
&references,
cancel.clone(),
tx.clone(),
)?;
let now = Instant::now();
in_flight.insert(
node.id.clone(),
Dispatch {
node,
cancel,
started: now,
last_progress: now,
reported_quiet: false,
control: None,
},
);
settled_here = true;
}
if settled_here {
*state = projection::fold(&journal::read(&paths.journal()));
}
Ok(settled_here)
}
#[allow(
clippy::too_many_arguments,
reason = "one dispatch's whole context: the run, the rules, the launch, the node, \
its cross-repository references, its cancellation, and where to report"
)]
fn spawn(
paths: &RunPaths,
rules: &ExecutorRules,
launch: &LaunchRecord,
node: &Node,
references: &[crate::plan::CrossRepoReference],
cancel: CancellationToken,
tx: Sender<Message>,
) -> Result<()> {
let labels = dispatch_labels(&paths.run, &node.id, None, node.persona.as_deref());
let executor_name = rules.select(node.executor.as_deref(), &labels, &|name| {
rules
.executors
.iter()
.find(|entry| entry.name == name)
.map(|entry| crate::rules::executor_for(entry).capacity())
.unwrap_or_default()
})?;
let entry = rules
.executors
.iter()
.find(|entry| entry.name == executor_name)
.ok_or_else(|| Error::Invalid(format!("executor '{executor_name}' is not declared")))?
.clone();
let run = paths.run.clone();
let node = node.clone();
let references = references.to_vec();
let paths = paths.clone();
let launched = crate::lifecycle::Launch {
node_graph: launch.node_graph.clone(),
pr_author_graph: launch.pr_author_graph().map(str::to_owned),
vcs_filter: launch.filters.vcs.clone(),
};
std::thread::Builder::new()
.name(format!("dispatch-{}", node.id))
.spawn(move || {
let executor = crate::rules::executor_for(&entry);
let settlement = if node.repo.is_some() {
crate::lifecycle::execute(
executor.as_ref(),
&paths,
&launched,
&node,
&references,
&cancel,
&tx,
)
} else {
execute_direct(
executor.as_ref(),
&run,
&launched.node_graph,
&node,
&references,
&cancel,
&tx,
)
};
let _ = tx.send(Message::Settled(Box::new(settlement)));
})
.map_err(|e| Error::Invalid(format!("cannot start a dispatch thread: {e}")))?;
Ok(())
}
fn execute_direct(
executor: &dyn Executor,
run: &str,
default_graph: &str,
node: &Node,
references: &[crate::plan::CrossRepoReference],
cancel: &CancellationToken,
tx: &Sender<Message>,
) -> Settlement {
let graph = node_graph(node.agent_graph.as_ref(), default_graph);
let controls = match crate::controls::NodeControls::of_node(node) {
Ok(controls) => controls,
Err(why) => {
return Settlement {
detail: Some(why),
..Settlement::plain(&node.id, NodeStatus::Failed, Some(INVALID_NODE))
}
}
}; let request = || DispatchRequest {
graph: graph.clone(),
task: node.rendered_task_with(references),
labels: dispatch_labels(run, &node.id, None, node.persona.as_deref()),
controls,
workspace: WorkspaceSpec::Path(project_dir()),
cancel: cancel.clone(),
};
attempt(executor, &node.id, cancel, tx, &request).settlement
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum Reached {
NotStarted,
Silence,
Speech,
}
pub(crate) struct Drained {
pub settlement: Settlement,
pub reached: Reached,
pub session: Option<onevcs::SessionToken>,
pub branch: Option<String>,
}
pub(crate) fn attempt(
executor: &dyn Executor,
node: &str,
cancel: &CancellationToken,
tx: &Sender<Message>,
request: &dyn Fn() -> DispatchRequest,
) -> Drained {
let attempts = boundary_attempts();
let mut backoff = Duration::from_secs(boundary_backoff_seconds());
let mut last = Drained {
settlement: failed(node, INFRASTRUCTURE_FAILURE),
reached: Reached::NotStarted,
session: None,
branch: None,
};
for attempt in 1..=attempts.get() {
let drained = match executor.dispatch(request()) {
Ok(mut handle) => drain(handle.as_mut(), tx, node, cancel),
Err(error) => Drained {
settlement: Settlement {
detail: Some(error.to_string()),
..failed(node, INFRASTRUCTURE_FAILURE)
},
reached: Reached::NotStarted,
session: None,
branch: None,
},
};
if drained.settlement.status != NodeStatus::Failed
|| drained.reached == Reached::Speech
|| cancel.is_cancelled()
{
return drained;
}
last = drained;
if attempt == attempts.get() {
if last.reached != Reached::NotStarted {
last.settlement = Settlement {
detail: last.settlement.detail.clone(),
..failed(node, NO_AGENT_PROGRESS)
};
}
break;
}
std::thread::sleep(backoff);
backoff = (backoff * 2).min(BOUNDARY_BACKOFF_CEILING);
let _ = tx.send(Message::Redispatched(Box::new(Redispatch {
node: node.to_string(),
attempt: NonZeroU32::MIN.saturating_add(attempt),
attempts,
reason: last.settlement.detail.clone().unwrap_or_default(),
})));
}
last
}
pub(crate) fn drain(
handle: &mut dyn DispatchHandle,
tx: &Sender<Message>,
node: &str,
cancel: &CancellationToken,
) -> Drained {
let grace = Duration::from_secs(cancel_grace_seconds());
let (relayed, arriving) = mpsc::channel();
let events = handle.events();
let _ = std::thread::Builder::new()
.name(format!("relay-{node}"))
.spawn(move || {
for envelope in events {
if relayed.send(envelope).is_err() {
return;
}
}
});
let mut spoke = false;
let mut addresses: Vec<TurnAddress> = Vec::new();
let mut death = Death::Unstated;
let mut turns = TurnRecords::default();
let mut asked_at: Option<Instant> = None;
let mut killed = false;
loop {
match arriving.recv_timeout(TEARDOWN_TICK) {
Ok(Ok(envelope)) => {
spoke = true;
if let Some(address) = addressed_by(&envelope) {
if !addresses.contains(&address) {
addresses.push(address);
}
}
turns.read(&envelope);
if matches!(death, Death::Unstated) {
if let Some(published) = MemberDeath::of(&envelope) {
death = if published.from_provider
&& member_of(&envelope)
.is_some_and(|member| turns.contradicts_a_death_of(member))
{
Death::Contradicted
} else {
Death::Published(published)
};
}
}
let _ = tx.send(Message::Event(Box::new(envelope)));
}
Ok(Err(_)) => {}
Err(mpsc::RecvTimeoutError::Timeout) => {}
Err(mpsc::RecvTimeoutError::Disconnected) => break,
}
match asked_at {
None if cancel.is_cancelled() => {
handle.cancel(CancelMode::Cooperative);
let said = interrupt_turns(tx, node, &addresses, grace);
report(tx, node, CancelPhase::Interrupted, said);
asked_at = Some(Instant::now());
}
Some(asked) if !killed && asked.elapsed() >= grace => {
killed = true;
handle.cancel(CancelMode::Kill);
report(
tx,
node,
CancelPhase::Killed,
format!(
"the dispatch had not exited {}s after it was asked to stop, so it \
was killed and its process tree reaped; anything its turn had not \
committed is gone",
grace.as_secs()
),
);
}
_ => {}
}
}
let waited = handle.wait();
let (session, branch) = match &waited {
Ok(outcome) => (
outcome.session.clone().map(onevcs::SessionToken),
outcome.branch.clone(),
),
Err(_) => (None, None),
};
let settlement = match waited {
Ok(outcome) if outcome.succeeded && !cancel.is_cancelled() => {
Settlement::plain(node, NodeStatus::Done, None)
}
Ok(_) if cancel.is_cancelled() => Settlement {
detail: asked_at.map(|_| stopped_how(killed, grace)),
..Settlement::plain(node, NodeStatus::Cancelled, None)
},
Ok(outcome) => failed_task(node, &outcome, session.as_ref(), &death),
Err(error) => Settlement {
detail: Some(error.to_string()),
..failed(node, INFRASTRUCTURE_FAILURE)
},
};
Drained {
settlement,
reached: if spoke {
Reached::Speech
} else {
Reached::Silence
},
session,
branch,
}
}
fn interrupt_turns(
tx: &Sender<Message>,
node: &str,
addresses: &[TurnAddress],
grace: Duration,
) -> String {
if addresses.is_empty() {
return format!(
"nothing of this dispatch has named a turn to interrupt, so there was nothing to \
ask; it is killed in {}s if it has not exited by then",
grace.as_secs()
);
}
let mut answers = Vec::new();
for address in addresses {
let interrupt = agentgraph::interrupt(address, CANCEL_INPUT);
for mut event in interrupt.events {
if event.labels.node.is_none() {
event.labels.node = Some(node.to_string());
}
let _ = tx.send(Message::Event(Box::new(event)));
}
answers.push(format!(
"{}: {}",
address.member(),
answered(&interrupt.outcome)
));
}
format!(
"asked {} turn(s) to stop, commit, and end without starting new work — {}; the \
dispatch is killed in {}s if it has not exited by then",
addresses.len(),
answers.join("; "),
grace.as_secs()
)
}
fn answered(outcome: &Interrupted) -> String {
match outcome {
Interrupted::Delivered => "the running turn took the redirection".to_string(),
Interrupted::NoTurn(reason) => format!("no turn to redirect ({reason})"),
Interrupted::Failed(reason) => format!("the lever failed ({reason})"),
}
}
fn stopped_how(killed: bool, grace: Duration) -> String {
if killed {
format!(
"the dispatch was asked to stop and had not exited {}s later, so it was killed",
grace.as_secs()
)
} else {
"the dispatch stopped after its turn was asked to commit and end".to_string()
}
}
fn report(tx: &Sender<Message>, node: &str, phase: CancelPhase, detail: String) {
let _ = tx.send(Message::Cancelling(Box::new(Cancelling {
node: node.to_string(),
phase,
detail,
})));
}
fn cancelling_surface(step: &Cancelling) -> Surface {
Surface {
id: 0,
kind: step.phase.kind().into(),
message: format!("{}: {}", step.phase.kind(), bounded(&step.detail)),
source: crate::channel::source::RECONCILER.into(),
blocking: false,
queued_at: sys::now_millis(),
abandoned: false,
asker: None,
workstream: Some(step.node.clone()),
}
}
pub const DISPATCH_DIED: &str = "dispatch-died";
pub const PROVIDER_FAILED: &str = "provider-failed";
pub const INVALID_NODE: &str = "invalid-node";
pub const NO_CHANGES: &str = "no-changes";
pub const INFRASTRUCTURE_FAILURE: &str = "infrastructure-failure";
pub const NO_AGENT_PROGRESS: &str = "no-agent-progress";
pub const TASK_FAILED: &str = "task-failed";
pub const TASK_FAILED_CHANGE_OPEN: &str = "task-failed-change-open";
const MACHINERY: [&str; 3] = ["harness", "provider", "spawn"];
const CLASSIFIED_IN: [(char, char); 2] = [('(', ')'), ('[', ']')];
struct MemberDeath {
cause: String,
from_provider: bool,
}
impl MemberDeath {
fn of(envelope: &Envelope) -> Option<Self> {
if envelope.source != crate::event::Source::Agentgraph
|| envelope.kind.0 != oneagentgraph::event::EventKind::MemberDied.as_str()
{
return None;
}
let cause = envelope.payload.get("cause")?.as_str()?;
is_a_classification(cause).then(|| Self {
cause: cause.to_owned(),
from_provider: envelope.payload.get("rule").and_then(Value::as_str)
== Some(oneagentgraph::member::Rule::ProviderFailure.as_str()),
})
}
}
#[derive(Debug, Default)]
struct TurnRecords {
last_started: BTreeMap<String, u64>,
with_billed_usage: BTreeSet<(String, u64)>,
}
impl TurnRecords {
fn read(&mut self, envelope: &Envelope) {
if envelope.source != crate::event::Source::Agentgraph {
return;
}
let Some(turn) = envelope.payload.get("turn").and_then(Value::as_u64) else {
return;
};
let Some(member) = member_of(envelope).map(str::to_owned) else {
return;
};
let kind = &envelope.kind.0;
if kind == oneagentgraph::event::EventKind::TurnStarted.as_str() {
self.last_started.insert(member, turn);
} else if kind == oneagentgraph::event::EventKind::TurnCompleted.as_str()
&& has_a_usage_figure(envelope.payload.get("usage"))
{
self.with_billed_usage.insert((member, turn));
}
}
fn contradicts_a_death_of(&self, member: &str) -> bool {
self.last_started
.get(member)
.is_some_and(|turn| self.with_billed_usage.contains(&(member.to_owned(), *turn)))
}
}
fn member_of(envelope: &Envelope) -> Option<&str> {
match envelope.labels.extra.get("member") {
None => Some(UNSTAMPED_MEMBER),
Some(Value::String(member)) => is_a_classification(member).then_some(member.as_str()),
Some(_) => None,
}
}
const UNSTAMPED_MEMBER: &str = "";
const USAGE_FIGURES: [&str; 3] = ["input_tokens", "output_tokens", "cost_usd"];
fn has_a_usage_figure(usage: Option<&Value>) -> bool {
let Some(usage) = usage else { return false };
USAGE_FIGURES
.into_iter()
.filter_map(|figure| usage.get(figure))
.any(|figure| figure.as_f64().is_some_and(|spent| spent > 0.0))
}
enum Death {
Published(MemberDeath),
Contradicted,
Unstated,
}
fn dispatch_death_cause(detail: &str) -> Option<String> {
let lowered = detail.to_ascii_lowercase();
if !MACHINERY.iter().any(|word| lowered.contains(word)) {
return None;
}
CLASSIFIED_IN
.into_iter()
.filter_map(|(open, close)| delimited(detail, open, close))
.max_by_key(|(at, _)| *at)
.map(|(_, word)| word)
}
const CLASSIFICATION_LIMIT: usize = 64;
fn delimited(detail: &str, open: char, close: char) -> Option<(usize, String)> {
let mut found = None;
let mut at = 0;
let mut rest = detail;
while let Some(start) = rest.find(open) {
let after = &rest[start + open.len_utf8()..];
let Some(end) = after.find(close) else { break };
let inside = &after[..end];
if is_a_classification(inside) {
found = Some((at + start, inside.to_owned()));
}
at += start + open.len_utf8() + end + close.len_utf8();
rest = &after[end + close.len_utf8()..];
}
found
}
pub(crate) fn is_a_classification(word: &str) -> bool {
!word.is_empty()
&& word.len() <= CLASSIFICATION_LIMIT
&& !word.chars().any(|c| c.is_whitespace() || c.is_control())
}
fn failed_task(
node: &str,
outcome: &DispatchOutcome,
session: Option<&onevcs::SessionToken>,
death: &Death,
) -> Settlement {
let detail = (!outcome.detail.is_empty()).then(|| outcome.detail.clone());
if let Some(url) = session.and_then(crate::vcs::change_opened_in) {
return Settlement {
detail,
change_url: Some(url),
..failed(node, TASK_FAILED_CHANGE_OPEN)
};
}
let head = || session.and_then(crate::vcs::branch_head_in);
let (cause, word) = match death {
Death::Contradicted => {
return Settlement {
detail,
head: head(),
..failed(node, TASK_FAILED)
}
}
Death::Published(published) => (
Some(published.cause.clone()),
if published.from_provider {
PROVIDER_FAILED
} else {
DISPATCH_DIED
},
),
Death::Unstated => (dispatch_death_cause(&outcome.detail), DISPATCH_DIED),
};
let Some(cause) = cause else {
return Settlement {
detail,
..failed(node, TASK_FAILED)
};
};
Settlement {
detail,
cause: Some(cause),
head: head(),
..failed(node, word)
}
}
fn cancel_grace_seconds() -> u64 {
std::env::var(CANCEL_GRACE_ENV)
.ok()
.and_then(|value| value.parse().ok())
.filter(|seconds| *seconds > 0)
.unwrap_or(DEFAULT_CANCEL_GRACE_SECONDS)
}
fn boundary_attempts() -> NonZeroU32 {
std::env::var(BOUNDARY_ATTEMPTS_ENV)
.ok()
.and_then(|value| value.parse().ok())
.or_else(|| NonZeroU32::new(DEFAULT_BOUNDARY_ATTEMPTS))
.unwrap_or(NonZeroU32::MIN)
}
pub(crate) fn publication_attempts() -> NonZeroU32 {
std::env::var(PUBLICATION_ATTEMPTS_ENV)
.ok()
.and_then(|value| value.parse().ok())
.unwrap_or(DEFAULT_PUBLICATION_ATTEMPTS)
}
pub(crate) fn merge_path_reads() -> NonZeroU32 {
std::env::var(MERGE_PATH_READS_ENV)
.ok()
.and_then(|value| value.parse().ok())
.unwrap_or(DEFAULT_MERGE_PATH_READS)
}
pub(crate) fn merge_path_backoff() -> Duration {
Duration::from_secs(
std::env::var(MERGE_PATH_BACKOFF_ENV)
.ok()
.and_then(|value| value.parse::<NonZeroU64>().ok())
.map_or(DEFAULT_MERGE_PATH_BACKOFF_SECONDS, NonZeroU64::get),
)
.min(BOUNDARY_BACKOFF_CEILING)
}
pub(crate) fn doubled(backoff: Duration) -> Duration {
(backoff * 2).min(BOUNDARY_BACKOFF_CEILING)
}
fn boundary_backoff_seconds() -> u64 {
std::env::var(BOUNDARY_BACKOFF_ENV)
.ok()
.and_then(|value| value.parse().ok())
.unwrap_or(DEFAULT_BOUNDARY_BACKOFF_SECONDS)
}
fn failed(node: &str, outcome: &str) -> Settlement {
Settlement::plain(node, NodeStatus::Failed, Some(outcome))
}
pub(crate) fn bounded(reason: &str) -> String {
reason
.chars()
.take(crate::event::MAX_PAYLOAD_TEXT_BYTES / 4)
.collect()
}
pub(crate) fn dispatch_labels(
run: &str,
node: &str,
step: Option<&str>,
persona: Option<&str>,
) -> Labels {
Labels {
run_id: Some(run.to_string()),
round: None,
node: Some(node.to_string()),
step: step.map(str::to_string),
persona: persona.map(str::to_string),
extra: serde_json::Map::new(),
}
}
pub(crate) fn node_graph(
override_ref: Option<&oneagentgraph::config::ConfigRef>,
default_graph: &str,
) -> oneagentgraph::config::ConfigRef {
override_ref
.cloned()
.unwrap_or_else(|| oneagentgraph::config::ConfigRef(default_graph.to_string()))
}
pub(crate) fn configured_node_graph() -> String {
std::env::var(NODE_GRAPH_ENV)
.ok()
.filter(|value| !value.is_empty())
.unwrap_or_else(|| DEFAULT_NODE_GRAPH.to_string())
}
pub(crate) fn configured_node_validator() -> Result<Option<String>> {
match std::env::var(NODE_VALIDATOR_ENV) {
Ok(value) => Ok(Some(value)),
Err(std::env::VarError::NotPresent) => Ok(None),
Err(std::env::VarError::NotUnicode(_)) => Err(Error::Invalid(format!(
"{NODE_VALIDATOR_ENV} holds something this build cannot read as text, so the \
command it names cannot be resolved — set it to the command, or unset it to \
declare that this launch has none"
))),
}
}
pub(crate) fn configured_envelope_reviewer() -> Result<Option<String>> {
match std::env::var(ENVELOPE_REVIEWER_ENV) {
Ok(value) => Ok(Some(value)),
Err(std::env::VarError::NotPresent) => Ok(None),
Err(std::env::VarError::NotUnicode(_)) => Err(Error::Invalid(format!(
"{ENVELOPE_REVIEWER_ENV} holds something this build cannot read as text, so the \
command it names cannot be resolved — set it to the command, or unset it to \
declare that this launch has none"
))),
}
}
fn project_dir() -> std::path::PathBuf {
std::env::var_os(PROJECT_DIR_ENV)
.map(std::path::PathBuf::from)
.filter(|path| !path.as_os_str().is_empty())
.unwrap_or_else(|| std::path::PathBuf::from("."))
}
fn stall_after_seconds() -> u64 {
std::env::var(STALL_AFTER_ENV)
.ok()
.and_then(|value| value.parse().ok())
.filter(|seconds| *seconds > 0)
.unwrap_or(DEFAULT_STALL_AFTER_SECONDS)
}
fn executor_rules() -> Result<ExecutorRules> {
match std::env::var_os(EXECUTOR_RULES_ENV) {
Some(path) if !path.is_empty() => ExecutorRules::load(std::path::Path::new(&path)),
_ => Ok(ExecutorRules::shipped_default()),
}
}
pub(crate) fn record_operation_facts(
paths: &RunPaths,
journal: &mut Journal,
author: crate::channel::Author,
operations: &[edits::Operation],
) -> Result<()> {
for operation in operations {
match operation {
edits::Operation::CompletionRequested { reason } => journal.emit(
journal::PipelineKind::CompletionRequested,
journal::labels(&paths.run, None),
journal::payload(&[("reason", json!(reason))]),
)?,
edits::Operation::HumanAttested { node } => journal.emit(
journal::PipelineKind::HumanAttested,
journal::labels(&paths.run, Some(node)),
journal::payload(&[("ref", json!(node))]),
)?,
edits::Operation::FindingRaised {
node,
message,
blocking,
} => raise(
paths,
journal,
finding_surface(author, node.clone(), message, *blocking),
)?,
edits::Operation::SettledFromEvidence {
node,
outcome,
evidence,
} => journal.emit(
journal::PipelineKind::NodeSettled,
journal::labels(&paths.run, Some(node)),
journal::settled_payload(
outcome.as_str(),
Some(journal::SETTLED_FROM_EVIDENCE),
Some(evidence),
),
)?,
_ => {}
}
}
Ok(())
}
fn settle(paths: &RunPaths, journal: &mut Journal, settlement: &Settlement) -> Result<()> {
let mut payload = journal::settled_payload(
settlement.status.as_str(),
settlement.outcome.as_deref(),
settlement.detail.as_deref(),
);
if let Some(branch) = &settlement.branch {
payload.insert("branch".into(), json!(branch));
}
if let Some(url) = &settlement.change_url {
payload.insert("change_url".into(), json!(url));
}
if let Some(cause) = &settlement.cause {
payload.insert(journal::SETTLED_CAUSE.into(), json!(cause));
}
if let Some(head) = &settlement.head {
payload.insert(journal::SETTLED_HEAD.into(), json!(head));
}
if let Some(landing) = settlement.landing {
payload.insert(journal::SETTLED_LANDING.into(), json!(landing.as_str()));
}
if !settlement.completed_steps.is_empty() {
payload.insert("completed_steps".into(), json!(settlement.completed_steps));
}
journal.emit(
journal::PipelineKind::NodeSettled,
journal::labels(&paths.run, Some(&settlement.node)),
payload,
)
}
pub(crate) fn monitor_edit(command: &Command) -> Option<Surface> {
if matches!(command, Command::Finding { .. }) {
return None;
}
Some(Surface {
id: 0,
kind: "monitor-edit".into(),
message: format!(
"monitor applied an edit: {}",
bounded(&serde_json::to_string(command).unwrap_or_default())
),
source: crate::channel::source::MONITOR.into(),
blocking: false,
queued_at: sys::now_millis(),
abandoned: false,
asker: None,
workstream: crate::channel::target_of(command),
})
}
pub(crate) fn finding_surface(
author: crate::channel::Author,
node: Option<String>,
message: &str,
blocking: bool,
) -> Surface {
Surface {
id: 0,
kind: crate::channel::SurfaceKind::Finding.as_str().into(),
message: message.to_string(),
source: match author {
crate::channel::Author::Monitor => crate::channel::source::MONITOR,
crate::channel::Author::Planner => crate::channel::source::PROPOSAL,
}
.into(),
blocking,
queued_at: sys::now_millis(),
abandoned: false,
asker: None,
workstream: node,
}
}
fn criterion_payload(checked: &CriterionChecked) -> serde_json::Map<String, serde_json::Value> {
let mut payload = journal::payload(&[
("criterion", json!(bounded(checked.check.criterion()))),
("file", json!(bounded(checked.check.file()))),
("expected", json!(bounded(checked.check.literal()))),
("answer", json!(checked.answer.as_str())),
]);
match &checked.answer {
crate::criteria::Answer::Match => {}
crate::criteria::Answer::Mismatch { holds } => {
payload.insert("holds".into(), json!(bounded(holds)));
}
crate::criteria::Answer::Unread { reason } => {
payload.insert("reason".into(), json!(bounded(reason)));
}
}
payload
}
fn criterion_finding(checked: &CriterionChecked, holds: &str) -> Surface {
let holds = bounded(holds);
Surface {
id: 0,
kind: crate::channel::SurfaceKind::Finding.as_str().into(),
message: format!(
"node '{node}' settled against a criterion its branch contradicts.\n\
criterion: {criterion}\n\
file: {file}\n\
expected: {expected}\n\
the file holds: {holds}\n\
The node settled on its own work as it always would have and nothing was \
failed on this: it is a reading of the branch, for you to rule on.",
node = checked.node.as_str(),
criterion = bounded(checked.check.criterion()),
file = bounded(checked.check.file()),
expected = bounded(checked.check.literal()),
),
source: crate::channel::source::PROPOSAL.into(),
blocking: false,
queued_at: sys::now_millis(),
abandoned: false,
asker: None,
workstream: Some(checked.node.as_str().to_owned()),
}
}
fn report_unprojected(
paths: &RunPaths,
journal: &mut Journal,
writeback: &crate::writeback::Writeback,
) -> Result<()> {
for failure in writeback.take_unprojected() {
raise(paths, journal, unprojected_surface(&failure))?;
}
Ok(())
}
fn one_line(said: &str) -> String {
said.split_whitespace().collect::<Vec<_>>().join(" ")
}
fn unprojected_surface(failure: &crate::writeback::Unprojected) -> Surface {
let items = failure.items.join(", ");
Surface {
id: 0,
kind: crate::channel::SurfaceKind::Finding.as_str().into(),
message: format!(
"the onetaskgraph project '{project}' did not take this run's projection.\n\
items: {items}\n\
reason: {reason}\n\
The run itself is unaffected — nothing was settled, scheduled or failed on \
this — but the project is behind what the run recorded until it is fixed.",
project = bounded(failure.project.as_str()),
items = bounded(&items),
reason = bounded(&one_line(&failure.reason)),
),
source: crate::channel::source::PROPOSAL.into(),
blocking: false,
queued_at: sys::now_millis(),
abandoned: false,
asker: None,
workstream: None,
}
}
pub(crate) fn raise(paths: &RunPaths, journal: &mut Journal, surface: Surface) -> Result<()> {
let queued = ChannelState::new(paths).push(surface)?;
journal.emit(
journal::PipelineKind::PlannerSurfaceQueued,
journal::labels(&paths.run, queued.workstream.as_deref()),
journal::payload(&[
("kind", json!(queued.kind)),
("message", json!(queued.message)),
("source", json!(queued.source)),
("blocking", json!(queued.blocking)),
]),
)
}
fn watch_for_quiet(
paths: &RunPaths,
journal: &mut Journal,
stall_after: Duration,
in_flight: &mut BTreeMap<String, Dispatch>,
) -> Result<()> {
let quiet: Vec<(String, u64, bool, String)> = in_flight
.iter()
.filter(|(_, dispatch)| !dispatch.reported_quiet)
.filter(|(_, dispatch)| dispatch.last_progress.elapsed() > stall_after)
.map(|(id, dispatch)| {
(
id.clone(),
dispatch.last_progress.elapsed().as_secs(),
dispatch.last_progress == dispatch.started,
dispatch.node.persona.clone().unwrap_or_else(|| "-".into()),
)
})
.collect();
for (node, quiet_for, never_spoke, persona) in quiet {
if let Some(dispatch) = in_flight.get_mut(&node) {
dispatch.reported_quiet = true;
}
let last = if never_spoke {
"nothing recorded since it was dispatched".to_string()
} else {
format!("last activity {quiet_for}s ago")
};
journal.emit(
journal::PipelineKind::QuietWorker,
journal::labels(&paths.run, Some(&node)),
journal::payload(&[
("quiet_for_seconds", json!(quiet_for)),
("threshold_seconds", json!(stall_after.as_secs())),
("persona", json!(persona)),
]),
)?;
raise(
paths,
journal,
Surface {
id: 0,
kind: "quiet-worker".into(),
message: format!(
"quiet-worker: no activity for {quiet_for}s (threshold {}s); {last}. \
The dispatch has not failed — decide whether to cancel it, retry it, \
or let it run.",
stall_after.as_secs()
),
source: crate::channel::source::PROPOSAL.into(),
blocking: false,
queued_at: sys::now_millis(),
abandoned: false,
asker: None,
workstream: Some(node.clone()),
},
)?;
}
Ok(())
}
fn record_result(paths: &RunPaths, state: &RunState, settled: GraphState) -> Result<RunResult> {
let statuses = state.statuses();
let landings = landings_after_asking_again(state);
let mut nodes: Vec<NodeResult> = state
.graph
.iter()
.map(|node| {
let status = statuses
.get(&node.id)
.copied()
.unwrap_or(NodeStatus::Pending);
NodeResult {
id: node.id.clone(),
status,
outcome: state.outcomes.get(&node.id).cloned(),
landing: landings.get(&node.id).copied(),
action: (status == NodeStatus::Waiting)
.then(|| node.task.clone())
.flatten(),
unblocks: if status == NodeStatus::Waiting {
graph::unblocks(&state.graph, &node.id)
} else {
Vec::new()
},
blocked_by: if status == NodeStatus::Blocked {
gating_humans(state, &statuses, &node.id)
} else {
Vec::new()
},
branch: state
.branches
.get(&node.id)
.cloned()
.or_else(|| node.branch.clone()),
change_url: state.change_urls.get(&node.id).cloned(),
cause: state.causes.get(&node.id).cloned(),
head: state.heads.get(&node.id).cloned(),
superseded_by: None,
}
})
.collect();
nodes.extend(superseded_results(state, &landings));
let result = RunResult {
run_id: paths.run.clone(),
state: settled,
nodes,
};
ledger::write_json(&paths.result(), &result)?;
Ok(result)
}
fn landings_after_asking_again(state: &RunState) -> BTreeMap<String, Landing> {
let mut landings = state.landings.clone();
let unlanded: Vec<String> = landings
.iter()
.filter(|(_, landing)| **landing == Landing::Unlanded)
.map(|(node, _)| node.clone())
.collect();
for node in unlanded {
let Some(branch) = state.branches.get(&node) else {
continue;
};
let repo = state.graph.get(&node).and_then(|node| node.repo.as_deref());
if crate::vcs::proved_landed(branch, repo) {
landings.insert(node, Landing::Landed);
}
}
landings
}
fn superseded_results(state: &RunState, landings: &BTreeMap<String, Landing>) -> Vec<NodeResult> {
state
.superseded
.iter()
.map(|(id, replacement)| NodeResult {
id: id.clone(),
status: NodeStatus::Cancelled,
outcome: state.outcomes.get(id).cloned(),
landing: landings.get(id).copied(),
action: None,
unblocks: Vec::new(),
blocked_by: Vec::new(),
branch: state.branches.get(id).cloned(),
change_url: state.change_urls.get(id).cloned(),
cause: state.causes.get(id).cloned(),
head: state.heads.get(id).cloned(),
superseded_by: Some(replacement.clone()),
})
.collect()
}
fn gating_humans(
state: &RunState,
statuses: &BTreeMap<String, NodeStatus>,
id: &str,
) -> Vec<String> {
let mut gates = BTreeSet::new();
let mut seen = BTreeSet::new();
let mut pending = vec![id.to_string()];
while let Some(current) = pending.pop() {
if !seen.insert(current.clone()) {
continue;
}
let Some(node) = state.graph.get(¤t) else {
continue;
};
for dep in &node.deps {
match statuses.get(dep) {
Some(NodeStatus::Waiting) => {
gates.insert(dep.clone());
}
Some(NodeStatus::Blocked) => pending.push(dep.clone()),
_ => {}
}
}
}
gates.into_iter().collect()
}
#[cfg(test)]
mod tests {
use super::*;
use crate::plan::{Plan, PLAN_SCHEMA_VERSION};
use crate::projection::Recorded;
#[test]
fn the_scratch_variable_and_the_provider_word_are_spelled_one_way_in_every_document() {
let read = |relative: &str| {
std::fs::read_to_string(std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join(relative))
.unwrap_or_else(|error| panic!("{relative} ships: {error}"))
};
let readme = read("README.md");
let divergences = read("docs/contract-divergences.md");
let contract = read("docs/contract.md");
for named in [
crate::executor::NODE_SCRATCH_DIR_ENV,
PROVIDER_FAILED,
DISPATCH_DIED,
] {
let quoted = format!("`{named}`");
assert!(
readme.contains("ed),
"the README does not name {quoted}"
);
assert!(
divergences.contains("ed),
"docs/contract-divergences.md does not name {quoted}"
);
}
for open in [crate::executor::NODE_SCRATCH_DIR_ENV, PROVIDER_FAILED] {
assert!(
!contract.contains(open),
"docs/contract.md now names {open:?}, so the README must stop calling it an \
open divergence and the entry must be marked ruled on"
);
}
assert!(
readme.contains("open divergence 48") && readme.contains("open divergence 49"),
"the README does not say which entries these two are waiting on"
);
for entry in ["## 48. ", "## 49. "] {
assert!(
divergences.contains(entry),
"docs/contract-divergences.md has no entry {entry:?}"
);
}
}
#[test]
fn the_publication_budget_is_the_one_the_contract_and_the_readme_publish() {
let contract = std::fs::read_to_string(
std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("docs/contract.md"),
)
.expect("the contract ships");
assert!(
contract.contains(&format!("`{PUBLICATION_ATTEMPTS_ENV}`")),
"docs/contract.md does not name the {PUBLICATION_ATTEMPTS_ENV} bound"
);
assert_eq!(DEFAULT_PUBLICATION_ATTEMPTS.get(), 3);
assert!(
contract.contains("and three by default"),
"docs/contract.md does not state the default this build ships"
);
assert_eq!(publication_attempts(), DEFAULT_PUBLICATION_ATTEMPTS);
let readme = std::fs::read_to_string(
std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("README.md"),
)
.expect("the README ships");
let prose = readme.split_whitespace().collect::<Vec<_>>().join(" ");
assert!(
prose.contains(&format!("`{PUBLICATION_ATTEMPTS_ENV}`, three by default")),
"the README does not state the {PUBLICATION_ATTEMPTS_ENV} bound and its default"
);
assert!(
prose.contains(&format!(
"settles `{}` under the last failure's word",
NodeStatus::Failed.as_str()
)),
"the README does not say what spending the budget settles the node as"
);
}
#[test]
fn the_wait_between_merge_path_reads_falls_back_and_is_held_to_the_ceiling() {
assert_eq!(
merge_path_backoff(),
Duration::from_secs(DEFAULT_MERGE_PATH_BACKOFF_SECONDS)
);
for unusable in ["", "not a number", "-1", "5.5", "0"] {
std::env::set_var(MERGE_PATH_BACKOFF_ENV, unusable);
assert_eq!(
merge_path_backoff(),
Duration::from_secs(DEFAULT_MERGE_PATH_BACKOFF_SECONDS),
"{unusable:?} was read as a wait rather than falling back"
);
}
std::env::set_var(MERGE_PATH_BACKOFF_ENV, "1000000");
assert_eq!(
merge_path_backoff(),
BOUNDARY_BACKOFF_CEILING,
"a value nobody meant holds a node open for as long as it says"
);
std::env::set_var(MERGE_PATH_BACKOFF_ENV, "1");
assert_eq!(merge_path_backoff(), Duration::from_secs(1));
std::env::remove_var(MERGE_PATH_BACKOFF_ENV);
}
#[test]
fn the_wait_between_merge_path_reads_grows_to_the_ceiling_and_stops_there() {
assert_eq!(doubled(Duration::from_secs(5)), Duration::from_secs(10));
assert_eq!(
doubled(BOUNDARY_BACKOFF_CEILING / 2),
BOUNDARY_BACKOFF_CEILING
);
assert_eq!(
doubled(BOUNDARY_BACKOFF_CEILING),
BOUNDARY_BACKOFF_CEILING,
"the wait grew past the ceiling every backoff in this crate shares"
);
}
#[test]
fn the_merge_path_read_budget_is_the_one_the_contract_and_the_readme_publish() {
let contract = std::fs::read_to_string(
std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("docs/contract.md"),
)
.expect("the contract ships");
assert!(
contract.contains(&format!("`{MERGE_PATH_READS_ENV}`")),
"docs/contract.md does not name the {MERGE_PATH_READS_ENV} bound"
);
assert_eq!(DEFAULT_MERGE_PATH_READS.get(), 3);
assert_eq!(merge_path_reads(), DEFAULT_MERGE_PATH_READS);
assert!(
contract.contains(&format!(
"`{MERGE_PATH_READS_ENV}`, three by default and the whole budget"
)),
"docs/contract.md does not state the default this build ships"
);
let readme = std::fs::read_to_string(
std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("README.md"),
)
.expect("the README ships");
let prose = readme.split_whitespace().collect::<Vec<_>>().join(" ");
assert!(
prose.contains(&format!("`{MERGE_PATH_READS_ENV}`, three by default")),
"the README does not state the {MERGE_PATH_READS_ENV} bound and its default"
);
assert!(
prose.contains("reads that never get one settle it `failed`"),
"the README does not say what spending the read budget settles the node as"
);
}
#[test]
fn the_words_this_crate_publishes_are_one_vocabulary() {
use crate::lifecycle::Undrafted;
use crate::vcs::outcome_of;
use onevcs::PublishOutcome;
let publications: std::collections::BTreeSet<&str> = [
PublishOutcome::Merged(onevcs::Sha("abc".into())),
PublishOutcome::ChangeOpen(url()),
PublishOutcome::ChangeDraft(url()),
PublishOutcome::Queued(url()),
PublishOutcome::NothingToPublish,
]
.iter()
.map(outcome_of)
.chain(EVERY_PUBLICATION_FAILURE.iter().map(|kind| {
outcome_of(&PublishOutcome::Failed {
kind: *kind,
reason: String::new(),
retained: None,
})
}))
.collect();
let draftings: std::collections::BTreeSet<&str> = [
Undrafted::Dispatch(String::new()),
Undrafted::SchemaRefused,
Undrafted::Bodyless,
]
.iter()
.map(Undrafted::ending)
.collect();
let mut seen: std::collections::BTreeMap<&str, usize> = std::collections::BTreeMap::new();
let settlements: std::collections::BTreeSet<&str> =
SETTLEMENT_OUTCOMES.iter().copied().collect();
assert_eq!(
settlements.len(),
SETTLEMENT_OUTCOMES.len(),
"one settlement outcome is spelled twice in the list itself"
);
for word in settlements
.iter()
.copied()
.chain(publications.iter().copied())
.chain(draftings.iter().copied())
{
*seen.entry(word).or_default() += 1;
}
assert_eq!(
seen.remove(NO_CHANGES),
Some(2),
"`{NO_CHANGES}` is documented as the one word two vocabularies share"
);
let collided: Vec<&str> = seen
.iter()
.filter(|(_, times)| **times > 1)
.map(|(word, _)| *word)
.collect();
assert!(
collided.is_empty(),
"these words mean two things to a reader who cannot tell which: {collided:?}"
);
assert!(SETTLEMENT_OUTCOMES.contains(&DISPATCH_DIED));
assert!(SETTLEMENT_OUTCOMES.contains(&PROVIDER_FAILED));
assert!(draftings.contains(&"dispatch-failed"));
}
const SETTLEMENT_OUTCOMES: [&str; 8] = [
INVALID_NODE,
NO_CHANGES,
INFRASTRUCTURE_FAILURE,
NO_AGENT_PROGRESS,
TASK_FAILED,
TASK_FAILED_CHANGE_OPEN,
DISPATCH_DIED,
PROVIDER_FAILED,
];
const EVERY_PUBLICATION_FAILURE: &[onevcs::FailureKind] = &[
onevcs::FailureKind::Gate,
onevcs::FailureKind::Invalid,
onevcs::FailureKind::SyncConflict,
onevcs::FailureKind::NotImplemented,
onevcs::FailureKind::ChecksFailed,
onevcs::FailureKind::ChecksUnsettled,
onevcs::FailureKind::PushRejected,
onevcs::FailureKind::PushedUnverified,
];
fn url() -> onevcs::Url {
"https://example.invalid/pull/7".parse().expect("a URL")
}
#[test]
fn a_dispatch_death_is_classified_out_of_the_detail_and_a_task_failure_is_not() {
for (detail, cause) in [
(
"oneagentgraph: member 'worker' failed: provider error (respond): harness failed (rate_limit)",
"rate_limit",
),
("harness failed (auth)", "auth"),
("provider error (quota)", "quota"),
(
"no candidate ran the turn: claude-code [auth], codex [spawn-error]",
"spawn-error",
),
(
"provider error (respond): no candidate ran the turn: codex [quota]",
"quota",
),
(
"the harness chain [claude-code, codex] ended: harness failed (overloaded)",
"overloaded",
),
] {
assert_eq!(
dispatch_death_cause(detail).as_deref(),
Some(cause),
"{detail:?} was not classified as the machinery stopping"
);
}
for verdict in [
"the node failed its gate",
"the judge refused the report form",
"the gate failed (clippy)",
"",
] {
assert_eq!(
dispatch_death_cause(verdict),
None,
"{verdict:?} was read as a dispatch that died rather than a task that failed"
);
}
}
#[test]
fn a_death_is_read_from_the_producers_own_event_and_not_from_anything_beside_it() {
let died = |source, kind: &str, cause| Envelope {
v: crate::event::ENVELOPE_VERSION,
ts: "2026-08-29T00:00:00.000Z".into(),
stream: "oneagentgraph-1".into(),
seq: 0,
source,
kind: crate::event::EventKind(kind.into()),
phase: None,
labels: Labels::default(),
payload: match cause {
Some(cause) => serde_json::json!({"rule": "provider-failure", "cause": cause})
.as_object()
.cloned()
.expect("a payload is an object"),
None => serde_json::Map::new(),
},
artifacts: Vec::new(),
};
use crate::event::Source;
let member_died = oneagentgraph::event::EventKind::MemberDied.as_str();
let read = MemberDeath::of(&died(
Source::Agentgraph,
member_died,
Some(serde_json::json!("quota")),
))
.expect("a death this build can take a classification from");
assert_eq!(read.cause, "quota");
assert!(read.from_provider);
let mut supervised = died(Source::Agentgraph, member_died, Some(json!("timeout")));
supervised.payload.insert(
"rule".into(),
json!(oneagentgraph::member::Rule::Heartbeat.as_str()),
);
assert_eq!(
MemberDeath::of(&supervised).map(|death| death.from_provider),
Some(false),
"a death under another liveness rule was read as the provider's"
);
for beside in [
died(Source::Pipeline, member_died, Some(json!("quota"))),
died(Source::Vcs, member_died, Some(json!("quota"))),
died(Source::Agentgraph, "member-settled", Some(json!("quota"))),
died(Source::Agentgraph, member_died, None),
died(Source::Agentgraph, member_died, Some(json!(""))),
died(Source::Agentgraph, member_died, Some(json!("rate limited"))),
died(Source::Agentgraph, member_died, Some(json!("quota\n"))),
died(Source::Agentgraph, member_died, Some(json!(3))),
died(
Source::Agentgraph,
member_died,
Some(json!("q".repeat(CLASSIFICATION_LIMIT + 1))),
),
] {
assert!(
MemberDeath::of(&beside).is_none(),
"{beside:?} was read as a member this producer said had died"
);
}
}
#[test]
fn the_usage_figures_this_crate_reads_are_the_ones_the_producer_writes() {
let written = serde_json::to_value(oneagentgraph::event::Usage {
input_tokens: Some(1),
output_tokens: Some(1),
cache_read_tokens: Some(1),
cache_write_tokens: Some(1),
cost_usd: Some(1.0),
})
.expect("the sibling's usage serializes");
for figure in USAGE_FIGURES {
assert!(
written.get(figure).is_some(),
"the producer no longer writes {figure:?}: {written}"
);
}
for cached in ["cache_read_tokens", "cache_write_tokens"] {
assert!(written.get(cached).is_some(), "{written}");
assert!(!USAGE_FIGURES.contains(&cached));
}
}
#[test]
fn a_turn_record_contradicts_only_the_death_of_the_member_and_turn_it_belongs_to() {
let of = |kind: &str, member: Option<&str>, payload: serde_json::Value| {
let mut envelope = Envelope {
v: crate::event::ENVELOPE_VERSION,
ts: "2026-08-29T00:00:00.000Z".into(),
stream: "oneagentgraph-1".into(),
seq: 0,
source: crate::event::Source::Agentgraph,
kind: crate::event::EventKind(kind.into()),
phase: None,
labels: Labels::default(),
payload: payload
.as_object()
.cloned()
.expect("a payload is an object"),
artifacts: Vec::new(),
};
if let Some(member) = member {
envelope.labels.extra.insert("member".into(), json!(member));
}
envelope
};
let started = oneagentgraph::event::EventKind::TurnStarted.as_str();
let completed = oneagentgraph::event::EventKind::TurnCompleted.as_str();
let billed = json!({"turn": 1, "usage": {"output_tokens": 12}});
let mut records = TurnRecords::default();
records.read(&of(started, Some("worker"), json!({"turn": 1})));
records.read(&of(completed, Some("worker"), billed.clone()));
assert!(records.contradicts_a_death_of("worker"));
records.read(&of(started, Some("worker"), json!({"turn": 2})));
assert!(
!records.contradicts_a_death_of("worker"),
"a record of an earlier turn was read as the record of the turn that died"
);
assert!(!records.contradicts_a_death_of("reviewer"));
let mut orphan = TurnRecords::default();
orphan.read(&of(completed, Some("worker"), billed.clone()));
assert!(!orphan.contradicts_a_death_of("worker"));
let mut unpaid = TurnRecords::default();
unpaid.read(&of(started, Some("worker"), json!({"turn": 1})));
for nothing in [
json!({"turn": 1}),
json!({"turn": 1, "usage": {}}),
json!({"turn": 1, "usage": {"cost_usd": 0.0}}),
json!({"turn": 1, "usage": {"output_tokens": "many"}}),
] {
unpaid.read(&of(completed, Some("worker"), nothing));
}
assert!(!unpaid.contradicts_a_death_of("worker"));
unpaid.read(&of(completed, Some("worker"), billed.clone()));
assert!(unpaid.contradicts_a_death_of("worker"));
let mut unstamped = TurnRecords::default();
unstamped.read(&of(started, None, json!({"turn": 1})));
unstamped.read(&of(completed, None, billed.clone()));
assert!(unstamped.contradicts_a_death_of(UNSTAMPED_MEMBER));
for label in [
json!(7),
json!(""),
json!("a name with spaces in it"),
json!("worker\n"),
json!("m".repeat(CLASSIFICATION_LIMIT + 1)),
] {
let mut unreadable = TurnRecords::default();
let mut opened = of(started, None, json!({"turn": 1}));
opened.labels.extra.insert("member".into(), label.clone());
unreadable.read(&opened);
let mut closed = of(completed, None, billed.clone());
closed.labels.extra.insert("member".into(), label);
unreadable.read(&closed);
assert!(!unreadable.contradicts_a_death_of(UNSTAMPED_MEMBER));
assert!(!unreadable.contradicts_a_death_of("worker"));
}
let mut beside = TurnRecords::default();
let mut relayed = of(started, Some("worker"), json!({"turn": 1}));
relayed.source = crate::event::Source::Vcs;
beside.read(&relayed);
beside.read(&of(started, Some("worker"), json!({})));
beside.read(&of("member-heartbeat", Some("worker"), json!({"turn": 1})));
beside.read(&of(completed, Some("worker"), billed));
assert!(
!beside.contradicts_a_death_of("worker"),
"a record from another producer opened a turn for this one"
);
}
#[test]
fn each_death_settles_under_the_word_its_own_rule_and_record_leave_it() {
let failed = DispatchOutcome {
succeeded: false,
detail: "oneagentgraph: member 'worker' failed: the turn exited 0 without a report"
.into(),
..DispatchOutcome::default()
};
let word = |death: &Death| {
failed_task("build", &failed, None, death)
.outcome
.expect("a failed dispatch settles under a word")
};
assert_eq!(
word(&Death::Published(MemberDeath {
cause: "quota".into(),
from_provider: true,
})),
PROVIDER_FAILED
);
assert_eq!(
word(&Death::Published(MemberDeath {
cause: "timeout".into(),
from_provider: false,
})),
DISPATCH_DIED
);
let contradicted = failed_task("build", &failed, None, &Death::Contradicted);
assert_eq!(contradicted.outcome.as_deref(), Some(TASK_FAILED));
assert_eq!(contradicted.cause, None);
assert_eq!(word(&Death::Unstated), TASK_FAILED);
}
const RUN_RESULT_GOLDEN: &str = include_str!("../tests/golden/run-result-v5.json");
use serde_json::Value;
fn settled(id: &str) -> NodeResult {
NodeResult {
id: id.into(),
status: NodeStatus::Done,
outcome: None,
landing: None,
action: None,
unblocks: Vec::new(),
blocked_by: Vec::new(),
branch: None,
change_url: None,
cause: None,
head: None,
superseded_by: None,
}
}
fn run_result_golden() -> RunResult {
RunResult {
run_id: "golden".into(),
state: GraphState::Complete,
nodes: vec![
NodeResult {
id: "merged".into(),
status: NodeStatus::Done,
outcome: Some("merged".into()),
landing: Some(Landing::Landed),
branch: Some("onepipeline/merged".into()),
..settled("merged")
},
NodeResult {
id: "opened".into(),
status: NodeStatus::Done,
outcome: Some("change-open".into()),
landing: Some(Landing::Unlanded),
branch: Some("onepipeline/opened".into()),
change_url: Some("https://example.invalid/pull/7".into()),
..settled("opened")
},
NodeResult {
id: "died".into(),
status: NodeStatus::Failed,
outcome: Some(DISPATCH_DIED.into()),
branch: Some("onepipeline/died".into()),
cause: Some("rate_limit".into()),
head: Some("0123456789abcdef0123456789abcdef01234567".into()),
..settled("died")
},
settled("built"),
NodeResult {
id: "replaced".into(),
status: NodeStatus::Cancelled,
outcome: Some(TASK_FAILED.into()),
branch: Some("onepipeline/replaced".into()),
superseded_by: Some("replaced-2".into()),
..settled("replaced")
},
],
}
}
#[test]
fn a_schema_5_run_result_is_the_shape_the_golden_pins() {
let rendered = serde_json::to_string_pretty(&run_result_golden()).expect("it serialises");
assert_eq!(
rendered.trim(),
RUN_RESULT_GOLDEN.trim(),
"the run result changed shape. If that was deliberate, bump \
RUN_RESULT_SCHEMA_VERSION and update tests/golden/run-result-v5.json together"
);
}
#[test]
fn a_schema_5_run_result_round_trips_and_omits_what_it_does_not_have() {
let value = run_result_golden();
let read: RunResult =
serde_json::from_str(RUN_RESULT_GOLDEN).expect("the golden reads back into the types");
assert_eq!(read, value);
let again: RunResult =
serde_json::from_str(&serde_json::to_string(&value).expect("it serialises"))
.expect("it reads back");
assert_eq!(again, value);
let document: Value =
serde_json::from_str(&serde_json::to_string(&value).expect("it serialises"))
.expect("it is JSON");
assert_eq!(document["nodes"][0]["landing"], json!("landed"));
assert_eq!(document["nodes"][1]["landing"], json!("unlanded"));
assert!(
document["nodes"][2].get("landing").is_none(),
"a node with no change to land carries a landing key anyway: {}",
document["nodes"][2]
);
assert_eq!(document["nodes"][4]["superseded_by"], json!("replaced-2"));
for at in 0..4 {
assert!(
document["nodes"][at].get("superseded_by").is_none(),
"a node nothing superseded carries a superseded_by key anyway: {}",
document["nodes"][at]
);
}
}
#[test]
fn the_run_result_schema_version_and_the_golden_name_the_same_number() {
assert_eq!(RUN_RESULT_SCHEMA_VERSION, 5);
let document: Value = serde_json::from_str(RUN_RESULT_GOLDEN).expect("the golden is JSON");
assert_eq!(document["schema_version"], RUN_RESULT_SCHEMA_VERSION);
assert!(
document.get("round").is_none(),
"the run's own result document names a round: {document}"
);
let written: Value = serde_json::from_str(
&serde_json::to_string(&run_result_golden()).expect("it serialises"),
)
.expect("it is JSON");
assert_eq!(written["schema_version"], RUN_RESULT_SCHEMA_VERSION);
}
#[test]
fn only_this_build_s_run_result_version_reads_and_every_other_is_refused_by_name() {
let document = serde_json::to_value(run_result_golden()).expect("it serialises");
let edit = |document: &Value, each: &dyn Fn(&mut serde_json::Map<String, Value>)| {
let mut copy = document.clone();
each(copy.as_object_mut().expect("it is an object"));
copy
};
for outside in [RUN_RESULT_SCHEMA_VERSION + 1, 4, 3, 2, 1, 0] {
let claimed = edit(&document, &|object| {
object.insert("schema_version".into(), json!(outside));
});
let refused = serde_json::from_value::<RunResult>(claimed)
.expect_err("a result this build never wrote was read as one it did");
let refusal = refused.to_string();
assert!(
refusal.contains(&outside.to_string())
&& refusal.contains(&RUN_RESULT_SCHEMA_VERSION.to_string()),
"the refusal of {outside} names neither version: {refusal}"
);
}
let unversioned = edit(&document, &|object| {
object.remove("schema_version");
});
let refusal = serde_json::from_value::<RunResult>(unversioned)
.expect_err("an unversioned result was read as this build's version")
.to_string();
assert!(
refusal.contains("schema_version"),
"the refusal of an unversioned result does not name the field: {refusal}"
);
}
fn agent(id: &str, deps: &[&str]) -> Node {
Node {
id: id.into(),
persona: Some("engineer".into()),
task: Some("## What\ndo it".into()),
deps: deps.iter().map(|d| (*d).to_string()).collect(),
..Node::default()
}
}
#[test]
fn a_node_whose_budget_no_dispatch_can_run_under_settles_rather_than_launching() {
let (tx, rx) = std::sync::mpsc::channel();
let node = Node {
max_turns: Some(0),
..agent("build", &[])
};
let settlement = execute_direct(
&crate::executor::LocalExecutor,
"demo",
"graphs/node-scope.yaml",
&node,
&[],
&CancellationToken::new(),
&tx,
);
assert_eq!(settlement.status, NodeStatus::Failed);
assert_eq!(settlement.outcome.as_deref(), Some("invalid-node"));
let detail = settlement.detail.expect("the settlement says why");
assert!(detail.contains("no turn at all"), "{detail}");
assert!(
rx.try_iter().count() == 0,
"a node that was never dispatched reported turns"
);
}
fn state_of(nodes: Vec<Node>, recorded: &[(&str, NodeStatus)]) -> RunState {
let plan = Plan {
schema_version: PLAN_SCHEMA_VERSION,
goal: None,
name: Some("demo".into()),
concurrency: 4,
tasks: nodes,
};
RunState {
graph: Graph::from_plan(&plan),
plan: Some(plan),
recorded: recorded
.iter()
.map(|(id, status)| ((*id).to_string(), Recorded::At(*status)))
.collect(),
..RunState::default()
}
}
#[test]
fn only_a_siblings_envelope_naming_a_run_and_a_member_addresses_a_turn() {
let envelope = |source: crate::event::Source, run: Option<&str>, member: Option<&str>| {
let mut labels = Labels {
run_id: run.map(str::to_string),
node: Some("build".into()),
..Labels::default()
};
if let Some(member) = member {
labels.extra.insert("member".into(), member.into());
}
Envelope {
v: crate::event::ENVELOPE_VERSION,
ts: "2026-08-12T00:00:00.000Z".into(),
stream: "oneagentgraph-1".into(),
seq: 0,
source,
kind: crate::event::EventKind("turn-started".into()),
phase: None,
labels,
payload: serde_json::Map::new(),
artifacts: Vec::new(),
}
};
use crate::event::Source;
assert_eq!(
addressed_by(&envelope(
Source::Agentgraph,
Some("node-scope-1786304152340-19"),
Some("worker")
)),
TurnAddress::of("node-scope-1786304152340-19", "worker")
);
for unaddressable in [
envelope(Source::Pipeline, Some("demo"), Some("worker")),
envelope(Source::Agentgraph, Some("graph-1"), None),
envelope(Source::Agentgraph, None, Some("worker")),
envelope(Source::Agentgraph, Some("graph-1"), Some(" ")),
envelope(Source::Agentgraph, Some(""), Some("worker")),
envelope(Source::Agentgraph, Some("graph-1"), Some("../elsewhere")),
] {
assert_eq!(
addressed_by(&unaddressable),
None,
"{unaddressable:?} was read as an address"
);
}
}
#[test]
fn a_blocked_node_names_the_ready_human_gating_it_transitively() {
let human = Node {
id: "approve".into(),
kind: NodeKind::Human,
task: Some("approve it".into()),
..Node::default()
};
let state = state_of(
vec![
human,
agent("ship", &["approve"]),
agent("after", &["ship"]),
],
&[("approve", NodeStatus::Waiting)],
);
let statuses = state.statuses();
assert_eq!(statuses["after"], NodeStatus::Blocked);
assert_eq!(
gating_humans(&state, &statuses, "after"),
vec!["approve".to_string()]
);
}
#[test]
fn a_waiting_human_holds_its_own_subtree_and_no_other_branch() {
let human = Node {
id: "approve".into(),
kind: NodeKind::Human,
task: Some("approve it".into()),
deps: vec!["seed".into()],
..Node::default()
};
let state = state_of(
vec![
agent("seed", &[]),
human,
agent("ship", &["approve"]),
agent("after", &["ship"]),
agent("probe", &[]),
agent("report", &["probe"]),
],
&[("seed", NodeStatus::Done), ("approve", NodeStatus::Waiting)],
);
let statuses = state.statuses();
let decisions = decisions_now(
&state,
&statuses,
&ChannelState::new(&RunPaths::under(
std::path::Path::new("/nonexistent"),
"demo",
)),
);
let held = decisions
.get(&DecisionRef::Attestation("approve".into()))
.expect("the human action holds");
assert_eq!(held.kind, "attestation");
assert_eq!(
held.unblocks,
vec!["ship".to_string(), "after".to_string()],
"the decision held more than its own subtree"
);
let paused = paused_by(&decisions);
assert!(
!paused.contains("probe"),
"an independent branch was paused"
);
assert!(
!paused.contains("report"),
"an independent branch was paused"
);
}
#[test]
fn a_decision_reference_spells_which_of_the_two_things_clears_it() {
assert_eq!(
DecisionRef::Attestation("approve".into()).as_wire(),
"approve"
);
assert_eq!(DecisionRef::Surface(7).as_wire(), "surface:7");
}
#[test]
fn a_decision_reference_reads_back_as_the_thing_that_wrote_it() {
for reference in [
DecisionRef::Attestation("approve".into()),
DecisionRef::Surface(7),
] {
assert_eq!(DecisionRef::of_wire(&reference.as_wire()), reference);
}
assert_eq!(
DecisionRef::of_wire("surface-check"),
DecisionRef::Attestation("surface-check".into())
);
}
#[test]
fn a_decision_is_reported_when_it_begins_holding_and_again_when_it_releases() {
let root = std::env::temp_dir().join(format!("onepipeline-decisions-{}", sys::pid()));
let _ = std::fs::remove_dir_all(&root);
let paths = RunPaths::under(&root, "demo");
paths.create().expect("the run directory");
let mut journal = Journal::open(&paths);
let decision = Decision {
reference: DecisionRef::Attestation("approve".into()),
kind: "attestation".into(),
unblocks: vec!["ship".into()],
};
let mut held = BTreeMap::new();
let pending: BTreeMap<DecisionRef, Decision> = [(
DecisionRef::Attestation("approve".to_string()),
decision.clone(),
)]
.into();
report_decisions(&paths, &mut journal, &pending, &mut held).expect("reported");
report_decisions(&paths, &mut journal, &pending, &mut held).expect("reported");
report_decisions(&paths, &mut journal, &BTreeMap::new(), &mut held).expect("reported");
let kinds: Vec<String> = journal::read(&paths.journal())
.iter()
.map(|event| event.kind.0.clone())
.collect();
assert_eq!(
kinds,
vec![
journal::PipelineKind::DecisionPending.as_str().to_string(),
journal::PipelineKind::DecisionCleared.as_str().to_string(),
]
);
let cleared = &journal::read(&paths.journal())[1];
assert_eq!(cleared.payload["released"], json!(["ship"]));
assert_eq!(cleared.labels.node.as_deref(), Some("approve"));
assert_eq!(cleared.labels.round, None, "a round was stamped");
let _ = std::fs::remove_dir_all(&root);
}
#[test]
fn a_node_is_announced_ready_once_and_again_when_it_becomes_ready_again() {
let root = std::env::temp_dir().join(format!("onepipeline-ready-{}", sys::pid()));
let _ = std::fs::remove_dir_all(&root);
let paths = RunPaths::under(&root, "demo");
paths.create().expect("the run directory");
let mut journal = Journal::open(&paths);
let mut announced = BTreeSet::new();
let ready: BTreeMap<String, NodeStatus> = [("build".to_string(), NodeStatus::Ready)].into();
let running: BTreeMap<String, NodeStatus> =
[("build".to_string(), NodeStatus::Running)].into();
announce_ready(&paths, &mut journal, &ready, &mut announced).expect("announced");
announce_ready(&paths, &mut journal, &ready, &mut announced).expect("announced");
announce_ready(&paths, &mut journal, &running, &mut announced).expect("announced");
announce_ready(&paths, &mut journal, &ready, &mut announced).expect("announced");
assert_eq!(
journal::read(&paths.journal()).len(),
2,
"a node was announced ready more than once per time it became ready"
);
let _ = std::fs::remove_dir_all(&root);
}
#[test]
fn the_hold_reasons_are_the_ones_the_divergence_record_names() {
let record = std::fs::read_to_string(
std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("docs/contract-divergences.md"),
)
.expect("the divergence record ships");
let entry = record
.split("\n## ")
.find(|entry| entry.starts_with("55."))
.expect("the record still carries entry 55");
let block: Value = entry
.split("```json")
.nth(1)
.and_then(|rest| rest.split("```").next())
.and_then(|block| serde_json::from_str(block).ok())
.expect("entry 55 carries the json block this test drives");
let reasons = [
HoldReason::Dependencies {
blocking: vec!["build".into()],
},
HoldReason::Concurrency {
ahead: vec!["build".into()],
limit: 1,
},
HoldReason::Decision {
reference: DecisionRef::Surface(7),
},
HoldReason::Release {
awaiting: vec!["build".into()],
},
];
let mine: Vec<String> = reasons
.iter()
.map(|reason| {
reason.payload()["kind"]
.as_str()
.expect("a kind")
.to_string()
})
.collect();
let named: Vec<String> = serde_json::from_value(block["reason_kinds"].clone())
.expect("entry 55 names its kinds");
assert_eq!(mine, named);
for reason in &reasons {
let payload = reason.payload();
let kind = payload["kind"].as_str().expect("a kind");
let fields: Vec<String> = serde_json::from_value(block["fields"][kind].clone())
.unwrap_or_else(|e| panic!("entry 55 names {kind}'s fields: {e}"));
let carried: Vec<String> = payload
.as_object()
.expect("an object")
.keys()
.filter(|key| *key != "kind")
.cloned()
.collect();
assert_eq!(carried, fields, "{kind}");
}
assert_eq!(block["held_payload"], json!("reasons"));
assert_eq!(block["unheld_payload"], json!("released"));
assert_eq!(
serde_json::from_value::<Vec<String>>(block["event_kinds"].clone())
.expect("entry 55 names its kinds"),
vec![
journal::PipelineKind::NodeHeld.as_str(),
journal::PipelineKind::NodeUnheld.as_str()
]
);
}
#[test]
fn a_hold_is_written_when_it_begins_when_it_changes_and_when_it_clears() {
let root = std::env::temp_dir().join(format!("onepipeline-holds-{}", sys::pid()));
let _ = std::fs::remove_dir_all(&root);
let paths = RunPaths::under(&root, "demo");
paths.create().expect("the run directory");
let mut journal = Journal::open(&paths);
let mut reported = BTreeMap::new();
let both: BTreeMap<String, Vec<HoldReason>> = [(
"ship".to_string(),
vec![
HoldReason::Dependencies {
blocking: vec!["build".into()],
},
HoldReason::Concurrency {
ahead: vec!["build".into()],
limit: 1,
},
],
)]
.into();
let one: BTreeMap<String, Vec<HoldReason>> = [(
"ship".to_string(),
vec![HoldReason::Concurrency {
ahead: vec!["build".into()],
limit: 1,
}],
)]
.into();
report_holds(&paths, &mut journal, &both, &mut reported).expect("reported");
report_holds(&paths, &mut journal, &both, &mut reported).expect("reported");
report_holds(&paths, &mut journal, &one, &mut reported).expect("reported");
report_holds(&paths, &mut journal, &one, &mut reported).expect("reported");
report_holds(&paths, &mut journal, &BTreeMap::new(), &mut reported).expect("reported");
report_holds(&paths, &mut journal, &BTreeMap::new(), &mut reported).expect("reported");
let written = journal::read(&paths.journal());
let kinds: Vec<String> = written.iter().map(|event| event.kind.0.clone()).collect();
assert_eq!(
kinds,
vec![
journal::PipelineKind::NodeHeld.as_str(),
journal::PipelineKind::NodeHeld.as_str(),
journal::PipelineKind::NodeUnheld.as_str(),
]
);
assert!(written
.iter()
.all(|event| event.labels.node.as_deref() == Some("ship")));
assert_eq!(
written[0].payload["reasons"],
json!([
{ "kind": "dependencies", "blocking": ["build"] },
{ "kind": "concurrency", "ahead": ["build"], "limit": 1 },
]),
"a node held two ways carries one entry per reason in one record"
);
assert_eq!(
written[1].payload["reasons"],
json!([{ "kind": "concurrency", "ahead": ["build"], "limit": 1 }]),
"ceasing to be held one way leaves only the reason that remains"
);
assert_eq!(
written[2].payload["released"],
json!([{ "kind": "concurrency", "ahead": ["build"], "limit": 1 }])
);
let _ = std::fs::remove_dir_all(&root);
}
#[test]
fn a_node_no_stated_reason_holds_carries_nothing_and_the_rest_name_theirs() {
let mut state = state_of(
vec![
agent("build", &[]),
agent("ship", &["build"]),
Node {
kind: NodeKind::Human,
..agent("approve", &[])
},
agent("after", &["approve"]),
agent("spare", &[]),
],
&[("approve", NodeStatus::Waiting)],
);
state.graph.concurrency = 1;
let statuses = state.statuses();
let in_flight: BTreeMap<String, Dispatch> = [(
"build".to_string(),
Dispatch {
node: agent("build", &[]),
cancel: CancellationToken::new(),
started: Instant::now(),
last_progress: Instant::now(),
reported_quiet: false,
control: None,
},
)]
.into();
let decisions: BTreeMap<DecisionRef, Decision> = [(
DecisionRef::Attestation("approve".into()),
Decision {
reference: DecisionRef::Attestation("approve".into()),
kind: "attestation".into(),
unblocks: vec!["after".into()],
},
)]
.into();
let awaiting: BTreeMap<String, Vec<String>> =
[("spare".to_string(), vec!["build".to_string()])].into();
let holds = holds_now(&state, &statuses, &in_flight, &decisions, &awaiting);
assert_eq!(holds.get("build"), None);
assert_eq!(holds.get("approve"), None);
assert_eq!(
holds.get("ship"),
Some(&vec![HoldReason::Dependencies {
blocking: vec!["build".into()]
}]),
"a node whose dependency is running is held by the dependency"
);
assert_eq!(
holds.get("after"),
Some(&vec![
HoldReason::Dependencies {
blocking: vec!["approve".into()]
},
HoldReason::Decision {
reference: DecisionRef::Attestation("approve".into())
},
])
);
assert_eq!(
holds.get("spare"),
Some(&vec![
HoldReason::Concurrency {
ahead: vec!["build".into()],
limit: 1,
},
HoldReason::Release {
awaiting: vec!["build".into()]
},
])
);
}
#[test]
fn a_cross_dag_dependency_that_has_not_settled_is_named_as_the_plan_wrote_it() {
let mut state = state_of(vec![agent("ship", &["run:other#build", "gone"])], &[]);
let statuses = state.statuses();
let holds = holds_now(
&state,
&statuses,
&BTreeMap::new(),
&BTreeMap::new(),
&BTreeMap::new(),
);
assert_eq!(
holds.get("ship"),
Some(&vec![HoldReason::Dependencies {
blocking: vec!["run:other#build".into()]
}]),
"a dependency the graph no longer holds was detached and holds nothing"
);
state
.cross_dag
.insert("run:other#build".into(), NodeStatus::Done);
let statuses = state.statuses();
assert_eq!(
holds_now(
&state,
&statuses,
&BTreeMap::new(),
&BTreeMap::new(),
&BTreeMap::new()
)
.get("ship"),
None
);
}
#[test]
fn paced_work_is_due_once_and_then_on_its_interval() {
assert!(due(None, Duration::from_secs(1)), "the first pass does it");
assert_eq!(until_due(None, Duration::from_secs(1)), Duration::ZERO);
let now = Instant::now();
assert!(!due(Some(now), Duration::from_secs(60)));
assert!(until_due(Some(now), Duration::from_secs(60)) > Duration::from_secs(50));
assert!(
due(Some(now), Duration::ZERO),
"a zero interval is always due"
);
assert_eq!(
next_quiet(&BTreeMap::new(), Duration::from_secs(2_400)),
Duration::MAX
);
let in_flight: BTreeMap<String, Dispatch> = [(
"build".to_string(),
Dispatch {
node: agent("build", &[]),
cancel: CancellationToken::new(),
started: now,
last_progress: now,
reported_quiet: false,
control: None,
},
)]
.into();
let next = next_quiet(&in_flight, Duration::from_secs(2_400));
assert!(
next > Duration::from_secs(2_000) && next <= Duration::from_secs(2_400),
"{next:?}"
);
}
#[test]
fn the_stall_threshold_falls_back_when_the_environment_is_unusable() {
assert!(stall_after_seconds() > 0);
}
#[test]
fn a_node_names_its_own_agent_graph_or_takes_the_shipped_default() {
let pinned = oneagentgraph::config::ConfigRef("./custom.yaml".into());
assert_eq!(node_graph(Some(&pinned), "default"), pinned);
assert_eq!(node_graph(None, "default").0, "default");
}
#[test]
fn only_the_commands_that_stop_a_dispatch_name_a_node_to_cancel() {
assert_eq!(
cancelled_by(&Command::Cancel {
id: "a".into(),
reason: None
}),
vec!["a".to_string()]
);
assert_eq!(
cancelled_by(&Command::Drop {
id: "a".into(),
dependents: crate::channel::Dependents::Detach
}),
vec!["a".to_string()]
);
assert!(cancelled_by(&Command::Complete { reason: "r".into() }).is_empty());
}
#[test]
fn dispatch_labels_carry_only_the_reserved_keys_and_never_a_round() {
let labels = dispatch_labels("demo", "build", Some("implement"), Some("engineer"));
assert_eq!(labels.run_id.as_deref(), Some("demo"));
assert_eq!(labels.node.as_deref(), Some("build"));
assert_eq!(labels.step.as_deref(), Some("implement"));
assert_eq!(labels.round, None, "a dispatch was stamped with a round");
assert!(labels.extra.is_empty());
}
}