#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
#[non_exhaustive]
pub enum DriverLiveness {
Driving,
DriverDead,
Parked,
Undriven,
}
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use crate::event::{Envelope, Source};
use crate::filter::EventFilter;
use crate::graph::{self, Landing, NodeStatus};
use crate::journal::PipelineKind;
use crate::ledger::{self, LaunchRecord};
use crate::projection::{self, MemberLabel, Refusal, RunState, Served};
use crate::report::{ToolText, Truncation};
use crate::sys;
pub use crate::ledger::Skipped;
pub use crate::ledger::RunPaths;
pub use crate::summary::{Listing, RunSummary, SUMMARY_SCHEMA_VERSION};
pub use crate::telemetry::{Bucket, BucketName, Party, RunTelemetry, Usage};
pub const DEFAULT_PARKED_AFTER_SECONDS: u64 = 1_800;
pub const PARKED_AFTER_ENV: &str = "ONEPIPELINE_PARKED_AFTER_SECONDS";
const ENDED_BY_THE_STOP: &str = "worker ended when the run was stopped";
const OUTLIVED_THE_STOP: &str = "worker may still be running: the stop could not reach it";
fn became_of_the_worker(state: &crate::projection::RunState) -> &'static str {
match state.stop {
crate::projection::StopState::WorkersUndetermined => OUTLIVED_THE_STOP,
_ => ENDED_BY_THE_STOP,
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
enum ObserverLiveness {
Watching,
ObserverDead,
ObserverNotRestarted,
Unobserved,
}
impl ObserverLiveness {
fn as_str(self) -> &'static str {
match self {
Self::Watching => "",
Self::ObserverDead => "OBSERVER DEAD",
Self::ObserverNotRestarted => "OBSERVER NOT RESTARTED",
Self::Unobserved => "NO OBSERVER",
}
}
}
fn observer_liveness(launch: &LaunchRecord) -> ObserverLiveness {
if launch.observer_graph().is_none() {
return ObserverLiveness::Unobserved;
}
if !launch.observer_ending.is_empty() {
return ObserverLiveness::ObserverNotRestarted;
}
if crate::agentgraph::graph_run_ended(&launch.graph_run, &launch.run_id) {
return ObserverLiveness::ObserverDead;
}
ObserverLiveness::Watching
}
impl DriverLiveness {
pub fn as_str(self) -> &'static str {
match self {
Self::Driving => "ACTIVE",
Self::DriverDead => "DRIVER DEAD",
Self::Parked => "PARKED",
Self::Undriven => "UNDRIVEN",
}
}
pub fn is_undriven(self) -> bool {
matches!(self, Self::DriverDead | Self::Parked)
}
}
pub fn parked_after_seconds() -> u64 {
std::env::var(PARKED_AFTER_ENV)
.ok()
.and_then(|value| value.parse().ok())
.filter(|seconds| *seconds > 0)
.unwrap_or(DEFAULT_PARKED_AFTER_SECONDS)
}
pub fn decision_outstanding(state: &RunState, paths: &RunPaths) -> bool {
state.awaiting_human_action() || blocking_surface(paths)
}
pub(crate) fn blocking_surface(paths: &RunPaths) -> bool {
let queue = crate::channel::ChannelState::new(paths).queue();
queue
.waiting
.iter()
.chain(queue.pending.iter())
.any(|surface| surface.blocking)
}
pub fn liveness(launch: &LaunchRecord, state: &RunState, paths: &RunPaths) -> DriverLiveness {
if state.stop_recorded() {
return DriverLiveness::DriverDead;
}
let ours = launch.recorded_host() == Some(sys::hostname().as_str());
if ours
&& launch
.driver_pid()
.is_some_and(|pid| !sys::process_may_be_live(pid.get()))
{
return DriverLiveness::DriverDead;
}
let quiet_for = state
.last_write_at
.map(|last| sys::now_millis().saturating_sub(last) / 1_000);
match quiet_for {
Some(seconds)
if seconds > parked_after_seconds() && !decision_outstanding(state, paths) =>
{
DriverLiveness::Parked
}
_ => DriverLiveness::Driving,
}
}
fn landings_the_run_re_read(state: &mut RunState, paths: &RunPaths) {
let Some(result) = ledger::read_json_opt::<crate::engine::RunResult>(&paths.result()) else {
return;
};
for node in result.nodes {
if node.landing == Some(Landing::Landed)
&& state.landings.get(&node.id) == Some(&Landing::Unlanded)
{
state.landings.insert(node.id, Landing::Landed);
}
}
}
#[derive(Debug)]
pub struct RunView {
pub paths: RunPaths,
pub launch: LaunchRecord,
pub events: Vec<Envelope>,
pub state: RunState,
}
impl RunView {
pub fn open(paths: &RunPaths) -> crate::Result<Self> {
if !paths.exists() {
return Err(crate::Error::NoSuchRun {
run: paths.run.clone(),
root: paths.dir.parent().unwrap_or(Path::new(".")).to_path_buf(),
});
}
let launch: LaunchRecord = ledger::read_json(&paths.launch())?;
let mut events = crate::journal::read(&paths.journal());
crate::journal::merge_order(&mut events);
let mut state = projection::fold(&events);
landings_the_run_re_read(&mut state, paths);
state.cross_dag = crate::crossdag::resolve_quietly(
&paths
.dir
.parent()
.map_or_else(ledger::runs_root, Path::to_path_buf),
&state.graph,
);
Ok(Self {
paths: paths.clone(),
launch,
events,
state,
})
}
pub fn liveness(&self) -> DriverLiveness {
liveness(&self.launch, &self.state, &self.paths)
}
pub fn unread_surfaces(&self) -> (usize, Option<u64>) {
let unread = self.unread();
(unread.count, unread.oldest_seconds)
}
pub(crate) fn unread(&self) -> Unread {
Unread::of(&crate::channel::ChannelState::new(&self.paths).queue())
}
pub fn summary(&self) -> String {
let statuses = self.state.statuses();
let done = statuses
.values()
.filter(|status| **status == NodeStatus::Done)
.count();
let unlanded = match unlanded_nodes(self).len() {
0 => String::new(),
count => format!(", {count} not landed as of settlement"),
};
let skipped = match statuses
.values()
.filter(|status| **status == NodeStatus::Skipped)
.count()
{
0 => String::new(),
count => format!(", {count} never attempted"),
};
format!("{done}/{} done{unlanded}{skipped}", statuses.len())
}
}
#[derive(Debug, Default)]
pub(crate) struct Unread {
pub(crate) count: usize,
pub(crate) oldest_seconds: Option<u64>,
pub(crate) kinds: Vec<(String, usize)>,
}
const MAX_NAMED_KINDS: usize = 4;
impl Unread {
pub(crate) fn of(queue: &crate::channel::Queue) -> Self {
let mut counts: BTreeMap<String, (bool, usize)> = BTreeMap::new();
for surface in &queue.waiting {
let seen = counts.entry(one_line(&surface.kind)).or_insert((false, 0));
seen.0 |= surface.blocking;
seen.1 += 1;
}
let mut ordered: Vec<(bool, usize, String)> = counts
.into_iter()
.map(|(kind, (blocking, count))| (blocking, count, kind))
.collect();
ordered.sort_by(|a, b| {
b.0.cmp(&a.0)
.then(a.1.cmp(&b.1))
.then_with(|| a.2.cmp(&b.2))
});
Self {
count: queue.waiting.len(),
oldest_seconds: queue
.waiting
.iter()
.map(|surface| sys::now_millis().saturating_sub(surface.queued_at) / 1_000)
.max(),
kinds: ordered
.into_iter()
.map(|(_, count, kind)| (kind, count))
.collect(),
}
}
pub(crate) fn phrase(&self) -> String {
let named: Vec<String> = self
.kinds
.iter()
.take(MAX_NAMED_KINDS)
.map(|(kind, count)| format!("{count} {kind}"))
.collect();
let rest = match self.kinds.len().saturating_sub(named.len()) {
0 => String::new(),
more => format!(", and {more} other kind(s)"),
};
format!("{}{rest}", named.join(", "))
}
}
#[derive(Debug)]
pub struct Survey {
pub root: PathBuf,
pub views: Vec<RunView>,
pub skipped: Vec<Skipped>,
}
impl Survey {
pub fn of(root: &Path) -> Self {
let index = ledger::all_runs(root);
let mut views = Vec::new();
let mut skipped = index.skipped;
for paths in index.runs {
match RunView::open(&paths) {
Ok(view) => views.push(view),
Err(error) => skipped.push(Skipped {
path: paths.dir,
reason: error.to_string(),
}),
}
}
skipped.sort_by(|a, b| a.path.cmp(&b.path));
Self {
root: root.to_path_buf(),
views,
skipped,
}
}
pub fn of_one(view: RunView) -> Self {
let root = view
.paths
.dir
.parent()
.map_or_else(ledger::runs_root, Path::to_path_buf);
Self {
root,
views: vec![view],
skipped: Vec::new(),
}
}
}
const MAX_NAMED_SKIPS: usize = 3;
fn skipped_lines(skipped: &[Skipped]) -> String {
if skipped.is_empty() {
return String::new();
}
let named: Vec<String> = skipped
.iter()
.take(MAX_NAMED_SKIPS)
.map(|root| {
format!(
"{}: {}",
one_line(&root.path.display().to_string()),
one_line(&root.reason)
)
})
.collect();
let rest = match skipped.len().saturating_sub(named.len()) {
0 => String::new(),
more => format!(", and {more} more"),
};
format!(
"{} run root(s) skipped: {}{rest}\n",
skipped.len(),
named.join("; ")
)
}
fn nothing_to_report(survey: &Survey) -> String {
let mut out = if survey.views.is_empty() && !survey.skipped.is_empty() {
format!(
"no run under {} could be read\n",
one_line(&survey.root.display().to_string())
)
} else {
"no runs recorded\n".to_string()
};
out.push_str(&skipped_lines(&survey.skipped));
out
}
struct Standing {
liveness: DriverLiveness,
work: WorkStanding,
convergence: Convergence,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Convergence {
Settled,
Moving,
}
impl Convergence {
fn of(converged: bool) -> Self {
if converged {
Self::Settled
} else {
Self::Moving
}
}
}
struct HeldNodes(Vec<String>);
impl HeldNodes {
fn of(nodes: Vec<String>) -> Option<Self> {
(!nodes.is_empty()).then_some(Self(nodes))
}
fn named(&self) -> String {
self.0.join(", ")
}
}
enum WorkStanding {
Complete,
Settled,
Outstanding,
Held(HeldWork),
}
enum HeldWork {
Parked(HeldNodes),
Rejected(HeldNodes),
Both {
parked: HeldNodes,
rejected: HeldNodes,
},
}
impl HeldWork {
fn of(parked: Vec<String>, rejected: Vec<String>) -> Option<Self> {
match (HeldNodes::of(parked), HeldNodes::of(rejected)) {
(Some(parked), Some(rejected)) => Some(Self::Both { parked, rejected }),
(Some(parked), None) => Some(Self::Parked(parked)),
(None, Some(rejected)) => Some(Self::Rejected(rejected)),
(None, None) => None,
}
}
fn ranked(&self) -> impl Iterator<Item = Intervention<'_>> {
let (parked, rejected) = match self {
Self::Parked(parked) => (Some(parked), None),
Self::Rejected(rejected) => (None, Some(rejected)),
Self::Both { parked, rejected } => (Some(parked), Some(rejected)),
};
parked
.map(Intervention::RequeueThenAdopt)
.into_iter()
.chain(rejected.map(Intervention::ReviewThenSupersede))
}
}
fn rejected_by_a_judge(view: &RunView, statuses: &BTreeMap<String, NodeStatus>) -> Vec<String> {
statuses
.iter()
.filter(|(_, status)| **status == NodeStatus::Failed)
.filter(|(id, _)| {
matches!(
view.state.outcomes.get(*id).map(String::as_str),
Some(crate::engine::TASK_FAILED | crate::engine::TASK_FAILED_CHANGE_OPEN)
)
})
.filter(|(id, _)| !crate::report::failed_verdicts(&view.events, id).is_empty())
.map(|(id, _)| id.clone())
.collect()
}
impl Standing {
fn of(view: &RunView) -> Self {
let statuses = view.state.statuses();
let converged = !statuses.is_empty() && graph::is_terminal(&statuses);
let parked: Vec<_> = if converged {
statuses
.iter()
.filter(|(_, status)| **status == NodeStatus::Parked)
.map(|(id, _)| id.clone())
.collect()
} else {
Vec::new()
};
let rejected = if converged {
rejected_by_a_judge(view, &statuses)
} else {
Vec::new()
};
let work = if converged && graph::state_of(&statuses) == graph::GraphState::Complete {
WorkStanding::Complete
} else if let Some(held) = HeldWork::of(parked, rejected) {
WorkStanding::Held(held)
} else if !converged
|| statuses
.values()
.any(|status| matches!(status, NodeStatus::Waiting | NodeStatus::Blocked))
{
WorkStanding::Outstanding
} else {
WorkStanding::Settled
};
Self {
liveness: view.liveness(),
work,
convergence: Convergence::of(converged),
}
}
fn word(&self) -> &'static str {
if matches!(&self.work, WorkStanding::Complete) {
"SETTLED"
} else {
self.liveness.as_str()
}
}
fn intervention(&self) -> Option<Intervention<'_>> {
if !self.liveness.is_undriven() {
return None;
}
match &self.work {
WorkStanding::Outstanding => Some(Intervention::Adopt),
WorkStanding::Held(held) => held.ranked().next(),
WorkStanding::Complete | WorkStanding::Settled => None,
}
}
}
enum Intervention<'a> {
Adopt,
RequeueThenAdopt(&'a HeldNodes),
ReviewThenSupersede(&'a HeldNodes),
}
fn requeue_then_adopt(run: &str, parked: &HeldNodes) -> String {
format!(
"its unfinished work is parked, and no driver dispatches a parked node: return {} \
to the frontier with a `requeue` on: onepipeline reply {run} — and only then \
attach a fresh driver with: onepipeline adopt {run}",
parked.named()
)
}
fn review_then_supersede(run: &str, rejected: &HeldNodes) -> String {
format!(
"its unfinished work is held up by {}, whose work a judge rejected, and no driver \
dispatches a rejected node as it stands: read the verdict with: onepipeline \
results {run} — and decide from it, most likely amending the task and superseding \
the node with an `amend` and a `retry` on: onepipeline reply {run}",
rejected.named()
)
}
pub fn liveness_word(view: &RunView) -> &'static str {
Standing::of(view).word()
}
pub(crate) fn has_settled(view: &RunView) -> bool {
Standing::of(view).convergence == Convergence::Settled
}
fn observer_verdict(view: &RunView, standing: &Standing) -> Option<ObserverLiveness> {
(standing.word() == DriverLiveness::Driving.as_str()).then(|| observer_liveness(&view.launch))
}
fn observer_suffix(view: &RunView, standing: &Standing) -> String {
match observer_verdict(view, standing) {
None | Some(ObserverLiveness::Watching) => String::new(),
Some(verdict @ ObserverLiveness::ObserverNotRestarted) => {
format!(" {}: {}", verdict.as_str(), view.launch.observer_ending)
}
Some(verdict) => format!(" {}", verdict.as_str()),
}
}
pub fn runs(root: &Path, mine_only: bool, session: &str) -> String {
let survey = Survey::of(root);
let mut out = String::new();
for view in &survey.views {
let owned = view.launch.owned_by(session);
if mine_only && !owned {
continue;
}
let marker = if owned { '*' } else { ' ' };
let standing = Standing::of(view);
out.push_str(&format!(
"{marker} {:<24} {:<24} {} {}{}\n",
view.paths.run,
view.launch.owner_label(session),
view.summary(),
standing.word(),
observer_suffix(view, &standing)
));
if let Some(intervention) = standing.intervention() {
out.push_str(&match intervention {
Intervention::Adopt => format!(
" {} — its ledger is intact; attach a fresh driver with: \
onepipeline adopt {}\n",
standing.word(),
view.paths.run
),
Intervention::RequeueThenAdopt(parked) => format!(
" {} — its ledger is intact; {}\n",
standing.word(),
requeue_then_adopt(&view.paths.run, parked)
),
Intervention::ReviewThenSupersede(rejected) => format!(
" {} — its ledger is intact; {}\n",
standing.word(),
review_then_supersede(&view.paths.run, rejected)
),
});
continue;
}
let unread = view.unread();
if let (count, Some(stale)) = (unread.count, unread.oldest_seconds) {
if count > 0 {
out.push_str(&format!(
" {count} planner update(s) waiting ({}), unread for {}; \
read them with: onepipeline next {}\n",
unread.phrase(),
crate::telemetry::duration(stale * 1_000),
view.paths.run
));
}
}
}
if out.is_empty() {
return nothing_to_report(&survey);
}
out.push_str(&skipped_lines(&survey.skipped));
out
}
pub fn status(survey: &Survey) -> String {
let mut out = String::new();
for view in &survey.views {
let standing = Standing::of(view);
out.push_str(&format!(
"{} {}{} {}\n",
view.paths.run,
standing.word(),
observer_suffix(view, &standing),
view.summary()
));
if let Some(intervention) = standing.intervention() {
out.push_str(&match intervention {
Intervention::Adopt => format!(
" {}: nothing is driving this run; adopt it or stop it\n",
standing.word()
),
Intervention::RequeueThenAdopt(parked) => format!(
" {}: nothing is driving this run and {}\n",
standing.word(),
requeue_then_adopt(&view.paths.run, parked)
),
Intervention::ReviewThenSupersede(rejected) => format!(
" {}: nothing is driving this run and {}\n",
standing.word(),
review_then_supersede(&view.paths.run, rejected)
),
});
}
if let Some(pending) = crate::channel::ChannelState::new(&view.paths).pending() {
out.push_str(&format!(
" waiting for planner {}: {} — {}\n",
if pending.blocking {
"decision"
} else {
"reply"
},
pending.kind,
pending.message
));
}
let unread = view.unread();
if unread.count > 0 {
out.push_str(&format!(
" {} planner update(s) waiting ({}), unread for {}\n",
unread.count,
unread.phrase(),
crate::telemetry::duration(unread.oldest_seconds.unwrap_or(0) * 1_000)
));
}
let statuses = view.state.statuses();
for (id, node_status) in &statuses {
if *node_status != NodeStatus::Running {
continue;
}
let age = view
.state
.dispatched_at
.get(id)
.map(|at| sys::now_millis().saturating_sub(*at));
let age = crate::telemetry::duration(age.unwrap_or(0));
if view.state.stop_recorded() {
let became = became_of_the_worker(&view.state);
out.push_str(&format!(" {id}: {became}, {age} in\n"));
continue;
}
out.push_str(&format!(" {id}: running for {age}"));
match view.state.activity.get(id) {
None => out.push_str(&format!(" — {}", DriverLiveness::Undriven.as_str())),
Some(activity) => out.push_str(&format!(" — {}", working(activity))),
}
out.push('\n');
}
for (id, node_status) in &statuses {
if *node_status != NodeStatus::Parked {
continue;
}
let Some(pending) = cancelling_for(&view.state, id) else {
continue;
};
out.push_str(&format!(
" {id}: cancelling — asked to stop {pending} ago and its dispatch has not \
settled; it still holds the node's workspace, so wait for it rather than \
requeueing the node\n"
));
}
for (id, node_status) in &statuses {
if *node_status != NodeStatus::Ready {
continue;
}
out.push_str(&format!(
" {id}: ready — {}\n",
waiting_on(&view.state, id)
));
}
for node in view.state.graph.iter() {
if let Some(amendment) = &node.amendment {
out.push_str(&format!(
" {}: amended — {}\n",
node.id,
one_line(amendment)
));
}
}
for (id, node_status) in &statuses {
if *node_status != NodeStatus::Failed {
continue;
}
let branch = view.state.branches.get(id).cloned().or_else(|| {
view.state
.graph
.get(id)
.and_then(|node| node.branch.clone())
});
if let Some(died) = death_phrase(&view.state, id, branch.as_deref()) {
out.push_str(&format!(" {id}: {died}\n"));
}
for record in chain_records(&view.state, id) {
out.push_str(&format!(
" {id}: {} — {}\n",
record.lead_in(),
chain_phrase(&record)
));
}
}
let drafted = drafted_nodes(view);
if !drafted.is_empty() {
out.push_str(&format!(
" {} node(s) complete and held as a draft: the run is waiting on the \
release(s) each names, and is neither stalled nor finished\n",
drafted.len()
));
for (id, awaiting) in &drafted {
let says = awaiting
.as_deref()
.map(|detail| format!(" — {detail}"))
.unwrap_or_default();
out.push_str(&format!(" {id}: complete-but-draft{says}\n"));
}
}
let unlanded = unlanded_nodes(view);
if !unlanded.is_empty() {
out.push_str(&format!(
" {} node(s) settled without landing: {} — as each settled, not as of now; \
`results {}` names the change to open\n",
unlanded.len(),
unlanded.join(", "),
view.paths.run
));
}
out.push_str(&journal_loss_line(view));
if let Some(health) = crate::agentgraph::health() {
out.push_str(&format!(" providers: {health}\n"));
}
}
if out.is_empty() {
return nothing_to_report(survey);
}
out.push_str(&skipped_lines(&survey.skipped));
out
}
fn death_phrase(state: &RunState, id: &str, branch: Option<&str>) -> Option<String> {
let word = match state.outcomes.get(id).map(String::as_str) {
Some(word @ (crate::engine::DISPATCH_DIED | crate::engine::PROVIDER_FAILED)) => word,
_ => return None,
};
let classified = match state.causes.get(id) {
Some(cause) => format!(" ({cause})"),
None => String::new(),
};
let where_the_work_is = match (branch, state.heads.get(id)) {
(Some(branch), Some(head)) => {
format!("{branch} may carry finished work, at {head}")
}
(Some(branch), None) => format!("{branch} may carry finished work"),
(None, _) => "it left no branch, so nothing of it survived".to_owned(),
};
let how = if word == crate::engine::PROVIDER_FAILED {
format!("the provider killed the dispatch{classified}, so nothing here is the work's fault")
} else {
format!("the dispatch died{classified} rather than failing its task")
};
Some(format!("{how}; {where_the_work_is}"))
}
fn cancelling_for(state: &RunState, id: &str) -> Option<String> {
let since = state.recorded.get(id)?.cancelling_since()?;
Some(crate::telemetry::duration(
sys::now_millis().saturating_sub(since),
))
}
#[derive(Debug, Clone, PartialEq, Eq)]
enum Fallthrough {
Served(String),
Refused,
Unrecorded,
}
struct ChainRecord<'a> {
refusal: &'a Refusal,
became: Fallthrough,
records: std::num::NonZeroU64,
}
impl ChainRecord<'_> {
fn lead_in(&self) -> &'static str {
match self.became {
Fallthrough::Refused => "failed",
Fallthrough::Served(_) | Fallthrough::Unrecorded => "fallback",
}
}
}
fn chain_records<'a>(state: &'a RunState, node: &str) -> Vec<ChainRecord<'a>> {
let mut records: Vec<ChainRecord<'a>> = Vec::new();
for refusal in refusals_of(state, node) {
let became = became_of(state, node, refusal);
if let Some(same) = records.iter_mut().find(|seen| {
seen.refusal.advanced.identity == refusal.advanced.identity
&& seen.refusal.advanced.role == refusal.advanced.role
&& seen.refusal.advanced.reason == refusal.advanced.reason
&& seen.refusal.member == refusal.member
&& seen.became == became
}) {
same.records = same.records.saturating_add(refusal.records.get());
continue;
}
records.push(ChainRecord {
refusal,
became,
records: refusal.records,
});
}
records
}
fn became_of(state: &RunState, node: &str, refusal: &Refusal) -> Fallthrough {
let (Some(role), Some(turn)) = (refusal.advanced.role, refusal.advanced.turn) else {
return Fallthrough::Unrecorded;
};
match served_in(state, node, &refusal.member, role, turn) {
Some(served) => Fallthrough::Served(served.session.identity.clone()),
None => Fallthrough::Refused,
}
}
fn served_in<'a>(
state: &'a RunState,
node: &str,
member: &MemberLabel,
role: oneagentgraph::event::Role,
turn: u64,
) -> Option<&'a Served> {
state.served.get(node)?.iter().find(|served| {
served.member == *member && served.session.role == role && served.session.turn == turn
})
}
fn refusals_of<'a>(state: &'a RunState, node: &str) -> &'a [Refusal] {
state.refusals.get(node).map_or(&[], Vec::as_slice)
}
fn chain_phrase(record: &ChainRecord) -> String {
let refusal = record.refusal;
let role = refusal
.advanced
.role
.and_then(|role| serde_json::to_value(role).ok());
let side = match (
role.as_ref().and_then(serde_json::Value::as_str),
&refusal.member,
) {
(Some(role), _) => format!("the {role} side"),
(None, MemberLabel::Named(member)) => format!("member '{member}'"),
(None, MemberLabel::Unstamped) => "a side the record does not name".to_string(),
(None, MemberLabel::Unreadable) => "a side this build cannot read".to_string(),
};
let reason = if refusal.advanced.reason.is_empty() {
"for a reason the record does not carry".to_string()
} else {
format!("({})", refusal.advanced.reason)
};
let again = if record.records.get() > 1 {
format!(", recorded {} times", record.records)
} else {
String::new()
};
let identity = &refusal.advanced.identity;
one_line(&match &record.became {
Fallthrough::Refused => format!("{side}: identity '{identity}' refused {reason}{again}"),
Fallthrough::Served(who) => {
format!("{side} fell through '{identity}' {reason} → served by '{who}'{again}")
}
Fallthrough::Unrecorded => format!(
"{side} fell through '{identity}' {reason}; nothing this run recorded names what \
served that turn{again}"
),
})
}
fn verdict_phrase(verdict: &crate::report::FailedVerdict) -> String {
let criterion = match &verdict.criterion {
Some(criterion) => format!("'{criterion}'"),
None => "a criterion the record does not name".to_string(),
};
let reason = match &verdict.reason {
Some(reason) => reason.clone(),
None => "the record carries no reason".to_string(),
};
one_line(&format!("{criterion} failed — {reason}"))
}
fn skipped_by_phrase(causes: &[(String, NodeStatus)]) -> String {
if causes.is_empty() {
return "a dependency this run can no longer name".to_string();
}
causes
.iter()
.map(|(dependency, status)| format!("{dependency} ({})", status.as_str()))
.collect::<Vec<_>>()
.join(", ")
}
fn attested_after_failing(view: &RunView, node: &str) -> bool {
view.state.attestations.contains(node)
&& view.events.iter().any(|event| {
event.kind.0 == PipelineKind::NodeSettled.as_str()
&& event.labels.node.as_deref() == Some(node)
&& event
.payload
.get("status")
.and_then(serde_json::Value::as_str)
== Some(NodeStatus::Failed.as_str())
})
}
fn unlanded_nodes(view: &RunView) -> Vec<String> {
let statuses = view.state.statuses();
view.state
.landings
.iter()
.filter(|(_, landing)| **landing == Landing::Unlanded)
.filter(|(node, _)| statuses.get(*node) != Some(&NodeStatus::CompleteDraft))
.map(|(node, _)| node.clone())
.collect()
}
fn drafted_nodes(view: &RunView) -> Vec<(String, Option<String>)> {
view.state
.statuses()
.into_iter()
.filter(|(_, status)| *status == NodeStatus::CompleteDraft)
.map(|(node, _)| {
let awaiting = settled_detail(view, &node).map(|detail| one_line(&detail));
(node, awaiting)
})
.collect()
}
fn settled_detail(view: &RunView, node: &str) -> Option<String> {
view.events
.iter()
.rev()
.find(|event| {
event.kind.0 == PipelineKind::NodeSettled.as_str()
&& event.labels.node.as_deref() == Some(node)
})
.and_then(|event| event.payload.get("detail"))
.and_then(|detail| detail.as_str())
.map(str::to_owned)
}
fn waiting_on(state: &RunState, id: &str) -> String {
const QUEUED: &str = "queued for dispatch";
let Some(repo) = state.graph.get(id).and_then(|node| node.repo.as_deref()) else {
return QUEUED.to_string();
};
let holders = match crate::vcs::holders_of(repo) {
Ok(holders) => holders,
Err(why) => {
return format!(
"{QUEUED}, and this host cannot say whether the '{repo}' workspace is \
free: {}",
one_line(&why)
)
}
};
let held: Vec<String> = holders
.into_iter()
.filter(|holder| {
holder.state == onevcs::Lifecycle::Open && holder.liveness == onevcs::Liveness::Live
})
.map(|holder| {
format!(
"session '{}' (owner_pid {})",
holder.token.0, holder.owner_pid
)
})
.collect();
if held.is_empty() {
return QUEUED.to_string();
}
format!(
"waiting for the '{repo}' workspace, held by {}",
held.join(", ")
)
}
fn working(activity: &crate::projection::NodeActivity) -> String {
let ago = |at: u64| crate::telemetry::duration(sys::now_millis().saturating_sub(at));
let alive = activity
.last_heartbeat_at
.filter(|beat| activity.progress.is_none_or(|done| *beat > done.last_at()))
.map(|beat| format!("; alive {} ago", ago(beat)))
.unwrap_or_default();
let Some(progress) = activity.progress else {
return format!("nothing recorded yet{alive}");
};
let counted = format!(
"{} event(s), {} ago{alive}",
progress.events(),
ago(progress.last_at())
);
match &activity.doing {
Some(doing) => format!("now {doing} ({counted})"),
None => counted,
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
enum Proof {
Held,
Stale(String),
Unproven(String),
}
enum Registry {
Read(BTreeMap<String, Vec<ledger::DispatchRecord>>),
Unreadable(String),
}
impl Registry {
fn of(paths: &RunPaths) -> Self {
match ledger::dispatches_of(paths) {
Ok(records) => {
let mut by_node: BTreeMap<String, Vec<ledger::DispatchRecord>> = BTreeMap::new();
for record in records {
by_node.entry(record.node.clone()).or_default().push(record);
}
Self::Read(by_node)
}
Err(error) => Self::Unreadable(error.to_string()),
}
}
fn proves(&self, node: &str) -> Proof {
let by_node = match self {
Self::Unreadable(why) => {
return Proof::Unproven(format!(
"the run's dispatch registry cannot be read: {why}"
))
}
Self::Read(by_node) => by_node,
};
let records = by_node.get(node).map(Vec::as_slice).unwrap_or_default();
let mut unproven = None;
let mut stale = None;
for record in records {
match proof_of(record) {
Proof::Held => return Proof::Held,
Proof::Unproven(why) => unproven.get_or_insert(why),
Proof::Stale(why) => stale.get_or_insert(why),
};
}
match (unproven, stale) {
(Some(why), _) => Proof::Unproven(why),
(None, Some(why)) => Proof::Stale(why),
(None, None) => Proof::Unproven(
"the run's dispatch registry holds no entry for it, so nothing here says which \
process it is in"
.to_string(),
),
}
}
}
fn proof_of(record: &ledger::DispatchRecord) -> Proof {
if record.host != sys::hostname() {
return Proof::Unproven(format!(
"its dispatch runs on {}, and a pid means nothing across machines",
record.host
));
}
if !sys::process_may_be_live(record.pid) {
return Proof::Stale(format!("its dispatch (pid {}) is gone", record.pid));
}
match sys::process_start_token(record.pid) {
None => Proof::Unproven(format!(
"this host will not say when pid {} started",
record.pid
)),
Some(token) if token.matches(&record.started) => Proof::Held,
Some(_) => Proof::Stale(format!(
"pid {} is a different process from the one its dispatch was recorded in",
record.pid
)),
}
}
pub fn host(survey: &Survey) -> String {
let mut out = format!("host {}\n", sys::hostname());
out.push_str(&format!(
" reading {}\n",
one_line(&survey.root.display().to_string())
));
let mut rendered = false;
let mut ignored: Vec<String> = Vec::new();
for view in &survey.views {
let registry = Registry::of(&view.paths);
for (id, status) in &view.state.statuses() {
if *status != NodeStatus::Running {
continue;
}
let proof = registry.proves(id);
if let Proof::Stale(why) = &proof {
ignored.push(one_line(&format!("{}/{id}: {why}", view.paths.run)));
continue;
}
let age = view
.state
.dispatched_at
.get(id)
.map(|at| sys::now_millis().saturating_sub(*at))
.unwrap_or(0);
rendered = true;
out.push_str(&format!(
" {:<24} {:<20} {:<16} {}",
view.paths.run,
id,
view.launch.launcher,
crate::telemetry::duration(age)
));
if let Proof::Unproven(why) = &proof {
out.push_str(&format!(" UNPROVEN: {}", one_line(why)));
}
out.push('\n');
}
}
if !rendered {
out.push_str(" no live dispatches\n");
}
if !ignored.is_empty() {
out.push_str(&format!(
" {} stale registry entr{} ignored: {}\n",
ignored.len(),
if ignored.len() == 1 { "y" } else { "ies" },
ignored.join("; ")
));
}
out.push_str(&skipped_lines(&survey.skipped));
out
}
pub fn shaped<'a>(view: &'a RunView, filter: &EventFilter) -> Vec<&'a Envelope> {
view.events
.iter()
.filter(|event| filter.matches(event))
.collect()
}
pub fn monitor(view: &RunView, filter: &EventFilter) -> String {
let mut out = String::from(
"Concise graph events; ask the producing library for full detail by stream id.\n",
);
for event in shaped(view, filter) {
out.push_str(&event_line(view, event));
out.push('\n');
}
out.push_str(&format!(
"-- {} {} {} {}\n",
view.paths.run,
view.summary(),
liveness_word(view),
graph::state_of(&view.state.statuses()).as_str()
));
out
}
pub(crate) fn event_line(view: &RunView, event: &Envelope) -> String {
let id = match event.source {
Source::Pipeline => format!("graph:{}", event.labels.node.as_deref().unwrap_or("-")),
Source::Agentgraph => format!("agent:{}", event.stream),
Source::Vcs => format!("vcs:{}", event.stream),
};
format!(
"{} {:<28} {}{}",
event.ts,
id,
summarize(event),
superseded_suffix(view, event)
)
}
fn superseded_suffix(view: &RunView, event: &Envelope) -> String {
if event.source != Source::Pipeline {
return String::new();
}
let Some(node) = event.labels.node.as_deref() else {
return String::new();
};
match view.state.superseded.get(node) {
Some(replacement) => format!(" — superseded, retried as {}", one_line(replacement)),
None => String::new(),
}
}
fn summarize(event: &Envelope) -> String {
const CAP: usize = 96;
let mut detail = event.kind.0.clone();
for key in ["status", "landing", "outcome", "state", "message", "reason"] {
if let Some(value) = event.payload.get(key).and_then(|v| v.as_str()) {
detail.push_str(&format!(" {value}"));
}
}
let stripped: String = detail
.chars()
.map(|c| if c.is_control() { ' ' } else { c })
.collect();
if stripped.chars().count() <= CAP {
return stripped;
}
stripped.chars().take(CAP).collect()
}
fn landed_phrase(landing: Landing, settled_at: Option<u64>) -> String {
let ago = match settled_at {
Some(at) => format!(
" {} ago",
crate::telemetry::duration(sys::now_millis().saturating_sub(at))
),
None => String::new(),
};
match landing {
Landing::Landed => "landed on its base".to_string(),
Landing::Unlanded => format!(
"NOT landed: the change had not reached its base when this settled{ago}, and \
no later read has said otherwise — open the change for where it is now"
),
}
}
fn journal_loss_line(view: &RunView) -> String {
let integrity = crate::journal::integrity(&view.paths.journal());
if integrity.is_whole() {
return String::new();
}
format!(
" journal: {} — this run's record of itself is incomplete\n",
integrity.phrase()
)
}
pub fn results(view: &RunView) -> String {
let mut out = format!(
"{} {}\n",
view.paths.run,
graph::state_of(&view.state.statuses()).as_str()
);
let statuses = view.state.statuses();
for node in view.state.graph.iter() {
let status = statuses
.get(&node.id)
.copied()
.unwrap_or(NodeStatus::Pending);
out.push_str(&format!(" {:<24} {}", node.id, status.as_str()));
if let Some(outcome) = view.state.outcomes.get(&node.id) {
out.push_str(&format!(" ({outcome})"));
}
if let Some(pending) = cancelling_for(&view.state, &node.id) {
out.push_str(&format!(" — cancelling, asked to stop {pending} ago"));
}
if let Some(landing) = view.state.landings.get(&node.id) {
let settled_at = view.state.settled_at.get(&node.id).copied();
out.push_str(&format!(" — {}", landed_phrase(*landing, settled_at)));
}
if attested_after_failing(view, &node.id) {
out.push_str(" — settled failed, attested as landed");
}
if status == NodeStatus::Running && view.state.stop_recorded() {
out.push_str(&format!(" — {}", became_of_the_worker(&view.state)));
}
let branch = view
.state
.branches
.get(&node.id)
.or(node.branch.as_ref())
.cloned();
if let (NodeStatus::Parked | NodeStatus::Failed | NodeStatus::Cancelled, Some(branch)) =
(status, &branch)
{
out.push_str(&format!(" — preserved on {branch}"));
}
if let Some(session) = view.state.abandoned.get(&node.id) {
out.push_str(&format!(
" — a dispatch was abandoned when the run was adopted; its work is on {} \
(onevcs session {})",
session.branch(),
session.token().0
));
}
if let Some(url) = view.state.change_urls.get(&node.id) {
out.push_str(&format!(" — {url}"));
}
out.push('\n');
if let Some(died) = death_phrase(&view.state, &node.id, branch.as_deref()) {
out.push_str(&format!(" died: {died}\n"));
}
if let Some(detail) = settled_detail(view, &node.id) {
out.push_str(&format!(" detail: {}\n", one_line(&detail)));
}
if let Some(amendment) = &node.amendment {
out.push_str(&format!(" amendment: {}\n", one_line(amendment)));
}
if status == NodeStatus::Failed {
for verdict in crate::report::failed_verdicts(&view.events, &node.id) {
out.push_str(&format!(" verdict: {}\n", verdict_phrase(&verdict)));
}
let chains = chain_records(&view.state, &node.id);
for record in chains
.iter()
.filter(|record| record.became == Fallthrough::Refused)
{
out.push_str(&format!(" provider: {}\n", chain_phrase(record)));
}
for record in chains
.iter()
.filter(|record| record.became != Fallthrough::Refused)
{
out.push_str(&format!(" fallback: {}\n", chain_phrase(record)));
}
}
if status == NodeStatus::Skipped {
out.push_str(&format!(
" never attempted; skipped by: {}\n",
skipped_by_phrase(&graph::skipped_by(&view.state.graph, &statuses, &node.id))
));
}
if status == NodeStatus::Waiting {
if let Some(task) = &node.task {
out.push_str(&format!(" action: {task}\n"));
}
let unblocks = graph::unblocks(&view.state.graph, &node.id);
if !unblocks.is_empty() {
out.push_str(&format!(" unblocks: {}\n", unblocks.join(", ")));
}
}
}
out.push_str(&superseded_lines(view));
out.push_str(&journal_loss_line(view));
out
}
fn superseded_lines(view: &RunView) -> String {
let mut out = String::new();
for (node, replacement) in &view.state.superseded {
out.push_str(&format!(
" {:<24} superseded — retried as {}\n",
one_line(node),
one_line(replacement)
));
}
out
}
pub fn transcript(view: &RunView, only: Option<&str>) -> String {
let mut out = String::new();
let settlements = crate::report::evidence(&view.paths, &view.events);
for node in nodes_with_agent_records(view, only) {
out.push_str(&format!("{} {}\n", view.paths.run, one_line(&node)));
for event in view
.events
.iter()
.filter(|event| event.source == Source::Agentgraph)
.filter(|event| event.labels.node.as_deref() == Some(node.as_str()))
{
let field = |key: &str| {
event
.payload
.get(key)
.and_then(|value| value.as_str())
.unwrap_or_default()
};
match event.kind.0.as_str() {
"turn-started" => out.push_str(&format!(
" turn {}\n",
event
.payload
.get("turn")
.map_or_else(|| "-".to_string(), ToString::to_string)
)),
"turn-activity" => out.push_str(&format!(
" {} {} {}\n",
one_line(field("kind")),
one_line(field("name")),
tool_text(&ToolText::of(field("kind"), |key| event.payload.get(key)))
)),
_ => {}
}
}
for settled in settlements
.iter()
.filter(|settled| settled.node.as_deref() == Some(node.as_str()))
{
out.push_str(&format!(
" report {} {}\n",
one_line(settled.member.as_deref().unwrap_or("-")),
one_line(&settled.named.display().to_string())
));
let Some(document) = crate::report::read(&settled.kept) else {
out.push_str(
" not retained by this run, so it is not read: only this run's own \
copy of a report is ever opened\n",
);
continue;
};
let turns = crate::report::turns(&document);
if turns.is_empty() {
out.push_str(" it carries no transcript\n");
}
for turn in turns {
out.push_str(&format!(" {}\n", one_line(&turn.role)));
for line in turn.text.lines() {
out.push_str(&format!(" {}\n", one_line(line)));
}
for tool in turn.tools {
out.push_str(&format!(
" {} {} {}\n",
one_line(&tool.kind),
one_line(&tool.name),
tool_text(&tool.text)
));
}
}
}
}
if out.is_empty() {
out.push_str("no dispatch has recorded a transcript\n");
}
out
}
pub(crate) fn nodes_with_agent_records(view: &RunView, only: Option<&str>) -> Vec<String> {
let mut nodes: Vec<String> = view
.events
.iter()
.filter(|event| event.source == Source::Agentgraph)
.filter_map(|event| event.labels.node.clone())
.filter(|node| only.is_none_or(|wanted| wanted == node))
.collect();
nodes.sort_unstable();
nodes.dedup();
nodes
}
const MAX_TOOL_OUTPUT_CHARS: usize = crate::event::MAX_PAYLOAD_TEXT_BYTES;
fn tool_text(text: &ToolText) -> String {
let (output, truncated) = match text {
ToolText::Acted(detail) => return one_line(detail),
ToolText::Returned { output, truncated } => (output, *truncated),
};
if output.is_empty() {
return String::new();
}
let stripped = one_line(output);
let whole = stripped.chars().count();
let mut text: String = stripped.chars().take(MAX_TOOL_OUTPUT_CHARS).collect();
let mut notes: Vec<String> = Vec::new();
if whole > MAX_TOOL_OUTPUT_CHARS {
notes.push(format!("{MAX_TOOL_OUTPUT_CHARS} of {whole} characters"));
}
match truncated {
Truncation::Whole => {}
Truncation::Cut => notes.push("already cut short by the producer".to_string()),
Truncation::Unreadable => {
notes.push("the producer's truncation flag is unreadable".to_string());
}
}
if !notes.is_empty() {
text.push_str(&format!(" … [{}]", notes.join("; ")));
}
text
}
pub(crate) fn one_line(text: &str) -> String {
text.chars()
.map(|c| if c.is_control() { ' ' } else { c })
.collect()
}
pub fn goals(survey: &Survey) -> String {
let mut out = String::new();
for view in &survey.views {
let goal = view
.state
.plan
.as_ref()
.and_then(|plan| plan.goal.as_ref())
.map(|goal| goal.text.clone())
.unwrap_or_else(|| crate::plan::NO_GOAL.to_string());
out.push_str(&format!(
"{} {}\n {}\n {}\n",
view.paths.run,
liveness_word(view),
goal,
view.summary()
));
let mut repos: Vec<&str> = view
.state
.graph
.iter()
.filter_map(|node| node.repo.as_deref())
.collect();
repos.sort_unstable();
repos.dedup();
if !repos.is_empty() {
out.push_str(&format!(" identities: {}\n", repos.join(", ")));
}
}
if out.is_empty() {
return nothing_to_report(survey);
}
out.push_str(&skipped_lines(&survey.skipped));
out
}
#[cfg(test)]
mod tests {
use super::*;
use crate::event::{EventKind, Labels, ENVELOPE_VERSION};
use crate::filter::Filters;
use crate::plan::{Node, Plan, PLAN_SCHEMA_VERSION};
use serde_json::json;
use std::path::PathBuf;
fn scratch(name: &str) -> PathBuf {
let dir = std::env::temp_dir().join(format!("onepipeline-views-{name}-{}", sys::pid()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).expect("a scratch root");
dir
}
fn plan() -> Plan {
Plan {
schema_version: PLAN_SCHEMA_VERSION,
goal: Some(crate::plan::Goal {
text: "close the coverage gap".into(),
}),
name: Some("demo".into()),
concurrency: 4,
tasks: vec![Node {
id: "build".into(),
persona: Some("engineer".into()),
task: Some("## What\ndo it".into()),
..Node::default()
}],
}
}
fn launch(pid: u32) -> LaunchRecord {
LaunchRecord {
run_id: "demo".into(),
project: "plans:demo".into(),
dir: PathBuf::from("/tmp/launch"),
graph: "graphs/dag-scope.yaml".into(),
graph_run: String::new(),
observer_runs: Vec::new(),
observer_ending: String::new(),
node_graph: String::new(),
pr_author_graph: String::new(),
node_validator: String::new(),
envelope_reviewer: String::new(),
launcher: "claude-code".into(),
session: "session-a".into(),
pid,
host: sys::hostname(),
started: sys::process_start_token(pid)
.map(|token| token.recorded().to_string())
.unwrap_or_default(),
started_at: sys::now_rfc3339(),
heartbeat_interval: 1_800,
dag_sets: Vec::new(),
node_sets: Vec::new(),
adoptions: 0,
filters: Filters::default(),
}
}
fn write_run(root: &Path, run: &str, pid: u32, events: &[Envelope]) -> RunPaths {
let paths = RunPaths::under(root, run);
paths.create().expect("the run directory");
let mut record = launch(pid);
record.run_id = run.to_string();
ledger::write_json(&paths.launch(), &record).expect("a launch record");
for event in events {
ledger::append_line(
&paths.journal(),
&serde_json::to_string(event).expect("an event"),
)
.expect("appended");
}
paths
}
fn register(paths: &RunPaths, record: &ledger::DispatchRecord) {
std::fs::create_dir_all(paths.dispatches()).expect("the registry directory");
ledger::write_json(&paths.dispatch(record.pid, 0), record).expect("a registry entry");
}
fn dispatched_here(node: &str) -> ledger::DispatchRecord {
ledger::DispatchRecord {
node: node.to_string(),
pid: sys::pid(),
host: sys::hostname(),
dispatched_at: sys::now_rfc3339(),
started: sys::process_start_token(sys::pid())
.map(|token| token.recorded().to_string())
.unwrap_or_default(),
}
}
fn event(
kind: crate::journal::PipelineKind,
node: Option<&str>,
fields: &[(&str, serde_json::Value)],
) -> Envelope {
relayed(
EventKind(kind.as_str().into()),
Source::Pipeline,
node,
fields,
)
}
fn relayed(
kind: EventKind,
source: Source,
node: Option<&str>,
fields: &[(&str, serde_json::Value)],
) -> Envelope {
Envelope {
v: ENVELOPE_VERSION,
ts: sys::now_rfc3339(),
stream: "s".into(),
seq: 0,
source,
kind,
phase: None,
labels: Labels {
run_id: Some("demo".into()),
round: Some(1),
node: node.map(str::to_string),
..Labels::default()
},
payload: crate::journal::payload(fields),
artifacts: Vec::new(),
}
}
fn dead_pid() -> u32 {
sys::reaped_pid()
}
#[test]
fn the_unread_line_names_the_kinds_waiting_and_leads_with_a_blocking_one() {
let root = scratch("unread-kinds");
let paths = write_run(
&root,
"demo",
sys::pid(),
&[event(
crate::journal::PipelineKind::RunStarted,
None,
&[("plan", json!(plan()))],
)],
);
let channel = crate::channel::ChannelState::new(&paths);
let queue = |kind: &str, blocking: bool| {
channel
.push(crate::channel::Surface {
id: 0,
kind: kind.into(),
message: format!("something about {kind}"),
source: "proposal".into(),
blocking,
queued_at: sys::now_millis(),
workstream: None,
})
.expect("the surface queues");
};
for _ in 0..6 {
queue("monitor", false);
}
queue("planner-question", true);
for kind in ["edit-rejected", "quiet-worker", "check-in", "proposal"] {
queue(kind, false);
}
let view = RunView::open(&paths).expect("the run reads");
assert_eq!(view.unread_surfaces().0, 11);
let unread = view.unread();
assert_eq!(
unread.phrase(),
"1 planner-question, 1 check-in, 1 edit-rejected, 1 proposal, and 2 other kind(s)"
);
let rendered = runs(&root, false, "session-a");
assert!(
rendered.contains("11 planner update(s) waiting (1 planner-question,"),
"{rendered}"
);
assert!(
status(&Survey::of(&root)).contains("1 planner-question,"),
"{}",
status(&Survey::of(&root))
);
std::fs::remove_dir_all(&root).ok();
}
#[test]
fn a_skip_with_no_cause_left_in_the_graph_is_still_phrased() {
assert_eq!(
skipped_by_phrase(&[]),
"a dependency this run can no longer name"
);
assert_eq!(
skipped_by_phrase(&[
("build".to_string(), NodeStatus::Failed),
("lint".to_string(), NodeStatus::Skipped),
]),
"build (failed), lint (skipped)"
);
}
#[test]
fn a_driver_this_host_can_prove_is_gone_reads_as_driver_dead() {
let root = scratch("dead");
write_run(
&root,
"demo",
dead_pid(),
&[event(
crate::journal::PipelineKind::RunStarted,
None,
&[("plan", json!(plan()))],
)],
);
let view = RunView::open(&RunPaths::under(&root, "demo")).expect("the run reads");
assert_eq!(view.liveness(), DriverLiveness::DriverDead);
assert!(view.liveness().is_undriven());
assert!(runs(&root, false, "session-a").contains("DRIVER DEAD"));
std::fs::remove_dir_all(&root).ok();
}
fn quiet_run(root: &Path, run: &str) -> RunPaths {
let mut stale = event(
crate::journal::PipelineKind::RunStarted,
None,
&[("plan", json!(plan()))],
);
stale.ts = "2020-01-01T00:00:00Z".into();
write_run(root, run, sys::pid(), &[stale])
}
#[test]
fn a_live_driver_that_has_gone_quiet_with_nothing_outstanding_reads_as_parked() {
let root = scratch("quiet-parked");
let paths = quiet_run(&root, "demo");
let view = RunView::open(&paths).expect("the run reads");
assert_eq!(view.liveness(), DriverLiveness::Parked);
std::fs::remove_dir_all(&root).ok();
}
#[test]
fn a_live_driver_quiet_behind_a_blocking_surface_reads_as_active() {
let root = scratch("quiet-blocking");
let paths = quiet_run(&root, "demo");
crate::channel::ChannelState::new(&paths)
.push(crate::channel::Surface {
id: 0,
kind: "blocker".into(),
message: "Node build needs a decision; proceed?".into(),
source: "monitor".into(),
blocking: true,
queued_at: sys::now_millis(),
workstream: Some("build".into()),
})
.expect("the surface queues");
let view = RunView::open(&paths).expect("the run reads");
assert_eq!(view.liveness(), DriverLiveness::Driving);
assert!(!view.liveness().is_undriven());
std::fs::remove_dir_all(&root).ok();
}
#[test]
fn a_live_driver_that_is_writing_reads_as_active() {
let root = scratch("live");
write_run(
&root,
"demo",
sys::pid(),
&[event(
crate::journal::PipelineKind::RunStarted,
None,
&[("plan", json!(plan()))],
)],
);
let view = RunView::open(&RunPaths::under(&root, "demo")).expect("the run reads");
assert_eq!(view.liveness(), DriverLiveness::Driving);
assert!(!view.liveness().is_undriven());
std::fs::remove_dir_all(&root).ok();
}
#[test]
fn a_launch_record_naming_no_driver_is_never_read_as_a_dead_one() {
let root = scratch("no-driver");
let live = event(
crate::journal::PipelineKind::RunStarted,
None,
&[("plan", json!(plan()))],
);
let paths = write_run(&root, "unclaimed", dead_pid(), std::slice::from_ref(&live));
without(&paths, &["pid", "started"]);
let view = RunView::open(&paths).expect("a record naming no pid still reads");
assert_eq!(view.launch.driver_pid(), None);
assert_eq!(view.launch.recorded_host(), Some(sys::hostname().as_str()));
assert_eq!(
view.liveness(),
DriverLiveness::Driving,
"a pid nobody recorded was probed, and its absence read as a dead driver"
);
let older = write_run(&root, "oldest", dead_pid(), std::slice::from_ref(&live));
without(
&older,
&[
"session",
"pid",
"host",
"started",
"started_at",
"heartbeat_interval",
],
);
let view = RunView::open(&older).expect("a record predating all five still reads");
assert_eq!(view.launch.driver_pid(), None);
assert_eq!(view.launch.recorded_host(), None);
assert_eq!(view.launch.launched_at(), None);
assert_eq!(view.launch.pacemaker_interval(), None);
assert_ne!(view.liveness(), DriverLiveness::DriverDead);
let rendered = runs(&root, false, "session-a");
assert!(rendered.contains("unclaimed"), "{rendered}");
assert!(rendered.contains("oldest"), "{rendered}");
assert!(rendered.contains("[unknown]"), "{rendered}");
assert!(
!rendered.contains("run root(s) skipped"),
"a record an older build wrote was refused:\n{rendered}"
);
std::fs::remove_dir_all(&root).ok();
}
fn without(paths: &RunPaths, keys: &[&str]) {
let mut document: serde_json::Value =
serde_json::from_str(&std::fs::read_to_string(paths.launch()).expect("the record"))
.expect("a launch record");
let fields = document.as_object_mut().expect("a launch record");
for key in keys {
fields.remove(*key);
}
std::fs::write(paths.launch(), document.to_string()).expect("an older build's record");
}
#[test]
fn a_pid_recorded_on_another_host_never_reads_as_dead() {
let root = scratch("elsewhere");
let paths = RunPaths::under(&root, "demo");
paths.create().expect("the run directory");
let mut record = launch(dead_pid());
record.host = "some-other-host".into();
ledger::write_json(&paths.launch(), &record).expect("a launch record");
ledger::append_line(
&paths.journal(),
&serde_json::to_string(&event(
crate::journal::PipelineKind::RunStarted,
None,
&[("plan", json!(plan()))],
))
.expect("an event"),
)
.expect("appended");
let view = RunView::open(&paths).expect("the run reads");
assert_eq!(
view.liveness(),
DriverLiveness::Driving,
"a pid means nothing across machines"
);
std::fs::remove_dir_all(&root).ok();
}
#[test]
fn a_run_nobody_recorded_is_no_such_run() {
let root = scratch("missing");
let error = RunView::open(&RunPaths::under(&root, "nowhere")).unwrap_err();
assert!(matches!(error, crate::Error::NoSuchRun { .. }));
assert!(runs(&root, false, "session-a").contains("no runs recorded"));
std::fs::remove_dir_all(&root).ok();
}
#[test]
fn only_the_reader_sees_mine_and_a_foreign_run_is_labelled_by_digest() {
let root = scratch("owner");
write_run(
&root,
"demo",
sys::pid(),
&[event(
crate::journal::PipelineKind::RunStarted,
None,
&[("plan", json!(plan()))],
)],
);
let listing = runs(&root, false, "session-a");
assert!(listing.contains("[mine]"), "{listing}");
let foreign = runs(&root, false, "session-b");
assert!(!foreign.contains("[mine]"), "{foreign}");
assert!(
!foreign.contains("session-a"),
"{foreign} leaks the session id"
);
assert!(runs(&root, true, "session-b").contains("no runs recorded"));
std::fs::remove_dir_all(&root).ok();
}
#[test]
fn a_run_root_this_build_refuses_is_named_rather_than_dropped() {
let root = scratch("skipped");
write_run(
&root,
"readable",
sys::pid(),
&[event(
crate::journal::PipelineKind::RunStarted,
None,
&[("plan", json!(plan()))],
)],
);
let newer = write_run(&root, "from-a-newer-build", sys::pid(), &[]);
let mut written: serde_json::Value = serde_json::from_str(
&std::fs::read_to_string(newer.launch()).expect("the launch record this build wrote"),
)
.expect("a launch record");
written["channel_id"] = json!("a field a later build removed");
std::fs::write(newer.launch(), written.to_string()).expect("a launch record");
std::fs::create_dir_all(root.join("no-launch")).expect("a directory with no launch");
let empty = RunPaths::under(&root, "not-a-record");
empty.create().expect("the run directory");
std::fs::write(empty.launch(), json!({"oops": true}).to_string())
.expect("a launch record this build cannot read");
let survey = Survey::of(&root);
assert_eq!(survey.views.len(), 2, "{:?}", survey.skipped);
assert_eq!(survey.skipped.len(), 2, "{:?}", survey.skipped);
for rendered in [
runs(&root, false, "session-a"),
status(&survey),
goals(&survey),
] {
assert!(rendered.contains("readable"), "{rendered}");
assert!(rendered.contains("from-a-newer-build"), "{rendered}");
}
for rendered in [
runs(&root, false, "session-a"),
status(&survey),
goals(&survey),
host(&survey),
] {
assert!(rendered.contains("2 run root(s) skipped"), "{rendered}");
assert!(rendered.contains("no-launch"), "{rendered}");
assert!(rendered.contains("launch.json"), "{rendered}");
assert!(rendered.contains("run_id"), "{rendered}");
}
std::fs::remove_dir_all(&root).ok();
}
#[test]
fn a_root_whose_every_run_is_refused_does_not_read_as_no_runs_recorded() {
let root = scratch("all-refused");
std::fs::create_dir_all(root.join("no-launch")).expect("a directory with no launch");
let survey = Survey::of(&root);
assert!(survey.views.is_empty());
for rendered in [
runs(&root, false, "session-a"),
status(&survey),
goals(&survey),
] {
assert!(
!rendered.contains("no runs recorded"),
"a rejected root reported as an absence: {rendered}"
);
assert!(rendered.contains("no run under"), "{rendered}");
assert!(rendered.contains("1 run root(s) skipped"), "{rendered}");
assert!(rendered.contains("no-launch"), "{rendered}");
}
let empty = scratch("all-refused-empty");
assert_eq!(runs(&empty, false, "session-a"), "no runs recorded\n");
std::fs::remove_dir_all(&root).ok();
std::fs::remove_dir_all(&empty).ok();
}
#[test]
fn a_host_row_whose_driver_is_gone_is_counted_rather_than_rendered_live() {
let root = scratch("host-stale");
let paths = write_run(
&root,
"ghosted",
sys::pid(),
&[
event(
crate::journal::PipelineKind::RunStarted,
None,
&[("plan", json!(plan()))],
),
event(
crate::journal::PipelineKind::NodeDispatched,
Some("build"),
&[],
),
],
);
register(
&paths,
&ledger::DispatchRecord {
pid: dead_pid(),
started: "a token from the process that died".into(),
..dispatched_here("build")
},
);
let rendered = host(&Survey::of(&root));
assert!(
!rendered.contains("ghosted "),
"a dispatch nothing is driving was rendered as a live row: {rendered}"
);
assert!(rendered.contains("no live dispatches"), "{rendered}");
assert!(
rendered.contains("1 stale registry entry ignored"),
"{rendered}"
);
assert!(rendered.contains("ghosted/build"), "{rendered}");
assert!(rendered.contains("is gone"), "{rendered}");
assert!(rendered.contains(&root.display().to_string()), "{rendered}");
std::fs::remove_dir_all(&root).ok();
}
#[test]
fn a_host_row_backed_by_a_live_registry_entry_renders_as_a_live_dispatch() {
let root = scratch("host-live");
let paths = write_run(
&root,
"driven",
sys::pid(),
&[
event(
crate::journal::PipelineKind::RunStarted,
None,
&[("plan", json!(plan()))],
),
event(
crate::journal::PipelineKind::NodeDispatched,
Some("build"),
&[],
),
],
);
register(&paths, &dispatched_here("build"));
let rendered = host(&Survey::of(&root));
assert!(rendered.contains("driven"), "{rendered}");
assert!(rendered.contains("build"), "{rendered}");
assert!(!rendered.contains("no live dispatches"), "{rendered}");
assert!(!rendered.contains("stale registry"), "{rendered}");
assert!(!rendered.contains("UNPROVEN"), "{rendered}");
std::fs::remove_dir_all(&root).ok();
}
#[test]
fn a_host_row_this_host_cannot_prove_either_way_says_so_rather_than_reading_live() {
let root = scratch("host-unproven");
let paths = write_run(
&root,
"elsewhere",
sys::pid(),
&[
event(
crate::journal::PipelineKind::RunStarted,
None,
&[("plan", json!(plan()))],
),
event(
crate::journal::PipelineKind::NodeDispatched,
Some("build"),
&[],
),
],
);
register(
&paths,
&ledger::DispatchRecord {
host: "some-other-host".into(),
..dispatched_here("build")
},
);
let rendered = host(&Survey::of(&root));
assert!(rendered.contains("elsewhere"), "{rendered}");
assert!(rendered.contains("UNPROVEN"), "{rendered}");
assert!(rendered.contains("some-other-host"), "{rendered}");
assert!(!rendered.contains("stale registry"), "{rendered}");
register(
&paths,
&ledger::DispatchRecord {
started: String::new(),
..dispatched_here("build")
},
);
let rendered = host(&Survey::of(&root));
assert!(rendered.contains("UNPROVEN"), "{rendered}");
assert!(rendered.contains("no start token"), "{rendered}");
register(
&paths,
&ledger::DispatchRecord {
started: "the process it was recorded in, which was not this one".into(),
..dispatched_here("build")
},
);
let rendered = host(&Survey::of(&root));
assert!(
rendered.contains("1 stale registry entry ignored"),
"{rendered}"
);
assert!(rendered.contains("different process"), "{rendered}");
std::fs::remove_dir_all(paths.dispatches()).expect("the registry is taken away");
std::fs::create_dir_all(paths.dispatches()).expect("an empty registry");
let rendered = host(&Survey::of(&root));
assert!(rendered.contains("UNPROVEN"), "{rendered}");
assert!(rendered.contains("holds no entry for it"), "{rendered}");
std::fs::remove_dir_all(&root).ok();
}
fn advanced(role: Option<&str>, turn: Option<u64>, identity: &str, reason: &str) -> Envelope {
advanced_for("worker", role, turn, identity, reason)
}
fn advanced_for(
member: &str,
role: Option<&str>,
turn: Option<u64>,
identity: &str,
reason: &str,
) -> Envelope {
let mut fields = vec![("identity", json!(identity)), ("reason", json!(reason))];
if let Some(role) = role {
fields.push(("role", json!(role)));
}
if let Some(turn) = turn {
fields.push(("turn", json!(turn)));
}
let mut envelope = relayed(
EventKind("fallback-advanced".into()),
Source::Agentgraph,
Some("build"),
&fields,
);
envelope.stream = "oneagentgraph-1".into();
envelope.labels.extra.insert("member".into(), member.into());
envelope
}
fn invocation(role: oneagentgraph::event::Role, turn: u64, identity: &str) -> Envelope {
invocation_for("worker", role, turn, identity)
}
fn invocation_for(
member: &str,
role: oneagentgraph::event::Role,
turn: u64,
identity: &str,
) -> Envelope {
let session = oneagentgraph::event::OneharnessSession {
role,
turn,
identity: identity.to_string(),
session_id: None,
history_id: "record-1".into(),
history_dir: "/store".into(),
history_project: "project".into(),
history_session: "record-1".into(),
};
let mut envelope = relayed(
EventKind("oneharness-session".into()),
Source::Agentgraph,
Some("build"),
&[],
);
envelope.stream = "oneagentgraph-1".into();
envelope.payload = match serde_json::to_value(&session) {
Ok(serde_json::Value::Object(payload)) => payload,
other => panic!("a session is not an object: {other:?}"),
};
envelope.labels.extra.insert("member".into(), member.into());
envelope
}
#[test]
fn a_dispatch_that_died_says_where_its_work_is_in_every_shape_a_settlement_has() {
let root = scratch("died");
let died = |node: &str, fields: &[(&str, serde_json::Value)]| {
let mut all = vec![
("status", json!("failed")),
("outcome", json!(crate::engine::DISPATCH_DIED)),
];
all.extend(fields.iter().cloned());
event(crate::journal::PipelineKind::NodeSettled, Some(node), &all)
};
let mut plan = plan();
for id in ["branchless", "uncommitted", "unclassified"] {
plan.tasks.push(Node {
id: id.into(),
task: Some("## What\ndo it".into()),
..Node::default()
});
}
write_run(
&root,
"died",
sys::pid(),
&[
event(
crate::journal::PipelineKind::RunStarted,
None,
&[("plan", json!(plan))],
),
died(
"build",
&[
("cause", json!("rate_limit")),
("branch", json!("b/one")),
("head", json!("abc123")),
],
),
died("branchless", &[("cause", json!("spawn-error"))]),
died(
"uncommitted",
&[("cause", json!("auth")), ("branch", json!("b/two"))],
),
died("unclassified", &[("branch", json!("b/three"))]),
],
);
let survey = Survey::of(&root);
let rendered = results(&survey.views[0]);
for said in [
"(rate_limit) rather than failing its task; b/one may carry finished work, at abc123",
"(spawn-error) rather than failing its task; it left no branch",
"(auth) rather than failing its task; b/two may carry finished work",
"the dispatch died rather than failing its task; b/three may carry finished work",
] {
assert!(rendered.contains(said), "{said:?} is not in:{rendered}");
}
let standing = status(&survey);
assert!(
standing.contains("the dispatch died (rate_limit) rather than failing its task"),
"{standing}"
);
}
#[test]
fn a_failed_node_tells_a_recovered_chain_from_one_that_ran_out() {
let root = scratch("refusal");
write_run(
&root,
"refused",
sys::pid(),
&[
event(
crate::journal::PipelineKind::RunStarted,
None,
&[("plan", json!(plan()))],
),
event(
crate::journal::PipelineKind::NodeDispatched,
Some("build"),
&[],
),
advanced(Some("agent"), Some(1), "claude-code", "quota"),
invocation(
oneagentgraph::event::Role::Agent,
1,
"claude-code:alternate",
),
advanced(Some("judge"), Some(1), "codex", "rate_limit"),
advanced(Some("judge"), Some(1), "codex", "rate_limit"),
event(
crate::journal::PipelineKind::NodeSettled,
Some("build"),
&[
("status", json!("failed")),
("outcome", json!("task-failed")),
],
),
],
);
let survey = Survey::of(&root);
let rendered = results(&survey.views[0]);
assert!(
rendered.contains(
"fallback: the agent side fell through 'claude-code' (quota) → served by \
'claude-code:alternate'"
),
"{rendered}"
);
assert!(
rendered.contains(
"provider: the judge side: identity 'codex' refused (rate_limit), recorded 2 times"
),
"{rendered}"
);
assert!(
!rendered.contains("provider: the agent side"),
"a recovered chain was reported as a refusal:\n{rendered}"
);
let rendered = status(&survey);
assert!(
rendered.contains("build: failed — the judge side: identity 'codex' refused"),
"{rendered}"
);
assert!(
rendered.contains("build: fallback — the agent side fell through 'claude-code'"),
"{rendered}"
);
std::fs::remove_dir_all(&root).ok();
}
#[test]
fn an_unattributed_refusal_is_never_given_a_side_it_did_not_carry() {
let advance = |reason: &str| oneagentgraph::event::FallbackAdvanced {
identity: "codex".into(),
reason: reason.into(),
role: None,
turn: None,
};
let single = Refusal {
advanced: advance("auth"),
member: MemberLabel::Named("worker".into()),
records: std::num::NonZeroU64::MIN,
};
assert_eq!(
chain_phrase(&ChainRecord {
refusal: &single,
became: Fallthrough::Unrecorded,
records: std::num::NonZeroU64::MIN,
}),
"member 'worker' fell through 'codex' (auth); nothing this run recorded names what \
served that turn"
);
let bare = Refusal {
advanced: advance(""),
member: MemberLabel::Unstamped,
records: std::num::NonZeroU64::MIN,
};
let phrase = chain_phrase(&ChainRecord {
refusal: &bare,
became: Fallthrough::Refused,
records: std::num::NonZeroU64::MIN,
});
assert!(
phrase.contains("a side the record does not name"),
"{phrase}"
);
assert!(
phrase.contains("for a reason the record does not carry"),
"{phrase}"
);
let unreadable = Refusal {
advanced: advance("auth"),
member: MemberLabel::Unreadable,
records: std::num::NonZeroU64::MIN,
};
let phrase = chain_phrase(&ChainRecord {
refusal: &unreadable,
became: Fallthrough::Served("codex:alternate".into()),
records: std::num::NonZeroU64::new(2).expect("two records"),
});
assert_eq!(
phrase,
"a side this build cannot read fell through 'codex' (auth) → served by \
'codex:alternate', recorded 2 times"
);
let mut nameless = relayed(
EventKind("fallback-advanced".into()),
Source::Agentgraph,
Some("build"),
&[("reason", json!("quota"))],
);
nameless.stream = "oneagentgraph-1".into();
assert!(projection::fold(&[nameless]).refusals.is_empty());
}
#[test]
fn one_chain_that_recovers_and_then_runs_out_says_both() {
let state = projection::fold(&[
advanced(Some("agent"), Some(1), "claude-code", "quota"),
invocation(
oneagentgraph::event::Role::Agent,
1,
"claude-code:alternate",
),
advanced(Some("agent"), Some(2), "claude-code", "quota"),
invocation(
oneagentgraph::event::Role::Agent,
2,
"claude-code:alternate",
),
advanced(Some("agent"), Some(3), "claude-code", "quota"),
]);
let records = chain_records(&state, "build");
let phrases = records.iter().map(chain_phrase).collect::<Vec<_>>();
assert_eq!(
phrases,
vec![
"the agent side fell through 'claude-code' (quota) → served by \
'claude-code:alternate', recorded 2 times"
.to_string(),
"the agent side: identity 'claude-code' refused (quota)".to_string(),
]
);
for crossing in [
invocation(oneagentgraph::event::Role::Agent, 1, "claude-code"),
invocation_for("reviewer", oneagentgraph::event::Role::Judge, 1, "codex-2"),
] {
let crossed =
projection::fold(&[advanced(Some("judge"), Some(1), "codex", "quota"), crossing]);
assert_eq!(
chain_records(&crossed, "build")
.iter()
.map(chain_phrase)
.collect::<Vec<_>>(),
vec!["the judge side: identity 'codex' refused (quota)".to_string()]
);
}
let paired = projection::fold(&[
advanced_for("reviewer", Some("judge"), Some(1), "codex", "quota"),
invocation_for("reviewer", oneagentgraph::event::Role::Judge, 1, "codex-2"),
]);
assert_eq!(
chain_records(&paired, "build")
.iter()
.map(chain_phrase)
.collect::<Vec<_>>(),
vec!["the judge side fell through 'codex' (quota) → served by 'codex-2'".to_string()]
);
}
#[test]
fn a_verdict_that_failed_a_node_names_its_criterion_and_its_reason() {
let root = scratch("verdict");
let settled = |verdicts: serde_json::Value| {
let mut envelope = relayed(
EventKind("member-settled".into()),
Source::Agentgraph,
Some("build"),
&[("completed", json!(false)), ("verdict", verdicts)],
);
envelope.stream = "oneagentgraph-1".into();
envelope
};
write_run(
&root,
"verdict",
sys::pid(),
&[
event(
crate::journal::PipelineKind::RunStarted,
None,
&[("plan", json!(plan()))],
),
event(
crate::journal::PipelineKind::NodeDispatched,
Some("build"),
&[],
),
settled(json!([
{"criterion": "the branch is pushed", "kind": "boolean",
"verdict": {"value": true, "reason": "it is"}},
{"criterion": "the change builds", "kind": "boolean",
"verdict": {"value": false, "reason": "cargo build fails in src/views.rs"}},
{"criterion": "how readable it is", "kind": "numeric",
"verdict": {"value": 2.0, "reason": "dense"}},
{"criterion": "", "kind": "boolean", "verdict": {"value": false}},
{"criterion": "the tests pass",
"verdict": {"value": false, "reason": "the suite is red"}},
])),
event(
crate::journal::PipelineKind::NodeSettled,
Some("build"),
&[
("status", json!("failed")),
("outcome", json!("task-failed")),
],
),
],
);
let rendered = results(&Survey::of(&root).views[0]);
assert!(
rendered.contains(
"verdict: 'the change builds' failed — cargo build fails in src/views.rs"
),
"{rendered}"
);
assert!(
rendered.contains(
"verdict: a criterion the record does not name failed — the record carries no \
reason"
),
"{rendered}"
);
for absent in [
"the branch is pushed",
"how readable it is",
"the suite is red",
] {
assert!(
!rendered.contains(absent),
"a verdict that failed nothing, or that this build cannot read, was named as \
the failure:\n{rendered}"
);
}
std::fs::remove_dir_all(&root).ok();
}
#[test]
fn only_a_node_a_judge_rejected_is_named_as_one() {
let root = scratch("rejected-advice");
let failing = |node: &str, verdicts: Option<serde_json::Value>| {
let mut events = vec![event(
crate::journal::PipelineKind::NodeDispatched,
Some(node),
&[],
)];
if let Some(verdicts) = verdicts {
let mut settled = relayed(
EventKind("member-settled".into()),
Source::Agentgraph,
Some(node),
&[("completed", json!(false)), ("verdict", verdicts)],
);
settled.stream = "oneagentgraph-1".into();
events.push(settled);
}
events.push(event(
crate::journal::PipelineKind::NodeSettled,
Some(node),
&[
("status", json!("failed")),
("outcome", json!(crate::engine::TASK_FAILED)),
],
));
events
};
let rejection = json!([
{"criterion": "the change builds", "kind": "boolean",
"verdict": {"value": false, "reason": "cargo build fails"}},
]);
let held_up = Plan {
tasks: vec![
Node {
id: "build".into(),
..Node::default()
},
Node {
id: "later".into(),
deps: vec!["build".into()],
..Node::default()
},
],
..plan()
};
for (run, verdicts) in [("judged", Some(rejection)), ("brokeoff", None)] {
let mut events = vec![event(
crate::journal::PipelineKind::RunStarted,
None,
&[("plan", json!(held_up))],
)];
events.extend(failing("build", verdicts));
write_run(&root, run, dead_pid(), &events);
}
let listing = runs(&root, false, "session-a");
let status_of = |run: &str| {
let paths = RunPaths::under(&root, run);
status(&Survey {
root: root.clone(),
views: vec![RunView::open(&paths).expect("the run reads back")],
skipped: Vec::new(),
})
};
let judged = status_of("judged");
for rendered in [&listing, &judged] {
assert!(
rendered.contains("build, whose work a judge rejected"),
"the rejected node is not named as one a judge rejected:\n{rendered}"
);
assert!(
rendered.contains("onepipeline results judged"),
"the verdict a planner has to read is not named:\n{rendered}"
);
assert!(
rendered.contains("superseding the node"),
"the step that moves the run is not named:\n{rendered}"
);
}
assert!(
!judged.contains("adopt"),
"a driver is prescribed for a frontier it cannot move:\n{judged}"
);
let broke = status_of("brokeoff");
assert!(
!broke.contains("a judge rejected"),
"a node that failed its own task was reported as judged:\n{broke}"
);
assert!(
!listing.contains("onepipeline results brokeoff"),
"a run nothing judged was given the judgement's advice:\n{listing}"
);
std::fs::remove_dir_all(&root).ok();
}
#[test]
fn a_relayed_value_never_carries_a_control_character_onto_a_line() {
let refusal = Refusal {
advanced: oneagentgraph::event::FallbackAdvanced {
identity: "codex".into(),
reason: "quota".into(),
role: Some(oneagentgraph::event::Role::Agent),
turn: Some(1),
},
member: MemberLabel::Named("worker".into()),
records: std::num::NonZeroU64::MIN,
};
let phrase = chain_phrase(&ChainRecord {
refusal: &refusal,
became: Fallthrough::Served("codex\r\nprovider: forged".into()),
records: std::num::NonZeroU64::MIN,
});
assert!(!phrase.contains('\n') && !phrase.contains('\r'), "{phrase}");
let phrase = verdict_phrase(&crate::report::FailedVerdict {
criterion: Some("it builds".into()),
reason: Some("no\nit does not".into()),
});
assert!(!phrase.contains('\n'), "{phrase}");
}
#[test]
fn every_view_renders_from_the_merged_stream() {
let root = scratch("render");
let mut agent = relayed(
EventKind("turn-finished".into()),
Source::Agentgraph,
Some("build"),
&[("message", json!("ran the gate"))],
);
agent.stream = "oneagentgraph-1".into();
let mut vcs = relayed(
EventKind("session-opened".into()),
Source::Vcs,
Some("build"),
&[("branch", json!("feature"))],
);
vcs.stream = "onevcs-tok".into();
write_run(
&root,
"demo",
sys::pid(),
&[
event(
crate::journal::PipelineKind::RunStarted,
None,
&[("plan", json!(plan()))],
),
event(crate::journal::PipelineKind::NodeReady, Some("build"), &[]),
event(
crate::journal::PipelineKind::NodeDispatched,
Some("build"),
&[],
),
agent,
vcs,
],
);
let view = RunView::open(&RunPaths::under(&root, "demo")).expect("the run reads");
let stream = monitor(&view, &EventFilter::default());
assert!(stream.starts_with("Concise graph events;"), "{stream}");
assert!(stream.contains("agent:oneagentgraph-1"), "{stream}");
assert!(stream.contains("vcs:onevcs-tok"), "{stream}");
assert!(stream.contains("graph:build"), "{stream}");
assert!(stream.contains("-- demo 0/1 done"), "{stream}");
assert!(
!stream.contains("round"),
"a round reached a view: {stream}"
);
let paths = RunPaths::under(&root, "demo");
register(&paths, &dispatched_here("build"));
let survey = Survey::of(&root);
assert!(status(&survey).contains("build: running"));
assert!(host(&survey).contains("build"));
assert!(goals(&survey).contains("close the coverage gap"));
assert!(results(&survey.views[0]).contains("build"));
std::fs::remove_dir_all(&root).ok();
}
#[test]
fn a_node_the_ledger_calls_running_that_nothing_drives_is_undriven() {
let root = scratch("undriven");
write_run(
&root,
"demo",
sys::pid(),
&[
event(
crate::journal::PipelineKind::RunStarted,
None,
&[("plan", json!(plan()))],
),
event(
crate::journal::PipelineKind::NodeDispatched,
Some("build"),
&[],
),
],
);
let rendered = status(&Survey::of(&root));
assert!(rendered.contains("UNDRIVEN"), "{rendered}");
std::fs::remove_dir_all(&root).ok();
}
#[test]
fn a_live_dispatch_reports_what_it_is_doing_now_with_a_count_and_an_age() {
let root = scratch("activity");
let mut turn = relayed(
EventKind("turn-activity".into()),
Source::Agentgraph,
Some("build"),
&[
("kind", json!("tool_call")),
("name", json!("Bash")),
("detail", json!("cargo llvm-cov --workspace")),
],
);
turn.stream = "oneagentgraph-1".into();
write_run(
&root,
"demo",
sys::pid(),
&[
event(
crate::journal::PipelineKind::RunStarted,
None,
&[("plan", json!(plan()))],
),
event(
crate::journal::PipelineKind::NodeDispatched,
Some("build"),
&[],
),
turn,
],
);
let rendered = status(&Survey::of(&root));
assert!(
rendered.contains("now Bash cargo llvm-cov --workspace"),
"{rendered}"
);
assert!(rendered.contains("1 event(s)"), "{rendered}");
assert!(rendered.contains("ago"), "{rendered}");
assert!(
!rendered.contains(DriverLiveness::Undriven.as_str()),
"a dispatch that is recording was reported as driving nothing: {rendered}"
);
std::fs::remove_dir_all(&root).ok();
}
fn recorded(events: u64, last_at: u64) -> Option<crate::projection::Progress> {
(0..events).fold(None, |progress, _| match progress {
None => crate::projection::Progress::first(Some(last_at)),
Some(progress) => Some(progress.and(Some(last_at))),
})
}
#[test]
fn a_dispatch_that_has_named_no_tool_reports_its_count_rather_than_a_guess() {
let rendered = working(&crate::projection::NodeActivity {
doing: None,
progress: recorded(3, sys::now_millis()),
last_heartbeat_at: None,
});
assert_eq!(rendered, "3 event(s), 0s ago");
assert!(!rendered.contains("now"), "{rendered}");
}
#[test]
fn a_heartbeat_is_reported_beside_the_age_of_the_work_rather_than_as_work() {
let now = sys::now_millis();
let rendered = working(&crate::projection::NodeActivity {
doing: Some("Bash red-green.sh".into()),
progress: recorded(4, now - 600_000),
last_heartbeat_at: Some(now),
});
assert!(
rendered.contains("4 event(s), 10m00s ago"),
"the age of the work was taken from the heartbeat: {rendered}"
);
assert!(
rendered.contains("alive 0s ago"),
"a dispatch that is alive and doing nothing is not reported as alive: {rendered}"
);
}
#[test]
fn a_dispatch_that_has_only_heartbeated_reports_no_work_and_still_reads_as_alive() {
let rendered = working(&crate::projection::NodeActivity {
doing: None,
progress: None,
last_heartbeat_at: Some(sys::now_millis()),
});
assert_eq!(rendered, "nothing recorded yet; alive 0s ago");
}
#[test]
fn a_transcript_renders_the_turns_tools_and_the_report_it_settled_with() {
let root = scratch("transcript");
let paths = RunPaths::under(&root, "demo");
let stored = paths.report_for("s", 0);
std::fs::create_dir_all(paths.reports_dir()).expect("the run's report storage");
std::fs::write(
&stored,
json!({
"schema_version": 7,
"transcript": {"messages": [
{"role": "assistant", "content": "Ran the gate.\nIt passed.", "events": [
{"kind": "tool_call", "name": "bash", "input": {"command": "just check"}},
]},
]},
})
.to_string(),
)
.expect("a stored report");
let mut started = relayed(
EventKind("turn-started".into()),
Source::Agentgraph,
Some("build"),
&[("turn", json!(1))],
);
started.stream = "oneagentgraph-1".into();
let mut activity = relayed(
EventKind("turn-activity".into()),
Source::Agentgraph,
Some("build"),
&[
("kind", json!("tool_call")),
("name", json!("bash")),
("detail", json!("just check")),
],
);
activity.stream = "oneagentgraph-1".into();
let settled = relayed(
EventKind(crate::report::MEMBER_SETTLED.into()),
Source::Agentgraph,
Some("build"),
&[(crate::report::REPORT_PATH, json!("/elsewhere/report.json"))],
);
write_run(
&root,
"demo",
sys::pid(),
&[
event(
crate::journal::PipelineKind::RunStarted,
None,
&[("plan", json!(plan()))],
),
started,
activity,
settled,
],
);
let view = RunView::open(&RunPaths::under(&root, "demo")).expect("the run reads");
let rendered = transcript(&view, None);
assert!(rendered.contains("demo build"), "{rendered}");
assert!(rendered.contains("turn 1"), "{rendered}");
assert!(
rendered.contains("tool_call bash just check"),
"{rendered}"
);
assert!(rendered.contains("assistant"), "{rendered}");
assert!(rendered.contains("Ran the gate."), "{rendered}");
assert!(rendered.contains("It passed."), "{rendered}");
assert!(transcript(&view, Some("elsewhere")).contains("no dispatch"));
std::fs::remove_dir_all(&root).ok();
}
const RECORDED_ACTIVITY: &str = include_str!("../tests/recorded/turn-activity.jsonl");
const RECORDED_SETTLEMENT: &str = include_str!("../tests/recorded/member-settled.json");
const RECORDED_REPORT: &str = include_str!("../tests/recorded/settled-report.json");
#[test]
fn a_recorded_tool_results_own_output_is_what_the_transcript_renders() {
let root = scratch("transcript-recorded-journal");
let mut events = vec![event(
crate::journal::PipelineKind::RunStarted,
None,
&[("plan", json!(plan()))],
)];
events.extend(RECORDED_ACTIVITY.lines().map(|line| {
serde_json::from_str::<Envelope>(line).expect("a recorded envelope reads back")
}));
write_run(&root, "demo", sys::pid(), &events);
let view = RunView::open(&RunPaths::under(&root, "demo")).expect("the run reads");
let rendered = transcript(&view, None);
assert!(
rendered.contains("tool_call Bash gh issue view 28"),
"{rendered}"
);
assert!(
rendered.contains("**Accepted fix.** Four parts:"),
"the recorded output is not on the line that answered for it:\n{rendered}"
);
assert!(
rendered.contains("the gate then passed without it."),
"the recorded output was cut before its end:\n{rendered}"
);
assert!(
rendered.contains("… [already cut short by the producer]"),
"an output the producer marked truncated is rendered as a whole \
one:\n{rendered}"
);
assert!(
!rendered.lines().any(|line| line.trim() == "tool_result"),
"an observation still renders as an empty column:\n{rendered}"
);
std::fs::remove_dir_all(&root).ok();
}
#[test]
fn a_recorded_reports_tool_output_is_rendered_and_bounded() {
let root = scratch("transcript-recorded-report");
let settled: Envelope =
serde_json::from_str(RECORDED_SETTLEMENT.trim()).expect("a recorded settlement");
let paths = RunPaths::under(&root, "demo");
paths.create().expect("the run directory");
std::fs::create_dir_all(paths.reports_dir()).expect("the run's report storage");
std::fs::write(
paths.report_for(&settled.stream, settled.seq),
RECORDED_REPORT,
)
.expect("this run's own copy of the report");
write_run(
&root,
"demo",
sys::pid(),
&[
event(
crate::journal::PipelineKind::RunStarted,
None,
&[("plan", json!(plan()))],
),
settled,
],
);
let view = RunView::open(&RunPaths::under(&root, "demo")).expect("the run reads");
let rendered = transcript(&view, None);
assert!(
rendered.contains("tool_result ///"),
"the report's first observation renders as an empty column:\n{rendered}"
);
assert!(
rendered.contains("// the second, because every family below it is the"),
"the report's longest observation is not rendered at all:\n{rendered}"
);
assert!(
rendered.contains("… [4096 of 4350 characters]"),
"an unbounded output was printed whole, or cut without saying \
so:\n{rendered}"
);
assert!(
!rendered.contains("Verdict::Reclaim(lease) => reclaim(&mut report"),
"the ceiling printed the tail of an output past it:\n{rendered}"
);
std::fs::remove_dir_all(&root).ok();
}
const RECORDED_WITHOUT_OUTPUT: &str =
include_str!("../tests/recorded/turn-activity-no-output.json");
#[test]
fn a_recorded_result_carrying_no_output_renders_empty_because_it_is_empty() {
let root = scratch("transcript-recorded-outputless");
let recorded: Envelope =
serde_json::from_str(RECORDED_WITHOUT_OUTPUT.trim()).expect("a recorded envelope");
assert_eq!(recorded.payload.get("output"), None, "{recorded:?}");
assert_eq!(
recorded
.payload
.get("detail")
.and_then(serde_json::Value::as_str),
Some(""),
"{recorded:?}"
);
write_run(
&root,
"demo",
sys::pid(),
&[
event(
crate::journal::PipelineKind::RunStarted,
None,
&[("plan", json!(plan()))],
),
recorded,
],
);
let view = RunView::open(&RunPaths::under(&root, "demo")).expect("the run reads");
let rendered = transcript(&view, None);
assert!(
rendered.lines().any(|line| line.trim() == "tool_result"),
"{rendered}"
);
assert!(!rendered.contains('…'), "{rendered}");
std::fs::remove_dir_all(&root).ok();
}
#[test]
fn a_control_character_in_an_output_is_stripped_like_every_other_value() {
let rendered = tool_text(&ToolText::Returned {
output: "first\r\nsecond\u{1b}[2K".to_string(),
truncated: Truncation::Whole,
});
assert_eq!(rendered, "first second [2K");
}
#[test]
fn a_truncation_flag_this_build_cannot_read_is_said_rather_than_assumed() {
let text = |flag: Option<serde_json::Value>| {
tool_text(&ToolText::Returned {
output: "what it returned".to_string(),
truncated: Truncation::of(flag.as_ref()),
})
};
assert_eq!(text(None), "what it returned");
assert_eq!(text(Some(json!(false))), "what it returned");
assert_eq!(text(Some(json!(null))), "what it returned");
assert_eq!(
text(Some(json!(true))),
"what it returned … [already cut short by the producer]"
);
for unreadable in [json!("true"), json!(1), json!({"cut": true})] {
assert_eq!(
text(Some(unreadable.clone())),
"what it returned … [the producer's truncation flag is unreadable]",
"{unreadable}"
);
}
}
#[test]
fn a_report_this_run_did_not_keep_is_named_as_unretained_and_never_opened() {
let root = scratch("transcript-unread");
let settled = relayed(
EventKind(crate::report::MEMBER_SETTLED.into()),
Source::Agentgraph,
Some("build"),
&[(
crate::report::REPORT_PATH,
json!("/nowhere/onepipeline/report.json"),
)],
);
write_run(
&root,
"demo",
sys::pid(),
&[
event(
crate::journal::PipelineKind::RunStarted,
None,
&[("plan", json!(plan()))],
),
settled,
],
);
let view = RunView::open(&RunPaths::under(&root, "demo")).expect("the run reads");
let rendered = transcript(&view, None);
assert!(rendered.contains("not retained by this run"), "{rendered}");
assert!(
rendered.contains("/nowhere/onepipeline/report.json"),
"the path that was not read is not named: {rendered}"
);
std::fs::remove_dir_all(&root).ok();
}
#[test]
fn a_report_without_a_transcript_says_so() {
let root = scratch("transcript-none");
let paths = RunPaths::under(&root, "demo");
std::fs::create_dir_all(paths.reports_dir()).expect("the run's report storage");
std::fs::write(
paths.report_for("s", 0),
json!({"usage": {"input_tokens": 1}}).to_string(),
)
.expect("a stored report");
let settled = relayed(
EventKind(crate::report::MEMBER_SETTLED.into()),
Source::Agentgraph,
Some("build"),
&[(crate::report::REPORT_PATH, json!("/elsewhere/report.json"))],
);
write_run(
&root,
"demo",
sys::pid(),
&[
event(
crate::journal::PipelineKind::RunStarted,
None,
&[("plan", json!(plan()))],
),
settled,
],
);
let view = RunView::open(&RunPaths::under(&root, "demo")).expect("the run reads");
assert!(transcript(&view, None).contains("carries no transcript"));
std::fs::remove_dir_all(&root).ok();
}
#[test]
fn a_run_that_dispatched_nothing_has_no_transcript_to_render() {
let root = scratch("transcript-empty");
write_run(
&root,
"demo",
sys::pid(),
&[event(
crate::journal::PipelineKind::RunStarted,
None,
&[("plan", json!(plan()))],
)],
);
let view = RunView::open(&RunPaths::under(&root, "demo")).expect("the run reads");
assert_eq!(
transcript(&view, None),
"no dispatch has recorded a transcript\n"
);
std::fs::remove_dir_all(&root).ok();
}
#[test]
fn a_waiting_human_reports_its_action_and_what_it_unblocks() {
let root = scratch("waiting");
let mut waiting_plan = plan();
waiting_plan.tasks = vec![
Node {
id: "approve".into(),
kind: crate::plan::NodeKind::Human,
task: Some("approve the release".into()),
..Node::default()
},
Node {
id: "ship".into(),
persona: Some("engineer".into()),
task: Some("## What\nship".into()),
deps: vec!["approve".into()],
..Node::default()
},
];
write_run(
&root,
"demo",
sys::pid(),
&[
event(
crate::journal::PipelineKind::RunStarted,
None,
&[("plan", json!(waiting_plan))],
),
event(
crate::journal::PipelineKind::NodeSettled,
Some("approve"),
&[("status", json!("waiting"))],
),
],
);
let view = RunView::open(&RunPaths::under(&root, "demo")).expect("the run reads");
let rendered = results(&view);
assert!(rendered.contains("approve the release"), "{rendered}");
assert!(rendered.contains("unblocks: ship"), "{rendered}");
assert!(
rendered.contains("ship") && rendered.contains("blocked"),
"{rendered}"
);
std::fs::remove_dir_all(&root).ok();
}
#[test]
fn both_views_render_the_amendment_a_node_is_currently_judged_against() {
let root = scratch("amendment");
let mut amended = plan();
amended.tasks[0].amendment =
Some("The four comment lines are out of scope: leave them.".into());
write_run(
&root,
"demo",
sys::pid(),
&[event(
crate::journal::PipelineKind::RunStarted,
None,
&[("plan", json!(amended))],
)],
);
let paths = RunPaths::under(&root, "demo");
let view = RunView::open(&paths).expect("the run reads");
let survey = Survey::of(&root);
for (which, rendered) in [("status", status(&survey)), ("results", results(&view))] {
assert!(
rendered.contains("The four comment lines are out of scope: leave them."),
"`{which}` does not say what `build` is judged against:\n{rendered}"
);
assert!(
rendered.contains("amend"),
"`{which}` renders the text without naming it an amendment:\n{rendered}"
);
}
let plain = scratch("amendment-none");
write_run(
&plain,
"demo",
sys::pid(),
&[event(
crate::journal::PipelineKind::RunStarted,
None,
&[("plan", json!(plan()))],
)],
);
let view = RunView::open(&RunPaths::under(&plain, "demo")).expect("the run reads");
assert!(!results(&view).contains("amendment:"), "{}", results(&view));
std::fs::remove_dir_all(&root).ok();
std::fs::remove_dir_all(&plain).ok();
}
#[test]
fn a_summary_line_is_capped_and_control_stripped() {
let long = "x".repeat(500);
let stripped = summarize(&relayed(
EventKind("kind".into()),
Source::Agentgraph,
None,
&[("message", json!(format!("a\nb{long}")))],
));
assert!(!stripped.contains('\n'), "{stripped}");
assert_eq!(stripped.chars().count(), 96);
}
#[test]
fn the_parked_threshold_is_read_from_the_environment_or_defaults() {
assert!(parked_after_seconds() > 0);
}
#[test]
fn every_liveness_verdict_has_the_word_the_contract_fixes() {
assert_eq!(DriverLiveness::Driving.as_str(), "ACTIVE");
assert_eq!(DriverLiveness::DriverDead.as_str(), "DRIVER DEAD");
assert_eq!(DriverLiveness::Parked.as_str(), "PARKED");
assert_eq!(DriverLiveness::Undriven.as_str(), "UNDRIVEN");
}
}