use std::io::{BufRead, Write};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::{Duration, Instant};
use serde_json::json;
use crate::agentgraph;
use crate::channel::{Author, ChannelState, Command, Reply, Surface, SurfaceKind};
use crate::cli::{
AttestArgs, ChannelCommand, Cli, OptionalRunArgs, ReadArgs, ReplyArgs, RunArgs, RunsArgs,
StartArgs, StopArgs, SurfaceArgs, TelemetryArgs, TranscriptArgs, DAG_GRAPH_OFF,
};
use crate::concurrency::{self, Liveness, State};
use crate::edits;
use crate::engine;
use crate::error::{Error, Result, EXIT_NOTHING_DRIVING, EXIT_QUEUED, EXIT_SUCCESS};
use crate::filter::{self, EventFilter};
use crate::graph::{self, GraphState};
use crate::journal::{self, Journal};
use crate::ledger::{self, LaunchRecord, RunPaths};
use crate::plan::Plan;
use crate::sys;
use crate::telemetry;
use crate::views::{self, RunView};
const ATTACH_POLL: Duration = Duration::from_millis(50);
const DRAIN_GRACE: Duration = Duration::from_secs(2);
const DRIVER_HANDOVER: Duration = Duration::from_secs(30);
const TEARDOWN_PATIENCE: Duration = Duration::from_secs(5);
const DRIVER_LOG_LINES: usize = 8;
pub fn dispatch(cli: Cli) -> Result<i32> {
use crate::cli::Command as Verb;
match cli.command {
Verb::Start(args) => start(&args),
Verb::Adopt(args) => adopt(&args),
Verb::DriveRun(args) => drive_run(&args),
Verb::Channel(ChannelCommand::Serve(args)) => serve(&args),
Verb::Next(args) => next(&args),
Verb::Reply(args) => reply(&args),
Verb::Surface(args) => surface(&args),
Verb::Attest(args) => attest(&args),
Verb::Stop(args) => stop(&args),
Verb::Runs(args) => runs(&args),
Verb::Status(args) => report(&args, views::status),
Verb::Host => report(&OptionalRunArgs { run: None }, views::host),
Verb::Monitor(args) => {
let view = RunView::open(&resolve(&args.run)?)?;
let filter = read_filter(&view, &args)?;
print!("{}", views::monitor(&view, &filter));
Ok(EXIT_SUCCESS)
}
Verb::Results(args) => {
print!("{}", views::results(&RunView::open(&resolve(&args.run)?)?));
Ok(EXIT_SUCCESS)
}
Verb::Goals(args) => report(&args, views::goals),
Verb::Transcript(args) => transcript(&args),
Verb::Telemetry(args) => report_telemetry(&args),
Verb::Drive(args) => agentgraph::drive(
&args.graph,
&args.task,
&args.dir,
&args.labels,
&args.sets,
args.event_filter.as_deref(),
),
}
}
fn resolve(run: &str) -> Result<RunPaths> {
if !ledger::is_valid_run_id(run) {
return Err(Error::Invalid(format!(
"'{run}' is not a run id: a run id names one directory under the runs root, \
so it may not be a path"
)));
}
let paths = RunPaths::new(run);
if !paths.exists() {
return Err(Error::NoSuchRun {
run: run.to_string(),
root: ledger::runs_root(),
});
}
Ok(paths)
}
fn launch_dir() -> Result<PathBuf> {
std::env::current_dir()
.map_err(|error| Error::Invalid(format!("cannot read the launch directory: {error}")))
}
fn recorded_dir(record: &LaunchRecord) -> Result<PathBuf> {
if record.dir.as_os_str().is_empty() {
return launch_dir();
}
if !record.dir.is_absolute() {
return Err(Error::Invalid(format!(
"run '{}' records the relative working directory '{}'; a run's directory has to be \
absolute, because the process that resolves it is not the one that launched it",
record.run_id,
record.dir.display()
)));
}
if !record.dir.is_dir() {
return Err(Error::Invalid(format!(
"run '{}' records the working directory '{}', which is not a directory on {}",
record.run_id,
record.dir.display(),
sys::hostname()
)));
}
Ok(record.dir.clone())
}
fn resolve_graph(reference: &str, base: &Path) -> Result<String> {
if reference.starts_with("https://") || Path::new(reference).is_absolute() {
return Ok(reference.to_string());
}
let resolved = base.join(reference);
std::fs::File::open(&resolved).map_err(|error| {
Error::Invalid(format!(
"cannot read graph '{}' resolved against launch directory '{}': {error}",
reference,
base.display()
))
})?;
Ok(resolved.to_string_lossy().into_owned())
}
fn resolve_plan_graphs(plan: &mut Plan, base: &Path) -> Result<()> {
for node in &mut plan.tasks {
if let Some(reference) = &mut node.agent_graph {
reference.0 = resolve_graph(&reference.0, base)?;
}
if let Some(steps) = &mut node.steps {
for step in steps {
if let Some(reference) = &mut step.agent_graph {
reference.0 = resolve_graph(&reference.0, base)?;
}
}
}
}
Ok(())
}
fn mint_run_id(plan: &Plan, path: &Path, root: &Path) -> String {
let base = plan
.name
.clone()
.or_else(|| {
path.file_stem()
.and_then(|stem| stem.to_str())
.map(|stem| stem.trim_end_matches(".plan").to_string())
})
.filter(|name| !name.is_empty())
.unwrap_or_else(|| "run".to_string());
let base: String = base
.chars()
.map(|c| {
if c.is_alphanumeric() || c == '-' || c == '_' {
c
} else {
'-'
}
})
.collect();
if !root.join(&base).exists() {
return base;
}
(2..)
.map(|n| format!("{base}-{n}"))
.find(|candidate| !root.join(candidate).exists())
.unwrap_or(base)
}
fn declared_filters(
config: crate::filter::Filters,
args: &StartArgs,
) -> Result<crate::filter::Filters> {
let mut filters = config;
for declaration in &args.filter_profiles {
let (name, spec) = declaration.split_once('=').ok_or_else(|| {
Error::Invalid(format!(
"--filter-profile takes NAME=SPEC; '{declaration}' names no profile"
))
})?;
if name.trim().is_empty() {
return Err(Error::Invalid(format!(
"--filter-profile takes NAME=SPEC; '{declaration}' has an empty name"
)));
}
filters
.profiles
.insert(name.to_string(), EventFilter::read(spec)?);
}
if let Some(spec) = args.filter_agentgraph.as_deref() {
filters.agentgraph = Some(EventFilter::read(spec)?);
}
if let Some(spec) = args.filter_vcs.as_deref() {
filters.vcs = Some(EventFilter::read(spec)?);
}
Ok(filters)
}
fn read_filter(view: &RunView, args: &ReadArgs) -> Result<EventFilter> {
if args.all {
return Ok(EventFilter::default());
}
let named = args.filter.as_deref().unwrap_or(filter::DEFAULT_PROFILE);
if named.trim_start().starts_with('{') {
return EventFilter::read(named);
}
match view.launch.filters.profile(named) {
Ok(filter) => Ok(filter),
Err(unknown) => match Path::new(named).is_file() {
true => EventFilter::read(named),
false => Err(unknown),
},
}
}
fn start(args: &StartArgs) -> Result<i32> {
let mut plan = Plan::load(&args.plan)?;
graph::validate(&plan)?;
let launch_dir = launch_dir()?;
let declared = match &args.launch_config {
Some(path) => crate::filter::LaunchConfig::load(path)?,
None => crate::filter::LaunchConfig::default(),
};
let graph_ref: Option<String> = match args.dag_graph.as_str() {
DAG_GRAPH_OFF => None,
reference => Some(resolve_graph(reference, &launch_dir)?),
};
let pr_author_graph_ref: Option<String> = match args
.pr_author_graph
.as_deref()
.or(declared.pr_author_graph.as_deref())
{
Some(reference) => Some(resolve_graph(reference, &launch_dir)?),
None => None,
}; let node_graph_ref = resolve_graph(&engine::configured_node_graph(), &launch_dir)?;
resolve_plan_graphs(&mut plan, &launch_dir)?;
let filters = declared_filters(declared.filters, args)?;
let root = ledger::runs_root();
let run = mint_run_id(&plan, &args.plan, &root);
let holders = concurrency::holders(&plan)?;
for holder in holders
.iter()
.filter(|holder| holder.state == State::Open && holder.liveness == Liveness::Stale)
{
eprintln!(
"onepipeline: stale repository holder: identity '{}' session '{}' owner_pid {}; proceeding",
holder.identity, holder.token.0, holder.owner_pid
);
}
let live: Vec<_> = holders
.iter()
.filter(|holder| holder.state == State::Open && holder.liveness == Liveness::Live)
.collect();
if !live.is_empty() && !args.acknowledge_concurrent {
let shared = live
.iter()
.map(|holder| {
format!(
"identity '{}' held by session '{}' (owner_pid {})",
holder.identity, holder.token.0, holder.owner_pid
)
})
.collect::<Vec<_>>()
.join(", ");
return Err(Error::Refused(format!(
"concurrent project work refused for run '{run}': {shared}; pass --acknowledge-concurrent to proceed deliberately"
)));
}
if !live.is_empty() {
let shared = live
.iter()
.map(|holder| {
format!(
"'{}' with session '{}' (owner_pid {})",
holder.identity, holder.token.0, holder.owner_pid
)
})
.collect::<Vec<_>>()
.join(", ");
eprintln!(
"onepipeline: --acknowledge-concurrent: launch '{run}' is proceeding alongside live run(s): {shared}"
);
}
let paths = RunPaths::under(&root, &run);
paths.create()?;
ledger::write_json(&paths.plan(), &plan)?;
let mut record = LaunchRecord {
run_id: run.clone(),
plan: args.plan.clone(),
dir: launch_dir.clone(),
graph: graph_ref.clone().unwrap_or_default(),
graph_run: String::new(),
node_graph: node_graph_ref,
pr_author_graph: pr_author_graph_ref.unwrap_or_default(),
launcher: sys::launcher(),
session: sys::launching_session(),
pid: 0,
host: String::new(),
started: String::new(),
started_at: sys::now_rfc3339(),
heartbeat_interval: args.heartbeat_interval,
dag_sets: args.dag_sets.clone(),
node_sets: args.node_sets.clone(),
adoptions: 0,
filters,
};
record.driven_by_this_process();
let mut open = Journal::open(&paths);
if !live.is_empty() {
open.emit(
journal::PipelineKind::ConcurrentAcknowledged,
journal::labels(&run, None),
journal::payload(&[
(
"shared_identities",
json!(live
.iter()
.map(|holder| holder.identity.to_string())
.collect::<Vec<_>>()),
),
(
"runs",
json!({
"launching": run,
"holding_sessions": live
.iter()
.map(|holder| holder.token.0.clone())
.collect::<Vec<_>>(),
}),
),
(
"holders",
json!(live
.iter()
.map(|holder| json!({
"session": holder.token.0.clone(),
"owner_pid": holder.owner_pid,
}))
.collect::<Vec<_>>()),
),
]),
)?;
}
open.emit(
journal::PipelineKind::RunStarted,
journal::labels(&run, None),
journal::payload(&[
("plan", json!(plan)),
("graph", json!(graph_ref)),
("dir", json!(launch_dir)),
("heartbeat_interval", json!(args.heartbeat_interval)),
]),
)?;
ledger::write_json(&paths.launch(), &record)?;
if args.detach {
sys::disown_standard_handles();
}
if args.detach {
let mut driver = retain_driver(&paths)?;
let pid = driver.id();
confirm_driving(&paths, &mut driver)?;
println!(
"{}",
json!({
"run_id": run,
"pid": pid,
"commands": {
"next": format!("onepipeline next {run}"),
"monitor": format!("onepipeline monitor {run}"),
},
})
);
return Ok(EXIT_SUCCESS);
}
let goal = plan.goal.as_ref().map(|goal| goal.text.as_str());
let lock = engine::claim(&paths)?;
let mut observer = observe(&paths, &mut record, goal, agentgraph::GraphOutput::Relayed)?;
ledger::write_json(&paths.launch(), &record)?;
attach(&paths, observer.as_mut(), lock)
}
fn observe(
paths: &RunPaths,
record: &mut LaunchRecord,
goal: Option<&str>,
output: agentgraph::GraphOutput<'_>,
) -> Result<Option<agentgraph::GraphRun>> {
if record.observer_graph().is_none() {
return Ok(None);
}
let launched = launch_graph(paths, record, goal, output)?;
record.graph_run = launched
.run_id()
.map(ToString::to_string)
.unwrap_or_default();
Ok(Some(launched))
}
fn confirm_driving(paths: &RunPaths, driver: &mut std::process::Child) -> Result<()> {
let pid = driver.id();
let deadline = Instant::now() + DRIVER_HANDOVER;
while Instant::now() < deadline {
let recorded: Option<LaunchRecord> = ledger::read_json_opt(&paths.launch());
if recorded.is_some_and(|record| record.pid == pid) {
return Ok(());
}
if matches!(driver.try_wait(), Ok(Some(_)) | Err(_)) {
break;
}
std::thread::sleep(ATTACH_POLL);
}
Err(Error::Refused(format!(
"the driver retained for run '{}' did not claim it: {}",
paths.run,
driver_said(paths)
)))
}
fn driver_said(paths: &RunPaths) -> String {
let log = paths.driver_log();
let said = std::fs::read_to_string(&log).unwrap_or_default();
let tail: String = said
.lines()
.rev()
.take(DRIVER_LOG_LINES)
.collect::<Vec<_>>()
.into_iter()
.rev()
.collect::<Vec<_>>()
.join("; ");
if tail.trim().is_empty() {
return format!("it said nothing; its output is in {}", log.display());
}
format!("{tail} (its whole output is in {})", log.display())
}
fn retain_driver(paths: &RunPaths) -> Result<std::process::Child> {
let log = std::fs::OpenOptions::new()
.create(true)
.append(true)
.open(paths.driver_log())
.map_err(|source| Error::Ledger {
path: paths.driver_log(),
source,
})?;
let errors = log.try_clone().map_err(|source| Error::Ledger {
path: paths.driver_log(),
source,
})?;
let exe = std::env::current_exe().map_err(|e| {
Error::Invalid(format!(
"cannot find this executable to retain a driver: {e}"
))
})?;
std::process::Command::new(exe)
.arg(engine::DRIVE_VERB)
.arg(&paths.run)
.stdin(std::process::Stdio::null())
.stdout(log)
.stderr(errors)
.spawn()
.map_err(|e| Error::Invalid(format!("cannot retain a driver for '{}': {e}", paths.run)))
}
fn drive_run(args: &RunArgs) -> Result<i32> {
let paths = resolve(&args.run)?;
let lock = engine::claim(&paths)?;
let mut record: LaunchRecord = ledger::read_json(&paths.launch())?;
let view = RunView::open(&paths)?;
let log = paths.driver_log();
let mut observer = observe(
&paths,
&mut record,
view.state
.plan
.as_ref()
.and_then(|plan| plan.goal.as_ref())
.map(|goal| goal.text.as_str()),
agentgraph::GraphOutput::Logged(&log),
)?;
record.driven_by_this_process();
ledger::write_json(&paths.launch(), &record)?;
let settled = {
let driving = AtomicBool::new(true);
let watched = observer.as_mut();
std::thread::scope(|scope| {
scope.spawn(|| watch_and_reap_observer(watched, &paths.run, &driving));
let settled = engine::drive_holding(&paths, lock);
driving.store(false, Ordering::Release);
settled
})?
};
if let Some(run) = observer.as_mut() {
run.cancel();
}
Ok(settled.exit_code())
}
fn watch_and_reap_observer(
observer: Option<&mut agentgraph::GraphRun>,
run: &str,
driving: &AtomicBool,
) {
let Some(observer) = observer else {
return;
};
while driving.load(Ordering::Acquire) {
if observer.has_exited() {
eprintln!(
"onepipeline: the observer graph for '{run}' has stopped watching; \
the run is still being driven"
);
return;
}
std::thread::sleep(ATTACH_POLL);
}
}
fn launch_graph(
paths: &RunPaths,
record: &LaunchRecord,
goal: Option<&str>,
output: agentgraph::GraphOutput<'_>,
) -> Result<agentgraph::GraphRun> {
let task = run_description(&paths.run, goal);
let mut launched = agentgraph::GraphRun::start(&agentgraph::Launch {
graph: &record.graph,
task: &task,
dir: &recorded_dir(record)?,
labels: &journal::labels(&paths.run, None),
env: &[
(agentgraph::RUN_ID_ENV.to_string(), paths.run.clone()),
(
ledger::RUNS_DIR_ENV.to_string(),
ledger::runs_root().to_string_lossy().into_owned(),
),
],
sets: &record.dag_sets,
filter: record.filters.agentgraph.as_ref(),
output,
})?;
launched.confirm_started()?;
Ok(launched)
}
fn run_description(run: &str, goal: Option<&str>) -> String {
let goal = goal.unwrap_or(crate::plan::NO_GOAL);
format!("onepipeline run `{run}`.\n\nGoal: {goal}")
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Settlement {
Complete,
AwaitingPlanner,
Unattended,
}
impl Settlement {
fn as_str(self) -> &'static str {
match self {
Self::Complete => "complete",
Self::AwaitingPlanner => "awaiting-planner",
Self::Unattended => "unattended",
}
}
fn exit_code(self) -> i32 {
match self {
Self::Unattended => EXIT_NOTHING_DRIVING,
_ => EXIT_SUCCESS,
}
}
}
fn attach(
paths: &RunPaths,
observer: Option<&mut agentgraph::GraphRun>,
lock: ledger::OwnershipLock,
) -> Result<i32> {
let (tx, rx) = std::sync::mpsc::channel();
let mut watched = observer;
if let Some(run) = watched.as_deref_mut() {
let events = run.events();
std::thread::Builder::new()
.name(format!("attach-{}", paths.run))
.spawn(move || {
for envelope in events.flatten() {
if tx.send(envelope).is_err() {
return;
}
}
})
.map_err(|e| Error::Invalid(format!("cannot start the attach relay: {e}")))?;
}
let mut reported = 0usize;
let mut observer_gone = false;
let mut journal = Journal::open(paths);
let driving = paths.clone();
let engine = std::thread::Builder::new()
.name(format!("engine-{}", paths.run))
.spawn(move || engine::drive_holding(&driving, lock))
.map_err(|e| Error::Invalid(format!("cannot start the engine loop: {e}")))?;
loop {
while let Ok(envelope) = rx.try_recv() {
journal.relay(&envelope)?;
}
if let Some(run) = watched.as_deref_mut() {
if run.has_exited() && !observer_gone {
observer_gone = true;
eprintln!(
"onepipeline: the observer graph for '{}' has stopped watching; \
the run is still being driven",
paths.run
);
}
}
let concluded = engine.is_finished();
if concluded {
let deadline = std::time::Instant::now() + DRAIN_GRACE;
while std::time::Instant::now() < deadline {
match rx.recv_timeout(ATTACH_POLL) {
Ok(envelope) => journal.relay(&envelope)?,
Err(std::sync::mpsc::RecvTimeoutError::Timeout) => {}
Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => break,
}
}
}
let view = RunView::open(paths)?;
let lines: Vec<String> = views::monitor(&view, &EventFilter::default())
.lines()
.map(str::to_string)
.collect();
for line in lines.iter().skip(reported) {
eprintln!("{line}");
}
reported = lines.len();
if concluded {
if let Some(run) = watched.as_deref_mut() {
run.cancel();
}
engine.join().map_err(|_| {
Error::Invalid(format!("the engine loop for '{}' panicked", paths.run))
})??;
let settlement = settlement_of(&view);
println!(
"{}",
json!({"run_id": paths.run, "settlement": settlement.as_str()})
);
return Ok(settlement.exit_code());
}
std::thread::sleep(ATTACH_POLL);
}
}
fn settlement_of(view: &RunView) -> Settlement {
let statuses = view.state.statuses();
if !statuses.is_empty() && graph::state_of(&statuses) == GraphState::Complete {
return Settlement::Complete;
}
if views::decision_outstanding(&view.state, &view.paths) {
return Settlement::AwaitingPlanner;
}
Settlement::Unattended
}
fn adopt(args: &RunArgs) -> Result<i32> {
let paths = resolve(&args.run)?;
let session = sys::launching_session();
let mut record: LaunchRecord = ledger::read_json(&paths.launch())?;
if !record.owned_by(&session) {
return Err(Error::NotOwned {
run: paths.run.clone(),
owner: record.owner_label(&session),
});
}
let view = RunView::open(&paths)?;
if !view.liveness().is_undriven() {
return Err(Error::Refused(format!(
"run '{}' is still being driven; end it with `onepipeline stop {}` first",
paths.run, paths.run
)));
}
displace_the_parked_driver(&record);
let lock = engine::claim(&paths)?;
record.adoptions += 1;
record.driven_by_this_process();
let previous = paths
.dir
.join(format!("launch.pre-adopt-{}.json", record.adoptions));
let _ = std::fs::copy(paths.launch(), previous);
ledger::write_json(&paths.launch(), &record)?;
let abandoned = view.state.sessions_in_flight();
for (node, session) in &abandoned {
eprintln!(
"onepipeline: '{node}' had a dispatch in flight; its work is on branch \
'{}' in onevcs session {}, and the node is pinned there so the run \
continues that branch rather than cutting a second one beside it",
session.branch(),
session.token().0
);
}
let mut adopted = vec![
("adoption", json!(record.adoptions)),
("pid", json!(record.pid)),
];
if !abandoned.is_empty() {
adopted.push((
journal::ADOPTED_ABANDONED,
json!(abandoned
.iter()
.map(|(node, session)| json!({
"node": node,
"session": session.token().0,
"branch": session.branch().as_str(),
}))
.collect::<Vec<_>>()),
));
}
let mut journal = Journal::open(&paths);
journal.emit(
journal::PipelineKind::DriverAdopted,
journal::labels(&paths.run, None),
journal::payload(&adopted),
)?;
let mut observer = if record.observer_graph().is_none() {
None
} else {
let launched = launch_graph(
&paths,
&record,
view.state
.plan
.as_ref()
.and_then(|plan| plan.goal.as_ref())
.map(|goal| goal.text.as_str()),
agentgraph::GraphOutput::Relayed,
)?;
record.graph_run = launched
.run_id()
.map(ToString::to_string)
.unwrap_or_default();
Some(launched)
};
ledger::write_json(&paths.launch(), &record)?;
attach(&paths, observer.as_mut(), lock)
}
fn displace_the_parked_driver(record: &LaunchRecord) {
if record.host != sys::hostname() || !sys::process_may_be_live(record.pid) {
return;
}
eprintln!(
"onepipeline: run '{}' is held by driver pid {}, which is not working; \
ending it to adopt the run",
record.run_id, record.pid
);
sys::stop(record.pid, sys::Stop::Politely);
let deadline = Instant::now() + DRIVER_HANDOVER;
while Instant::now() < deadline && sys::process_may_be_live(record.pid) {
std::thread::sleep(ATTACH_POLL);
}
}
fn stop(args: &StopArgs) -> Result<i32> {
let paths = resolve(&args.run)?;
let session = sys::launching_session();
let record: LaunchRecord = ledger::read_json(&paths.launch())?;
let owner = record.owner_label(&session);
if !record.owned_by(&session) {
if !args.force {
return Err(Error::NotOwned {
run: paths.run.clone(),
owner,
});
}
eprintln!(
"onepipeline: run '{}' belongs to {owner}; stopping it anyway",
paths.run
);
}
let teardown = terminate(&paths, &record).map_err(|why| {
Error::Refused(format!(
"run '{}' was not stopped: this build cannot establish what it is running — {why}. \
The run is untouched; nothing was signalled. Fix or remove the entry the path \
above names and run `onepipeline stop {}` again",
paths.run, paths.run
))
})?;
let established = match teardown {
None => journal::StopTeardown::Elsewhere,
Some(sys::Teardown::Signalled) => journal::StopTeardown::Signalled,
Some(sys::Teardown::NothingToStop) => journal::StopTeardown::NothingToStop,
Some(sys::Teardown::NotAttempted) => journal::StopTeardown::NotAttempted,
Some(sys::Teardown::PartlySignalled) => journal::StopTeardown::PartlySignalled,
#[cfg(unix)]
Some(sys::Teardown::Refused) => journal::StopTeardown::Refused,
};
let mut journal = Journal::open(&paths);
journal.emit(
journal::PipelineKind::RunStopped,
journal::labels(&paths.run, None),
journal::payload(&[
("owner", json!(owner)),
("forced", json!(args.force)),
(journal::STOP_TEARDOWN, json!(established)),
]),
)?;
let run = &paths.run;
match teardown {
Some(sys::Teardown::NotAttempted) => {
return Err(Error::Refused(format!(
"run '{run}' was not stopped: this host gave no answer its tree could be \
read from — no process listing, or nothing that says whether a pid it \
recorded is still the process it named, each said above — so the \
processes the run started could not be found, and ending its driver \
alone would have orphaned them. The run is untouched — run \
`onepipeline stop {run}` again once this host answers"
)));
}
Some(sys::Teardown::PartlySignalled) => {
return Err(Error::Refused(format!(
"run '{run}' was only partly stopped: part of its process tree was \
signalled and at least one process in it is still running — one this \
session could not signal, or one that took the ask and stayed. Find it \
in this host's process list and end it as the user that owns it"
)));
}
#[cfg(unix)]
Some(sys::Teardown::Refused) => {
return Err(Error::Refused(format!(
"run '{run}' was not stopped: its process tree was found and every \
process in it refused this session's signal, so nothing was signalled \
and all of it is still running. Running `onepipeline stop {run}` again \
as this user will be refused the same way — find the tree in this \
host's process list and end it as the user that owns it"
)));
} None | Some(sys::Teardown::Signalled) | Some(sys::Teardown::NothingToStop) => {}
}
println!(
"{}",
json!({
"run_id": paths.run,
"stopped": true,
"owner": owner,
journal::STOP_TEARDOWN: established,
})
);
Ok(EXIT_SUCCESS)
}
fn terminate(paths: &RunPaths, record: &LaunchRecord) -> Result<Option<sys::Teardown>> {
let Aim::Here { roots, unproven } = roots_to_stop(paths, record)? else {
return Ok(None);
};
let established = sys::stop_and_confirm(&roots, sys::Stop::Politely, TEARDOWN_PATIENCE);
if unproven.is_empty() {
return Ok(Some(established));
}
Ok(Some(match established {
sys::Teardown::NothingToStop | sys::Teardown::NotAttempted => sys::Teardown::NotAttempted,
sys::Teardown::Signalled | sys::Teardown::PartlySignalled => sys::Teardown::PartlySignalled,
#[cfg(unix)]
sys::Teardown::Refused => sys::Teardown::Refused,
}))
}
#[derive(Debug, PartialEq, Eq)]
enum Aim {
Elsewhere,
Here {
roots: Vec<u32>,
unproven: Vec<u32>,
},
}
fn roots_to_stop(paths: &RunPaths, record: &LaunchRecord) -> Result<Aim> {
let here = sys::hostname();
let mut on_this_host = false;
let mut roots: Vec<u32> = Vec::new();
let mut unproven: Vec<u32> = Vec::new();
let claimed = std::iter::once((
RECORDED_DRIVER,
record.pid,
record.host.clone(),
record.started.clone(),
))
.chain(lock_held_on(paths).map(|held| (LOCK_HOLDER, held.pid, held.host, held.started)))
.chain(ledger::dispatches_of(paths)?.into_iter().map(|running| {
(
REGISTERED_DISPATCH,
running.pid,
running.host,
running.started,
)
}));
for (named_by, pid, host, started) in claimed {
if host != here {
continue;
}
on_this_host = true;
if roots.contains(&pid) || unproven.contains(&pid) {
continue;
}
match claim_on(pid, &started) {
Claim::Proved => roots.push(pid),
Claim::Gone => {}
Claim::Reissued => eprintln!(
"onepipeline: run '{}': the {named_by} names pid {pid}, which this host has \
since given to another process, so it was not signalled",
paths.run
),
Claim::Unstamped => {
left_alone(
&paths.run,
named_by,
pid,
"its record carries no start token",
);
unproven.push(pid);
}
Claim::HostSilent => {
left_alone(
&paths.run,
named_by,
pid,
"this host will not say when it started",
);
unproven.push(pid);
}
}
}
if !on_this_host {
return Ok(Aim::Elsewhere);
}
Ok(Aim::Here { roots, unproven })
}
fn left_alone(run: &str, named_by: &str, pid: u32, why: &str) {
eprintln!(
"onepipeline: run '{run}': the {named_by} names pid {pid}, which is running on this \
host — {why}, so nothing says it is still this run's process and it was not signalled"
);
}
const RECORDED_DRIVER: &str = "launch record";
const LOCK_HOLDER: &str = "ownership lock";
const REGISTERED_DISPATCH: &str = "dispatch registry";
#[derive(Debug, PartialEq, Eq)]
enum Claim {
Proved,
Gone,
Reissued,
Unstamped,
HostSilent,
}
fn claim_on(pid: u32, started: &str) -> Claim {
let reading = sys::process_start_token(pid);
match reading {
Some(ref token) if token.matches(started) => Claim::Proved,
_ if !sys::process_may_be_live(pid) => Claim::Gone,
Some(_) if !started.is_empty() => Claim::Reissued,
Some(_) => Claim::Unstamped,
None => Claim::HostSilent,
}
}
fn lock_held_on(paths: &RunPaths) -> Option<ledger::LockRecord> {
let path = paths.lock();
match ledger::read_json::<ledger::LockRecord>(&path) {
Ok(held) => Some(held),
Err(Error::Ledger { source, .. }) if source.kind() == std::io::ErrorKind::NotFound => None,
Err(error) => {
eprintln!(
"onepipeline: the ownership lock of run '{}' cannot be read, so this stop aims \
only at the driver the launch record names: {error}",
paths.run
);
None
}
}
}
fn next(args: &ReadArgs) -> Result<i32> {
let paths = resolve(&args.run)?;
let view = RunView::open(&paths)?;
let filter = read_filter(&view, args)?;
let events = views::shaped(&view, &filter);
let channel = ChannelState::new(&paths);
let Some(surface) = channel.claim()? else {
let settled = view.liveness().is_undriven();
let status = if settled { "finished" } else { "running" };
println!(
"{}",
json!({"status": status, "surface": null, "events": events})
);
return Ok(EXIT_SUCCESS);
};
let mut journal = Journal::open(&paths);
journal.emit(
journal::PipelineKind::PlannerSurfaced,
journal::labels(&paths.run, surface.workstream.as_deref()),
journal::payload(&[
("kind", json!(surface.kind)),
("message", json!(surface.message)),
("source", json!(surface.source)),
("blocking", json!(surface.blocking)),
]),
)?;
if let Err(error) = agentgraph::recorded_graph_run(&view.launch.graph_run, &paths.run)
.and_then(|graph_run| agentgraph::reset_timer(&graph_run, agentgraph::CHECK_IN_MEMBER))
{
eprintln!("onepipeline: could not reset the check-in pacemaker: {error}");
}
println!(
"{}",
json!({"status": "surface", "surface": surface, "events": events})
);
Ok(EXIT_SUCCESS)
}
fn surface(args: &SurfaceArgs) -> Result<i32> {
let paths = resolve(&args.run)?;
let kind = match args.kind {
SurfaceKind::CheckIn => crate::channel::source::CHECK_IN,
};
let queued = ChannelState::new(&paths).push(Surface {
id: 0,
kind: kind.to_string(),
message: args.message.clone(),
source: kind.to_string(),
blocking: false,
queued_at: sys::now_millis(),
workstream: None,
})?;
let mut journal = Journal::open(&paths);
journal.emit(
journal::PipelineKind::PlannerSurfaceQueued,
journal::labels(&paths.run, None),
journal::payload(&[
("kind", json!(queued.kind)),
("message", json!(queued.message)),
("source", json!(queued.source)),
("blocking", json!(false)),
]),
)?;
println!("{}", json!({"surface": queued.id, "state": "queued"}));
Ok(EXIT_SUCCESS)
}
fn attest(args: &AttestArgs) -> Result<i32> {
submit(
&resolve(&args.run)?,
&Reply {
version: Some(crate::channel::REPLY_ENVELOPE_VERSION),
author: Author::Planner,
commands: vec![Command::Attest {
reference: args.reference.clone(),
}],
..Reply::default()
},
)
}
fn reply(args: &ReplyArgs) -> Result<i32> {
let paths = resolve(&args.run)?;
let text = match &args.file {
Some(path) => std::fs::read_to_string(path).map_err(|e| Error::Ledger {
path: path.clone(),
source: e,
})?,
None => {
let mut buffer = String::new();
std::io::stdin()
.read_to_string_compat(&mut buffer)
.map_err(|e| Error::Refused(format!("cannot read the reply from stdin: {e}")))?;
buffer
}
};
let envelope: Reply = serde_json::from_str(text.trim()).map_err(|e| {
let why = serde_json::from_str::<serde_json::Value>(text.trim())
.ok()
.as_ref()
.and_then(crate::plan::retired_field_refusal)
.unwrap_or_else(|| e.to_string());
Error::Refused(format!("the reply is malformed: {why}"))
})?;
submit(&paths, &envelope)
}
fn submit(paths: &RunPaths, envelope: &Reply) -> Result<i32> {
let view = RunView::open(paths)?;
let channel = ChannelState::new(paths);
if envelope.commands.is_empty() {
crate::channel::allows_completion(envelope.author, envelope.completion)?;
if channel.pending().is_none() && view.liveness().is_undriven() {
return Err(Error::Refused(format!(
"run '{}' has settled, so nothing will ever read a reply to it; \
no reply was queued",
paths.run
)));
}
let id = channel.answer(envelope)?;
let mut journal = Journal::open(paths);
journal.emit(
journal::PipelineKind::PlannerReplied,
journal::labels(&paths.run, None),
journal::payload(&[
("author", json!(envelope.author)),
("completion", json!(envelope.completion)),
("reason", json!(envelope.reason)),
]),
)?;
if let Some(reason) = &envelope.reason {
if envelope.completion == Some(true) {
journal.emit(
journal::PipelineKind::CompletionRequested,
journal::labels(&paths.run, None),
journal::payload(&[("reason", json!(reason))]),
)?;
}
}
println!("{}", json!({"reply": id, "state": "delivered"}));
return Ok(EXIT_SUCCESS);
}
if envelope.version != Some(crate::channel::REPLY_ENVELOPE_VERSION) {
return Err(Error::Refused(format!(
"an edit envelope requires version {}",
crate::channel::REPLY_ENVELOPE_VERSION
)));
}
for command in &envelope.commands {
crate::channel::allows(envelope.author, command)?;
}
let mut projected = view.state.graph.clone();
let frontier = view.state.frontier();
for command in &envelope.commands {
edits::compile(&mut projected, &frontier, command)?;
}
match ledger::OwnershipLock::acquire(paths, "reply") {
Ok(lock) => {
let mut journal = Journal::open(paths);
let mut graph = view.state.graph.clone();
for command in &envelope.commands {
let operations = edits::compile(&mut graph, &frontier, command)?;
journal.emit(
journal::PipelineKind::EditCommitted,
journal::labels(&paths.run, None),
journal::payload(&[
("author", json!(envelope.author)),
("command", json!(command)),
("operations", json!(operations)),
]),
)?;
for operation in &operations {
match operation {
edits::Operation::CompletionRequested { reason } => journal.emit(
journal::PipelineKind::CompletionRequested,
journal::labels(&paths.run, None),
journal::payload(&[("reason", json!(reason))]),
)?,
edits::Operation::HumanAttested { node } => journal.emit(
journal::PipelineKind::HumanAttested,
journal::labels(&paths.run, Some(node)),
journal::payload(&[("ref", json!(node))]),
)?,
_ => {}
}
}
if envelope.author == Author::Monitor {
engine::raise(paths, &mut journal, engine::monitor_edit(command))?;
}
}
lock.release();
channel.answer_if_verdict(envelope)?;
println!("{}", json!({"reply": 0, "state": "applied"}));
Ok(EXIT_SUCCESS)
}
Err(Error::Locked { .. }) => {
let id = channel.submit(envelope.author, &envelope.commands)?;
let deadline = Instant::now() + Duration::from_secs(reply_timeout_seconds());
while Instant::now() < deadline {
if let Some(outcome) = channel.outcome_of(id) {
channel.answer_if_verdict(envelope)?;
if outcome.applied {
println!("{}", json!({"reply": id, "state": "applied"}));
return Ok(EXIT_SUCCESS);
}
return Err(Error::Refused(
outcome
.reason
.unwrap_or_else(|| "the reconciler rejected the edit".into()),
));
}
std::thread::sleep(ATTACH_POLL);
}
channel.answer_if_verdict(envelope)?;
println!("{}", json!({"reply": id, "state": "queued"}));
Ok(EXIT_QUEUED)
}
Err(other) => Err(other),
}
}
fn reply_timeout_seconds() -> u64 {
std::env::var(crate::channel::REPLY_TIMEOUT_ENV)
.ok()
.and_then(|value| value.parse().ok())
.filter(|seconds| *seconds > 0)
.unwrap_or(crate::channel::DEFAULT_REPLY_TIMEOUT_SECONDS)
}
fn serve(args: &RunArgs) -> Result<i32> {
let paths = resolve(&args.run)?;
let channel = ChannelState::new(&paths);
let stdin = std::io::stdin();
for line in stdin.lock().lines() {
let line = line.map_err(|e| Error::Sibling {
tool: "oneagentgraph",
message: format!("the observer's frame stream could not be read: {e}"),
})?;
if line.trim().is_empty() {
continue;
}
let frame: ObserverFrame = serde_json::from_str(line.trim())
.map_err(|e| Error::Refused(format!("the observer emitted a bad frame: {e}")))?;
if let Some(node) = &frame.node {
let graph = RunView::open(&paths)?.state.graph;
if !graph.contains(node) {
return Err(Error::Refused(format!(
"the observer raised a frame about node '{node}', which run '{}' does not \
have; it has: {}",
paths.run,
graph.ids().cloned().collect::<Vec<_>>().join(", ")
)));
}
}
let queued = channel.push(Surface {
id: 0,
kind: frame.kind,
message: frame.message,
source: crate::channel::source::PROPOSAL.to_string(),
blocking: frame.blocking,
queued_at: sys::now_millis(),
workstream: frame.node,
})?;
let mut journal = Journal::open(&paths);
journal.emit(
journal::PipelineKind::PlannerSurfaceQueued,
journal::labels(&paths.run, queued.workstream.as_deref()),
journal::payload(&[
("kind", json!(queued.kind)),
("message", json!(queued.message)),
("source", json!(queued.source)),
("blocking", json!(queued.blocking)),
]),
)?;
let answer = wait_for_reply(&channel)?;
println!(
"{}",
serde_json::to_string(&answer).map_err(|e| Error::Invalid(format!("verdict: {e}")))?
);
std::io::stdout()
.flush()
.map_err(|e| Error::Refused(format!("cannot write the verdict: {e}")))?;
if answer.completion == Some(true) {
break;
}
}
Ok(EXIT_SUCCESS)
}
#[derive(Debug, serde::Deserialize)]
#[serde(deny_unknown_fields)]
struct ObserverFrame {
kind: String,
message: String,
#[serde(default = "blocking_by_default")]
blocking: bool,
#[serde(default)]
node: Option<String>,
}
fn blocking_by_default() -> bool {
true
}
fn wait_for_reply(channel: &ChannelState) -> Result<Reply> {
let deadline = Instant::now() + Duration::from_secs(reply_timeout_seconds());
while Instant::now() < deadline {
if let Some(claimed) = channel.claim_reply()? {
return Ok(claimed.reply);
}
std::thread::sleep(ATTACH_POLL);
}
Ok(Reply {
completion: Some(false),
message: Some("no planner reply within the timeout; continue".into()),
reason: Some("the channel timed out waiting for a verdict".into()),
..Reply::default()
})
}
fn runs(args: &RunsArgs) -> Result<i32> {
print!(
"{}",
views::runs(&ledger::runs_root(), args.mine, &sys::launching_session())
);
Ok(EXIT_SUCCESS)
}
fn report(args: &OptionalRunArgs, render: fn(&views::Survey) -> String) -> Result<i32> {
let survey = match &args.run {
Some(run) => views::Survey::of_one(RunView::open(&resolve(run)?)?),
None => views::Survey::of(&ledger::runs_root()),
};
print!("{}", render(&survey));
Ok(EXIT_SUCCESS)
}
fn transcript(args: &TranscriptArgs) -> Result<i32> {
let view = RunView::open(&resolve(&args.run)?)?;
if let Some(node) = &args.node {
if views::nodes_with_agent_records(&view, Some(node)).is_empty() {
let recorded = views::nodes_with_agent_records(&view, None);
return Err(Error::Refused(format!(
"run '{}' has recorded nothing for node '{node}'; it has records for: {}",
args.run,
if recorded.is_empty() {
"nothing yet".to_string()
} else {
recorded.join(", ")
}
)));
}
}
print!("{}", views::transcript(&view, args.node.as_deref()));
Ok(EXIT_SUCCESS)
}
fn report_telemetry(args: &TelemetryArgs) -> Result<i32> {
let survey = match &args.run {
Some(run) => views::Survey::of_one(RunView::open(&resolve(run)?)?),
None => views::Survey::of(&ledger::runs_root()),
};
for view in &survey.views {
let aggregated = telemetry::of_run(&view.paths, &view.events);
if args.breakdown {
print!("{}", telemetry::render_breakdown(&aggregated));
} else {
println!(
"{}",
serde_json::to_string(&aggregated)
.map_err(|e| Error::Invalid(format!("telemetry: {e}")))?
);
}
}
Ok(EXIT_SUCCESS)
}
trait ReadToStringCompat {
fn read_to_string_compat(&self, buffer: &mut String) -> std::io::Result<usize>;
}
impl ReadToStringCompat for std::io::Stdin {
fn read_to_string_compat(&self, buffer: &mut String) -> std::io::Result<usize> {
use std::io::Read;
self.lock().read_to_string(buffer)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::plan::{Node, PLAN_SCHEMA_VERSION};
use crate::views::DriverLiveness;
use std::path::PathBuf;
fn plan(name: Option<&str>) -> Plan {
Plan {
schema_version: PLAN_SCHEMA_VERSION,
goal: None,
name: name.map(str::to_string),
concurrency: 4,
tasks: vec![Node {
id: "build".into(),
persona: Some("engineer".into()),
task: Some("## What\ndo it".into()),
..Node::default()
}],
}
}
const ROLE_PROSE: &[&str] = &[
"Drive",
"drive",
"to settlement",
"observe",
"Observe",
"nothing else",
"run state",
"you",
"You",
];
fn assert_role_neutral(task: &str) {
for prose in ROLE_PROSE {
assert!(
!task.contains(prose),
"the launched graph's task tells a member what to do ({prose:?}): {task}"
);
}
}
#[test]
fn the_launched_graphs_task_names_the_run_and_its_goal_and_no_role() {
let task = run_description("tracked-release", Some("close the coverage gap"));
assert!(
task.contains("tracked-release"),
"the task does not name the run: {task}"
);
assert!(
task.contains("close the coverage gap"),
"the task does not say what the run is for: {task}"
);
assert_role_neutral(&task);
}
#[test]
fn a_run_whose_plan_states_no_goal_says_so_in_the_same_shape() {
let task = run_description("nameless", None);
assert!(
task.contains("nameless") && task.contains(crate::plan::NO_GOAL),
"the task for a goalless run reads: {task}"
);
assert_role_neutral(&task);
}
fn scratch(name: &str) -> PathBuf {
let dir = std::env::temp_dir().join(format!("onepipeline-driver-{name}-{}", sys::pid()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).expect("a scratch root");
dir
}
#[test]
fn a_run_id_comes_from_the_plans_name_and_is_made_unique() {
let root = scratch("mint");
let path = Path::new("plans/release.plan.json");
assert_eq!(
mint_run_id(&plan(Some("tracked-release")), path, &root),
"tracked-release"
);
std::fs::create_dir_all(root.join("tracked-release")).expect("an existing run");
assert_eq!(
mint_run_id(&plan(Some("tracked-release")), path, &root),
"tracked-release-2"
);
std::fs::remove_dir_all(&root).ok();
}
#[test]
fn a_nameless_plan_takes_its_run_id_from_the_file() {
let root = scratch("mint-file");
assert_eq!(
mint_run_id(&plan(None), Path::new("plans/release.plan.json"), &root),
"release"
);
assert_eq!(
mint_run_id(&plan(None), Path::new("plans/odd name!.json"), &root),
"odd-name-"
);
std::fs::remove_dir_all(&root).ok();
}
#[test]
fn every_settlement_carries_the_exit_code_the_contract_assigns() {
assert_eq!(Settlement::Complete.exit_code(), EXIT_SUCCESS);
assert_eq!(Settlement::AwaitingPlanner.exit_code(), EXIT_SUCCESS);
assert_eq!(Settlement::Unattended.exit_code(), EXIT_NOTHING_DRIVING);
assert_eq!(Settlement::Complete.as_str(), "complete");
assert_eq!(Settlement::AwaitingPlanner.as_str(), "awaiting-planner");
assert_eq!(Settlement::Unattended.as_str(), "unattended");
}
#[test]
fn the_launched_graphs_task_never_asks_a_member_to_drive_the_engine() {
let task = run_description("tracked-release", Some("close the coverage gap"));
for verb in ["round run", "round next", "drive-run"] {
assert!(
!task.contains(verb),
"the observer's task names {verb}: {task}"
);
}
}
#[test]
fn an_undriven_run_is_the_settlement_a_planner_must_intervene_in() {
assert!(DriverLiveness::DriverDead.is_undriven());
assert!(DriverLiveness::Parked.is_undriven());
assert!(!DriverLiveness::Driving.is_undriven());
}
#[test]
fn the_reply_timeout_falls_back_when_the_environment_is_unusable() {
assert!(reply_timeout_seconds() > 0);
}
fn launched_by(pid: u32, host: &str, started: &str) -> LaunchRecord {
LaunchRecord {
run_id: "stopped".into(),
plan: PathBuf::from("plan.json"),
dir: PathBuf::from("/tmp/launch"),
graph: String::new(),
graph_run: String::new(),
node_graph: "graphs/node-scope.yaml".into(),
pr_author_graph: String::new(),
launcher: "e2e".into(),
session: "session-a".into(),
pid,
host: host.to_string(),
started: started.to_string(),
started_at: sys::now_rfc3339(),
heartbeat_interval: 1_800,
dag_sets: Vec::new(),
node_sets: Vec::new(),
adoptions: 0,
filters: crate::filter::Filters::default(),
}
}
#[test]
fn a_stop_aims_at_every_stamped_claim_the_run_holds_and_never_at_a_pid_nothing_proves() {
let root = scratch("roots");
let paths = RunPaths::under(&root, "stopped");
paths.create().expect("the run directory");
let here = sys::hostname();
let dead = sys::reaped_pid();
let stamp = |started: &str| ledger::LockRecord {
pid: sys::pid(),
host: here.clone(),
acquired_at: sys::now_rfc3339(),
verb: "drive".into(),
started: started.to_string(),
};
let held = |record: &ledger::LockRecord| {
ledger::write_json(&paths.lock(), record).expect("a held lock");
};
let proven = sys::process_start_token(sys::pid())
.expect("this host says when a process started")
.recorded()
.to_string();
let aimed_at =
|record: &LaunchRecord| roots_to_stop(&paths, record).expect("this run's records read");
let roots = |record: &LaunchRecord| match aimed_at(record) {
Aim::Here { roots, .. } => roots,
Aim::Elsewhere => panic!("a run this host's own records name read as another host's"),
};
assert_eq!(
aimed_at(&launched_by(dead, &here, &proven)),
Aim::Here {
roots: Vec::new(),
unproven: Vec::new()
}
);
assert_eq!(
roots(&launched_by(sys::pid(), &here, &proven)),
vec![sys::pid()]
);
assert_eq!(
aimed_at(&launched_by(
sys::pid(),
&here,
"the driver it named, which is not this process",
)),
Aim::Here {
roots: Vec::new(),
unproven: Vec::new()
},
"a stop aimed at a pid the host has since given to another process"
);
assert_eq!(
aimed_at(&launched_by(sys::pid(), &here, "")),
Aim::Here {
roots: Vec::new(),
unproven: vec![sys::pid()]
}
);
held(&stamp(&proven));
assert_eq!(
roots(&launched_by(dead, &here, &proven)),
vec![sys::pid()],
"a stop did not aim at the process the lock stamps as driving the run"
);
held(&stamp("the process that took it, which is not this one"));
assert!(
roots(&launched_by(dead, &here, &proven)).is_empty(),
"a stop aimed at a pid the lock's own stamp disowns"
);
held(&stamp(""));
assert_eq!(
aimed_at(&launched_by(dead, &here, &proven)),
Aim::Here {
roots: Vec::new(),
unproven: vec![sys::pid()]
}
);
held(&ledger::LockRecord {
host: "a-host-this-is-not".into(),
..stamp(&proven)
});
assert!(roots(&launched_by(dead, &here, &proven)).is_empty());
std::fs::write(paths.lock(), "not json at all").expect("a lock nobody can read");
assert!(roots(&launched_by(dead, &here, &proven)).is_empty());
assert_eq!(
aimed_at(&launched_by(dead, "a-host-this-is-not", &proven)),
Aim::Elsewhere
);
std::fs::remove_file(paths.lock()).expect("the lock is given up");
let running = |pid: u32, host: &str, started: &str| ledger::DispatchRecord {
node: "build".into(),
pid,
host: host.to_string(),
dispatched_at: sys::now_rfc3339(),
started: started.to_string(),
};
let recorded = |record: &ledger::DispatchRecord| {
ledger::write_json(&paths.dispatch(record.pid, 0), record)
.expect("a recorded dispatch");
};
recorded(&running(sys::pid(), &here, &proven));
assert_eq!(
roots(&launched_by(dead, &here, &proven)),
vec![sys::pid()],
"a stop did not aim at the process the registry says the run's work is in"
);
for disowned in [
running(
sys::pid(),
&here,
"the process that took it, which is not this one",
),
running(sys::pid(), "a-host-this-is-not", &proven),
] {
recorded(&disowned);
assert!(
roots(&launched_by(dead, &here, &proven)).is_empty(),
"a stop aimed at a pid the registry cannot prove: {disowned:?}"
);
}
recorded(&running(sys::pid(), &here, &proven));
held(&stamp(&proven));
assert_eq!(
roots(&launched_by(sys::pid(), &here, &proven)),
vec![sys::pid()]
);
std::fs::remove_dir_all(&root).ok();
}
#[test]
fn a_registry_this_build_cannot_read_refuses_the_stop_rather_than_narrowing_it() {
let root = scratch("roots-unreadable");
let paths = RunPaths::under(&root, "stopped");
paths.create().expect("the run directory");
let here = sys::hostname();
let launch = launched_by(sys::reaped_pid(), &here, "a driver that has since died");
let usable = ledger::DispatchRecord {
node: "build".into(),
pid: sys::pid(),
host: here.clone(),
dispatched_at: sys::now_rfc3339(),
started: sys::process_start_token(sys::pid())
.expect("this host says when a process started")
.recorded()
.to_string(),
};
assert!(roots_to_stop(&paths, &launch).is_ok());
for (what, entry) in [
(
"a record that is not JSON at all",
"not an entry".to_string(),
),
(
"a record carrying a field this build does not know",
serde_json::to_string(&serde_json::json!({
"node": "build",
"pid": sys::pid(),
"host": here,
"dispatched_at": sys::now_rfc3339(),
"started": usable.started,
"reaped_by": "a build that came later",
}))
.expect("an entry from a newer writer"),
),
(
"a record whose stamp proves nothing",
serde_json::to_string(&ledger::DispatchRecord {
started: String::new(),
..usable.clone()
})
.expect("an unstamped entry"),
),
] {
std::fs::write(paths.dispatch(usable.pid, 0), entry).expect("an entry");
let refused = roots_to_stop(&paths, &launch)
.expect_err(&format!("{what} was read as a registry to act on"));
assert!(
refused.to_string().contains(&usable.pid.to_string()),
"the refusal does not name the entry that caused it: {refused}"
);
}
std::fs::remove_dir_all(paths.dispatches()).expect("the registry is taken away");
let refused = roots_to_stop(&paths, &launch)
.expect_err("a run with no registry at all was read as a run with nothing running");
assert!(
refused
.to_string()
.contains(&paths.dispatches().display().to_string()),
"the refusal does not name the registry it could not read: {refused}"
);
std::fs::remove_dir_all(&root).ok();
}
}