use std::io::{BufRead, Write};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::{Duration, Instant};
use serde_json::json;
use crate::agentgraph;
use crate::channel::{Author, ChannelState, Command, Reply, Surface, SurfaceKind};
use crate::cli::{
AdoptArgs, AttestArgs, ChannelCommand, Cli, DriveRunArgs, OptionalRunArgs, ReadArgs, ReplyArgs,
RunArgs, RunsArgs, StartArgs, StopArgs, SurfaceArgs, TelemetryArgs, TranscriptArgs, ADOPT_FLAG,
DAG_GRAPH_OFF,
};
use crate::concurrency::{self, Liveness, State};
use crate::edits::{self, Frontier};
use crate::engine;
use crate::error::{Error, Result, EXIT_NOTHING_DRIVING, EXIT_QUEUED, EXIT_SUCCESS};
use crate::filter::{self, EventFilter};
use crate::graph::{self, GraphState};
use crate::journal::{self, Journal};
use crate::ledger::{self, LaunchRecord, RunPaths};
use crate::plan::Plan;
use crate::sys;
use crate::telemetry;
use crate::views::{self, RunView};
const ATTACH_POLL: Duration = Duration::from_millis(50);
const DRAIN_GRACE: Duration = Duration::from_secs(2);
const DRIVER_HANDOVER: Duration = Duration::from_secs(30);
const TEARDOWN_PATIENCE: Duration = Duration::from_secs(5);
const DRIVER_LOG_LINES: usize = 8;
pub fn dispatch(cli: Cli) -> Result<i32> {
use crate::cli::Command as Verb;
match cli.command {
Verb::Start(args) => start(&args),
Verb::Plan(crate::cli::PlanCommand::Check(args)) => crate::plancheck::check(&args),
Verb::Adopt(args) => adopt(&args),
Verb::DriveRun(args) => drive_run(&args),
Verb::Channel(ChannelCommand::Serve(args)) => serve(&args),
Verb::Next(args) => next(&args),
Verb::Reply(args) => reply(&args),
Verb::Surface(args) => surface(&args),
Verb::Attest(args) => attest(&args),
Verb::Stop(args) => stop(&args),
Verb::Runs(args) => runs(&args),
Verb::Status(args) => report(&args, views::status),
Verb::Host => report(&OptionalRunArgs { run: None }, views::host),
Verb::Monitor(args) => {
let view = RunView::open(&resolve(&args.run)?)?;
let filter = read_filter(&view, &args)?;
print!("{}", views::monitor(&view, &filter));
Ok(EXIT_SUCCESS)
}
Verb::Watch(args) => {
let paths = resolve(&args.read.run)?;
let view = RunView::open(&paths)?;
let filter = read_filter(&view, &args.read)?;
crate::watch::watch(&args, &paths, &filter)
}
Verb::Results(args) => {
print!("{}", views::results(&RunView::open(&resolve(&args.run)?)?));
Ok(EXIT_SUCCESS)
}
Verb::Goals(args) => report(&args, views::goals),
Verb::Transcript(args) => transcript(&args),
Verb::Telemetry(args) => report_telemetry(&args),
Verb::Drive(args) => agentgraph::drive(
&args.graph,
&args.task,
&args.dir,
&args.labels,
&args.sets,
args.event_filter.as_deref(),
),
}
}
fn resolve(run: &str) -> Result<RunPaths> {
if !ledger::is_valid_run_id(run) {
return Err(Error::Invalid(format!(
"'{run}' is not a run id: a run id names one directory under the runs root, \
so it may not be a path"
)));
}
let paths = RunPaths::new(run);
if !paths.exists() {
return Err(Error::NoSuchRun {
run: run.to_string(),
root: ledger::runs_root(),
});
}
Ok(paths)
}
fn launch_dir() -> Result<PathBuf> {
std::env::current_dir()
.map_err(|error| Error::Invalid(format!("cannot read the launch directory: {error}")))
}
fn recorded_dir(record: &LaunchRecord) -> Result<PathBuf> {
if record.dir.as_os_str().is_empty() {
return launch_dir();
}
if !record.dir.is_absolute() {
return Err(Error::Invalid(format!(
"run '{}' records the relative working directory '{}'; a run's directory has to be \
absolute, because the process that resolves it is not the one that launched it",
record.run_id,
record.dir.display()
)));
}
if !record.dir.is_dir() {
return Err(Error::Invalid(format!(
"run '{}' records the working directory '{}', which is not a directory on {}",
record.run_id,
record.dir.display(),
sys::hostname()
)));
}
Ok(record.dir.clone())
}
fn resolve_graph(reference: &str, base: &Path) -> Result<String> {
if reference.trim().is_empty() {
return Err(Error::Invalid(
"graph reference is blank: name a path, an `https://` URL, or no graph at all"
.to_string(),
));
}
if reference.starts_with("https://") || Path::new(reference).is_absolute() {
return Ok(reference.to_string());
}
let resolved = base.join(reference);
std::fs::File::open(&resolved).map_err(|error| {
Error::Invalid(format!(
"cannot read graph '{}' resolved against launch directory '{}': {error}",
reference,
base.display()
))
})?;
Ok(resolved.to_string_lossy().into_owned())
}
fn resolve_plan_graphs(plan: &mut Plan, base: &Path) -> Result<()> {
for node in &mut plan.tasks {
if let Some(reference) = &mut node.agent_graph {
reference.0 = resolve_graph(&reference.0, base)?;
}
if let Some(steps) = &mut node.steps {
for step in steps {
if let Some(reference) = &mut step.agent_graph {
reference.0 = resolve_graph(&reference.0, base)?;
}
}
}
}
Ok(())
}
fn mint_run_id(plan: &Plan, native: &str, root: &Path) -> String {
let base = plan
.name
.clone()
.filter(|name| !name.is_empty())
.unwrap_or_else(|| native.to_string());
let base: String = base
.chars()
.map(|c| {
if c.is_alphanumeric() || c == '-' || c == '_' {
c
} else {
'-'
}
})
.collect();
if !root.join(&base).exists() {
return base;
}
(2..)
.map(|n| format!("{base}-{n}"))
.find(|candidate| !root.join(candidate).exists())
.unwrap_or(base)
}
fn declared_filters(
config: crate::filter::Filters,
args: &StartArgs,
) -> Result<crate::filter::Filters> {
let mut filters = config;
for declaration in &args.filter_profiles {
let (name, spec) = declaration.split_once('=').ok_or_else(|| {
Error::Invalid(format!(
"--filter-profile takes NAME=SPEC; '{declaration}' names no profile"
))
})?;
if name.trim().is_empty() {
return Err(Error::Invalid(format!(
"--filter-profile takes NAME=SPEC; '{declaration}' has an empty name"
)));
}
filters
.profiles
.insert(name.to_string(), EventFilter::read(spec)?);
}
if let Some(spec) = args.filter_agentgraph.as_deref() {
filters.agentgraph = Some(EventFilter::read(spec)?);
}
if let Some(spec) = args.filter_vcs.as_deref() {
filters.vcs = Some(EventFilter::read(spec)?);
}
Ok(filters)
}
fn read_filter(view: &RunView, args: &ReadArgs) -> Result<EventFilter> {
if args.all {
return Ok(EventFilter::default());
}
let named = args.filter.as_deref().unwrap_or(filter::DEFAULT_PROFILE);
if named.trim_start().starts_with('{') {
return EventFilter::read(named);
}
match view.launch.filters.profile(named) {
Ok(filter) => Ok(filter),
Err(unknown) => match Path::new(named).is_file() {
true => EventFilter::read(named),
false => Err(unknown),
},
}
}
fn start(args: &StartArgs) -> Result<i32> {
let store = crate::taskgraph::Store::resolve()?;
let project: crate::taskgraph::QualifiedId = args.project.parse()?;
let mut plan = store.plan(&project)?;
graph::validate(&plan)?;
let launch_dir = launch_dir()?;
let declared = match &args.launch_config {
Some(path) => crate::filter::LaunchConfig::load(path)?,
None => crate::filter::LaunchConfig::default(),
};
let graph_ref: Option<String> = match args.dag_graph.as_str() {
DAG_GRAPH_OFF => None,
reference => Some(resolve_graph(reference, &launch_dir)?),
};
let pr_author_graph_ref: Option<String> = match args
.pr_author_graph
.as_deref()
.or(declared.pr_author_graph.as_deref())
.filter(|reference| !reference.trim().is_empty())
{
Some(reference) => Some(resolve_graph(reference, &launch_dir)?),
None => None,
};
let named = match args.node_validator.clone() {
flag @ Some(_) => flag,
None => match engine::configured_node_validator()? {
variable @ Some(_) => variable,
None => declared.node_validator.clone(),
},
};
let node_validator: Option<String> = named
.map(|command| command.trim().to_string())
.filter(|command| !command.is_empty());
let asked = match args.envelope_reviewer.clone() {
flag @ Some(_) => flag,
None => match engine::configured_envelope_reviewer()? {
variable @ Some(_) => variable,
None => declared.envelope_reviewer.clone(),
},
};
let envelope_reviewer: Option<String> = asked
.map(|command| command.trim().to_string())
.filter(|command| !command.is_empty());
let node_graph_ref = resolve_graph(&engine::configured_node_graph(), &launch_dir)?;
resolve_plan_graphs(&mut plan, &launch_dir)?;
let filters = declared_filters(declared.filters, args)?;
let root = ledger::runs_root();
let run = mint_run_id(&plan, project.native(), &root);
let holders = concurrency::holders(&plan)?;
for holder in holders
.iter()
.filter(|holder| holder.state == State::Open && holder.liveness == Liveness::Stale)
{
eprintln!(
"onepipeline: stale repository holder: identity '{}' session '{}' owner_pid {}; proceeding",
holder.identity, holder.token.0, holder.owner_pid
);
}
let live: Vec<_> = holders
.iter()
.filter(|holder| holder.state == State::Open && holder.liveness == Liveness::Live)
.collect();
if !live.is_empty() && !args.acknowledge_concurrent {
let shared = live
.iter()
.map(|holder| {
format!(
"identity '{}' held by session '{}' (owner_pid {})",
holder.identity, holder.token.0, holder.owner_pid
)
})
.collect::<Vec<_>>()
.join(", ");
return Err(Error::Refused(format!(
"concurrent project work refused for run '{run}': {shared}; pass --acknowledge-concurrent to proceed deliberately"
)));
}
if !live.is_empty() {
let shared = live
.iter()
.map(|holder| {
format!(
"'{}' with session '{}' (owner_pid {})",
holder.identity, holder.token.0, holder.owner_pid
)
})
.collect::<Vec<_>>()
.join(", ");
eprintln!(
"onepipeline: --acknowledge-concurrent: launch '{run}' is proceeding alongside live run(s): {shared}"
);
}
let paths = RunPaths::under(&root, &run);
paths.create()?;
ledger::write_json(&paths.plan(), &plan)?;
let mut record = LaunchRecord {
run_id: run.clone(),
project: args.project.clone(),
dir: launch_dir.clone(),
graph: graph_ref.clone().unwrap_or_default(),
graph_run: String::new(),
observer_runs: Vec::new(),
observer_ending: String::new(),
node_graph: node_graph_ref,
pr_author_graph: pr_author_graph_ref.unwrap_or_default(),
node_validator: node_validator.unwrap_or_default(),
envelope_reviewer: envelope_reviewer.unwrap_or_default(),
launcher: sys::launcher(),
session: sys::launching_session(),
pid: 0,
host: String::new(),
started: String::new(),
started_at: sys::now_rfc3339(),
heartbeat_interval: args.heartbeat_interval,
dag_sets: args.dag_sets.clone(),
node_sets: args.node_sets.clone(),
adoptions: 0,
filters,
};
record.driven_by_this_process();
let mut open = Journal::open(&paths);
if !live.is_empty() {
open.emit(
journal::PipelineKind::ConcurrentAcknowledged,
journal::labels(&run, None),
journal::payload(&[
(
"shared_identities",
json!(live
.iter()
.map(|holder| holder.identity.to_string())
.collect::<Vec<_>>()),
),
(
"runs",
json!({
"launching": run,
"holding_sessions": live
.iter()
.map(|holder| holder.token.0.clone())
.collect::<Vec<_>>(),
}),
),
(
"holders",
json!(live
.iter()
.map(|holder| json!({
"session": holder.token.0.clone(),
"owner_pid": holder.owner_pid,
}))
.collect::<Vec<_>>()),
),
]),
)?;
}
open.emit(
journal::PipelineKind::RunStarted,
journal::labels(&run, None),
journal::payload(&[
("plan", json!(plan)),
("graph", json!(graph_ref)),
("dir", json!(launch_dir)),
("heartbeat_interval", json!(args.heartbeat_interval)),
]),
)?;
ledger::write_json(&paths.launch(), &record)?;
if args.detach {
sys::disown_standard_handles();
}
if args.detach {
let mut driver = retain_driver(&paths, Retained::Driving)?;
let pid = driver.id();
confirm_driving(&paths, &mut driver)?;
announce_launch(&run, pid);
return Ok(EXIT_SUCCESS);
}
let goal = plan.goal.as_ref().map(|goal| goal.text.clone());
let lock = engine::claim(&paths)?;
let output = agentgraph::GraphOutput::Relayed;
let mut observer = observe(&paths, &mut record, goal.as_deref(), output)?;
ledger::write_json(&paths.launch(), &record)?;
let mut watch = ObserverWatch::of(&paths, record, goal, output);
attach(&paths, observer.as_mut(), &mut watch, lock)
}
fn observe(
paths: &RunPaths,
record: &mut LaunchRecord,
goal: Option<&str>,
output: agentgraph::GraphOutput<'_>,
) -> Result<Option<agentgraph::GraphRun>> {
if record.observer_graph().is_none() {
return Ok(None);
}
let launched = launch_graph(paths, record, goal, output)?;
record.watched_by(
launched
.run_id()
.map(ToString::to_string)
.unwrap_or_default(),
);
Ok(Some(launched))
}
fn confirm_driving(paths: &RunPaths, driver: &mut std::process::Child) -> Result<()> {
let pid = driver.id();
let deadline = Instant::now() + DRIVER_HANDOVER;
while Instant::now() < deadline {
let recorded: Option<LaunchRecord> = ledger::read_json_opt(&paths.launch());
if recorded.is_some_and(|record| record.pid == pid) {
return Ok(());
}
if matches!(driver.try_wait(), Ok(Some(_)) | Err(_)) {
break;
}
std::thread::sleep(ATTACH_POLL);
}
Err(Error::Refused(format!(
"the driver retained for run '{}' did not claim it: {}",
paths.run,
driver_said(paths)
)))
}
fn driver_said(paths: &RunPaths) -> String {
let log = paths.driver_log();
let said = std::fs::read_to_string(&log).unwrap_or_default();
let tail: String = said
.lines()
.rev()
.take(DRIVER_LOG_LINES)
.collect::<Vec<_>>()
.into_iter()
.rev()
.collect::<Vec<_>>()
.join("; ");
if tail.trim().is_empty() {
return format!("it said nothing; its output is in {}", log.display());
}
format!("{tail} (its whole output is in {})", log.display())
}
fn announce_launch(run: &str, pid: u32) {
println!(
"{}",
json!({
"run_id": run,
"pid": pid,
"commands": {
"next": format!("onepipeline next {run}"),
"monitor": format!("onepipeline monitor {run}"),
},
})
);
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Retained {
Driving,
Adopting,
}
fn retain_driver(paths: &RunPaths, retained: Retained) -> Result<std::process::Child> {
let log = std::fs::OpenOptions::new()
.create(true)
.append(true)
.open(paths.driver_log())
.map_err(|source| Error::Ledger {
path: paths.driver_log(),
source,
})?;
let errors = log.try_clone().map_err(|source| Error::Ledger {
path: paths.driver_log(),
source,
})?;
let exe = std::env::current_exe().map_err(|e| {
Error::Invalid(format!(
"cannot find this executable to retain a driver: {e}"
))
})?;
let mut command = std::process::Command::new(exe);
command.arg(engine::DRIVE_VERB).arg(&paths.run);
if retained == Retained::Adopting {
command.arg(format!("--{ADOPT_FLAG}"));
}
command
.stdin(std::process::Stdio::null())
.stdout(log)
.stderr(errors)
.spawn()
.map_err(|e| Error::Invalid(format!("cannot retain a driver for '{}': {e}", paths.run)))
}
fn drive_run(args: &DriveRunArgs) -> Result<i32> {
let paths = resolve(&args.run)?;
let lock = engine::claim(&paths)?;
let mut record: LaunchRecord = ledger::read_json(&paths.launch())?;
let view = RunView::open(&paths)?;
let log = paths.driver_log();
if args.adopt {
take_the_run_over(&paths, &mut record)?;
report_and_journal_adoption(&paths, &record, &view)?;
}
let goal = view
.state
.plan
.as_ref()
.and_then(|plan| plan.goal.as_ref())
.map(|goal| goal.text.clone());
let output = agentgraph::GraphOutput::Logged(&log);
let mut observer = observe(&paths, &mut record, goal.as_deref(), output)?;
record.driven_by_this_process();
ledger::write_json(&paths.launch(), &record)?;
let settled = {
let driving = AtomicBool::new(true);
let watched = observer.as_mut();
let mut watch = ObserverWatch::of(&paths, record, goal, output);
std::thread::scope(|scope| {
scope.spawn(|| keep_the_run_watched(watched, &mut watch, &driving));
let settled = engine::drive_holding(&paths, lock);
driving.store(false, Ordering::Release);
settled
})?
};
if let Some(run) = observer.as_mut() {
run.cancel();
}
Ok(settled.exit_code())
}
fn keep_the_run_watched(
observer: Option<&mut agentgraph::GraphRun>,
watch: &mut ObserverWatch<'_>,
driving: &AtomicBool,
) {
let Some(observer) = observer else {
return;
};
while driving.load(Ordering::Acquire) {
if observer.has_exited() {
observer_stopped_watching(&watch.paths.run);
if watch.restart(observer).is_none() {
return;
}
}
std::thread::sleep(ATTACH_POLL);
}
}
fn observer_stopped_watching(run: &str) {
eprintln!(
"onepipeline: the observer graph for '{run}' has stopped watching; \
the run is still being driven"
);
}
const DEFAULT_OBSERVER_RESTARTS: u32 = 8;
const OBSERVER_RESTARTS_ENV: &str = "ONEPIPELINE_OBSERVER_RESTARTS";
fn observer_restart_limit() -> u32 {
std::env::var(OBSERVER_RESTARTS_ENV)
.ok()
.and_then(|value| value.trim().parse().ok())
.unwrap_or(DEFAULT_OBSERVER_RESTARTS)
}
struct ObserverWatch<'a> {
paths: &'a RunPaths,
record: LaunchRecord,
goal: Option<String>,
output: agentgraph::GraphOutput<'a>,
limit: u32,
restarted: u32,
}
impl<'a> ObserverWatch<'a> {
fn of(
paths: &'a RunPaths,
record: LaunchRecord,
goal: Option<String>,
output: agentgraph::GraphOutput<'a>,
) -> Self {
Self {
paths,
record,
goal,
output,
limit: observer_restart_limit(),
restarted: 0,
}
}
fn restart(&mut self, observer: &mut agentgraph::GraphRun) -> Option<()> {
if self.limit == 0 {
return None;
}
if self.restarted >= self.limit {
return self.gave_out(format!(
"this driver's bound of {} restart(s) is spent; take the run over to start \
another: onepipeline adopt {}",
self.limit, self.paths.run
));
}
let started =
match launch_graph(self.paths, &self.record, self.goal.as_deref(), self.output) {
Ok(started) => started,
Err(error) => return self.gave_out(format!("it would not start again: {error}")),
};
self.restarted += 1;
self.record.watched_by(
started
.run_id()
.map(ToString::to_string)
.unwrap_or_default(),
);
self.write_down();
eprintln!(
"onepipeline: started another observer graph for '{}' ({} of {} restart(s))",
self.paths.run, self.restarted, self.limit
);
*observer = started;
Some(())
}
fn gave_out(&mut self, reason: String) -> Option<()> {
eprintln!(
"onepipeline: no observer graph is watching '{}': {reason}",
self.paths.run
);
self.record.observer_ending = reason;
self.write_down();
None
}
fn write_down(&self) {
if let Err(error) = ledger::write_json(&self.paths.launch(), &self.record) {
eprintln!(
"onepipeline: could not record what is watching '{}': {error}",
self.paths.run
);
}
}
}
fn launch_graph(
paths: &RunPaths,
record: &LaunchRecord,
goal: Option<&str>,
output: agentgraph::GraphOutput<'_>,
) -> Result<agentgraph::GraphRun> {
let task = run_description(&paths.run, goal);
let mut launched = agentgraph::GraphRun::start(&agentgraph::Launch {
graph: &record.graph,
task: &task,
dir: &recorded_dir(record)?,
labels: &journal::labels(&paths.run, None),
env: &[
(agentgraph::RUN_ID_ENV.to_string(), paths.run.clone()),
(
ledger::RUNS_DIR_ENV.to_string(),
ledger::runs_root().to_string_lossy().into_owned(),
),
],
environment: agentgraph::Environment::Shared,
sets: &record.dag_sets,
filter: record.filters.agentgraph.as_ref(),
output,
})?;
launched.confirm_started()?;
Ok(launched)
}
fn run_description(run: &str, goal: Option<&str>) -> String {
let goal = goal.unwrap_or(crate::plan::NO_GOAL);
format!("onepipeline run `{run}`.\n\nGoal: {goal}")
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Settlement {
Complete,
AwaitingPlanner,
Unattended,
}
impl Settlement {
fn as_str(self) -> &'static str {
match self {
Self::Complete => "complete",
Self::AwaitingPlanner => "awaiting-planner",
Self::Unattended => "unattended",
}
}
fn exit_code(self) -> i32 {
match self {
Self::Unattended => EXIT_NOTHING_DRIVING,
_ => EXIT_SUCCESS,
}
}
}
fn relay_observer(
paths: &RunPaths,
run: &mut agentgraph::GraphRun,
tx: std::sync::mpsc::Sender<crate::event::Envelope>,
) -> Result<()> {
let events = run.events();
std::thread::Builder::new()
.name(format!("attach-{}", paths.run))
.spawn(move || {
for envelope in events.flatten() {
if tx.send(envelope).is_err() {
return;
}
}
})
.map(|_| ())
.map_err(|e| Error::Invalid(format!("cannot start the attach relay: {e}")))
}
fn attach(
paths: &RunPaths,
observer: Option<&mut agentgraph::GraphRun>,
watch: &mut ObserverWatch<'_>,
lock: ledger::OwnershipLock,
) -> Result<i32> {
let (tx, rx) = std::sync::mpsc::channel();
let mut watched = observer;
let mut relay = Some(tx.clone());
if let Some(run) = watched.as_deref_mut() {
relay_observer(paths, run, tx)?;
}
let mut reported = 0usize;
let mut observer_gone = false;
let mut journal = Journal::open(paths);
let driving = paths.clone();
let engine = std::thread::Builder::new()
.name(format!("engine-{}", paths.run))
.spawn(move || engine::drive_holding(&driving, lock))
.map_err(|e| Error::Invalid(format!("cannot start the engine loop: {e}")))?;
loop {
while let Ok(envelope) = rx.try_recv() {
journal.relay(&envelope)?;
}
if let Some(run) = watched.as_deref_mut() {
if run.has_exited() && !observer_gone {
observer_stopped_watching(&paths.run);
match watch.restart(run) {
Some(()) => {
if let Some(tx) = relay.as_ref() {
relay_observer(paths, run, tx.clone())?;
}
}
None => observer_gone = true,
}
}
}
let concluded = engine.is_finished();
if concluded {
relay = None;
let deadline = std::time::Instant::now() + DRAIN_GRACE;
while std::time::Instant::now() < deadline {
match rx.recv_timeout(ATTACH_POLL) {
Ok(envelope) => journal.relay(&envelope)?,
Err(std::sync::mpsc::RecvTimeoutError::Timeout) => {}
Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => break,
}
}
}
let view = RunView::open(paths)?;
let lines: Vec<String> = views::monitor(&view, &EventFilter::default())
.lines()
.map(str::to_string)
.collect();
for line in lines.iter().skip(reported) {
eprintln!("{line}");
}
reported = lines.len();
if concluded {
if let Some(run) = watched.as_deref_mut() {
run.cancel();
}
engine.join().map_err(|_| {
Error::Invalid(format!("the engine loop for '{}' panicked", paths.run))
})??;
let settlement = settlement_of(&view);
println!(
"{}",
json!({"run_id": paths.run, "settlement": settlement.as_str()})
);
return Ok(settlement.exit_code());
}
std::thread::sleep(ATTACH_POLL);
}
}
fn settlement_of(view: &RunView) -> Settlement {
let statuses = view.state.statuses();
if !statuses.is_empty() && graph::state_of(&statuses) == GraphState::Complete {
return Settlement::Complete;
}
if views::decision_outstanding(&view.state, &view.paths) {
return Settlement::AwaitingPlanner;
}
Settlement::Unattended
}
fn adopt(args: &AdoptArgs) -> Result<i32> {
let paths = resolve(&args.run)?;
let (mut record, view) = validate_and_displace_for_adoption(&paths)?;
if args.detach {
sys::disown_standard_handles();
let mut driver = retain_driver(&paths, Retained::Adopting)?;
let pid = driver.id();
confirm_driving(&paths, &mut driver)?;
announce_launch(&paths.run, pid);
return Ok(EXIT_SUCCESS);
}
let lock = engine::claim(&paths)?;
take_the_run_over(&paths, &mut record)?;
ledger::write_json(&paths.launch(), &record)?;
report_and_journal_adoption(&paths, &record, &view)?;
let goal = view
.state
.plan
.as_ref()
.and_then(|plan| plan.goal.as_ref())
.map(|goal| goal.text.clone());
let output = agentgraph::GraphOutput::Relayed;
let mut observer = observe(&paths, &mut record, goal.as_deref(), output)?;
ledger::write_json(&paths.launch(), &record)?;
let mut watch = ObserverWatch::of(&paths, record, goal, output);
attach(&paths, observer.as_mut(), &mut watch, lock)
}
fn validate_and_displace_for_adoption(paths: &RunPaths) -> Result<(LaunchRecord, RunView)> {
let session = sys::launching_session();
let record: LaunchRecord = ledger::read_json(&paths.launch())?;
if !record.owned_by(&session) {
return Err(Error::NotOwned {
run: paths.run.clone(),
owner: record.owner_label(&session),
});
}
let view = RunView::open(paths)?;
if !view.liveness().is_undriven() {
return Err(Error::Refused(format!(
"run '{}' is still being driven; end it with `onepipeline stop {}` first",
paths.run, paths.run
)));
}
displace_the_parked_driver(&record);
Ok((record, view))
}
fn take_the_run_over(paths: &RunPaths, record: &mut LaunchRecord) -> Result<()> {
record.adoptions += 1;
record.driven_by_this_process();
let previous = paths
.dir
.join(format!("launch.pre-adopt-{}.json", record.adoptions));
std::fs::copy(paths.launch(), &previous).map_err(|source| Error::Ledger {
path: previous,
source,
})?;
Ok(())
}
fn report_and_journal_adoption(
paths: &RunPaths,
record: &LaunchRecord,
view: &RunView,
) -> Result<()> {
let abandoned = view.state.sessions_in_flight();
for (node, session) in &abandoned {
eprintln!(
"onepipeline: '{node}' had a dispatch in flight; its work is on branch \
'{}' in onevcs session {}, and the node is pinned there so the run \
continues that branch rather than cutting a second one beside it",
session.branch(),
session.token().0
);
}
let mut adopted = vec![
("adoption", json!(record.adoptions)),
("pid", json!(record.pid)),
];
if !abandoned.is_empty() {
adopted.push((
journal::ADOPTED_ABANDONED,
json!(abandoned
.iter()
.map(|(node, session)| json!({
"node": node,
"session": session.token().0,
"branch": session.branch().as_str(),
}))
.collect::<Vec<_>>()),
));
}
let mut journal = Journal::open(paths);
journal.emit(
journal::PipelineKind::DriverAdopted,
journal::labels(&paths.run, None),
journal::payload(&adopted),
)
}
fn displace_the_parked_driver(record: &LaunchRecord) {
if record.host != sys::hostname() || !sys::process_may_be_live(record.pid) {
return;
}
eprintln!(
"onepipeline: run '{}' is held by driver pid {}, which is not working; \
ending it to adopt the run",
record.run_id, record.pid
);
sys::stop(record.pid, sys::Stop::Politely);
let deadline = Instant::now() + DRIVER_HANDOVER;
while Instant::now() < deadline && sys::process_may_be_live(record.pid) {
std::thread::sleep(ATTACH_POLL);
}
}
fn stop(args: &StopArgs) -> Result<i32> {
let paths = resolve(&args.run)?;
let session = sys::launching_session();
let record: LaunchRecord = ledger::read_json(&paths.launch())?;
let owner = record.owner_label(&session);
if !record.owned_by(&session) {
if !args.force {
return Err(Error::NotOwned {
run: paths.run.clone(),
owner,
});
}
eprintln!(
"onepipeline: run '{}' belongs to {owner}; stopping it anyway",
paths.run
);
}
let teardown = terminate(&paths, &record).map_err(|why| {
Error::Refused(format!(
"run '{}' was not stopped: this build cannot establish what it is running — {why}. \
The run is untouched; nothing was signalled. Fix or remove the entry the path \
above names and run `onepipeline stop {}` again",
paths.run, paths.run
))
})?;
let established = match teardown {
None => journal::StopTeardown::Elsewhere,
Some(sys::Teardown::Signalled) => journal::StopTeardown::Signalled,
Some(sys::Teardown::NothingToStop) => journal::StopTeardown::NothingToStop,
Some(sys::Teardown::IdentityDeclined) => journal::StopTeardown::IdentityDeclined,
Some(sys::Teardown::NotAttempted) => journal::StopTeardown::NotAttempted,
Some(sys::Teardown::PartlySignalled) => journal::StopTeardown::PartlySignalled,
#[cfg(unix)]
Some(sys::Teardown::Refused) => journal::StopTeardown::Refused,
};
let mut journal = Journal::open(&paths);
journal.emit(
journal::PipelineKind::RunStopped,
journal::labels(&paths.run, None),
journal::payload(&[
("owner", json!(owner)),
("forced", json!(args.force)),
(journal::STOP_TEARDOWN, json!(established)),
]),
)?;
let run = &paths.run;
match teardown {
Some(sys::Teardown::NotAttempted) => {
return Err(Error::Refused(format!(
"run '{run}' was not stopped: this host gave no answer its tree could be \
read from — no process listing, or nothing that says whether a pid it \
recorded is still the process it named, each said above — so the \
processes the run started could not be found, and ending its driver \
alone would have orphaned them. The run is untouched — run \
`onepipeline stop {run}` again once this host answers"
)));
}
Some(sys::Teardown::PartlySignalled) => {
return Err(Error::Refused(format!(
"run '{run}' was only partly stopped: part of its process tree was \
signalled and at least one process in it is still running — one this \
session could not signal, or one that took the ask and stayed. Find it \
in this host's process list and end it as the user that owns it"
)));
}
Some(sys::Teardown::IdentityDeclined) => {
return Err(Error::Refused(format!(
"run '{run}' was not stopped: live processes were found, but every recorded \
identity disagreed with the process now holding its pid, so none was safe \
to signal. This is distinct from a run with nothing left to stop; inspect \
the declined claims above and retry only after correcting the run records"
)));
}
#[cfg(unix)]
Some(sys::Teardown::Refused) => {
return Err(Error::Refused(format!(
"run '{run}' was not stopped: its process tree was found and every \
process in it refused this session's signal, so nothing was signalled \
and all of it is still running. Running `onepipeline stop {run}` again \
as this user will be refused the same way — find the tree in this \
host's process list and end it as the user that owns it"
)));
} None | Some(sys::Teardown::Signalled) | Some(sys::Teardown::NothingToStop) => {}
}
println!(
"{}",
json!({
"run_id": paths.run,
"stopped": true,
"owner": owner,
journal::STOP_TEARDOWN: established,
})
);
Ok(EXIT_SUCCESS)
}
fn terminate(paths: &RunPaths, record: &LaunchRecord) -> Result<Option<sys::Teardown>> {
let Aim::Here {
roots,
unproven,
declined,
} = roots_to_stop(paths, record)?
else {
return Ok(None);
};
if roots.is_empty() && unproven.is_empty() && !declined.is_empty() {
return Ok(Some(sys::Teardown::IdentityDeclined));
}
let established = sys::stop_and_confirm(&roots, sys::Stop::Politely, TEARDOWN_PATIENCE);
if unproven.is_empty() {
return Ok(Some(established));
}
Ok(Some(match established {
sys::Teardown::NothingToStop | sys::Teardown::NotAttempted => sys::Teardown::NotAttempted,
sys::Teardown::IdentityDeclined => sys::Teardown::IdentityDeclined,
sys::Teardown::Signalled | sys::Teardown::PartlySignalled => sys::Teardown::PartlySignalled,
#[cfg(unix)]
sys::Teardown::Refused => sys::Teardown::Refused,
}))
}
#[derive(Debug, PartialEq, Eq)]
enum Aim {
Elsewhere,
Here {
roots: Vec<u32>,
unproven: Vec<u32>,
declined: Vec<u32>,
},
}
fn roots_to_stop(paths: &RunPaths, record: &LaunchRecord) -> Result<Aim> {
let here = sys::hostname();
let mut on_this_host = false;
let mut roots: Vec<u32> = Vec::new();
let mut unproven: Vec<u32> = Vec::new();
let mut declined: Vec<u32> = Vec::new();
let claimed = std::iter::once((
RECORDED_DRIVER,
record.pid,
record.host.clone(),
record.started.clone(),
))
.chain(lock_held_on(paths).map(|held| (LOCK_HOLDER, held.pid, held.host, held.started)))
.chain(ledger::dispatches_of(paths)?.into_iter().map(|running| {
(
REGISTERED_DISPATCH,
running.pid,
running.host,
running.started,
)
}));
for (named_by, pid, host, started) in claimed {
if host != here {
continue;
}
on_this_host = true;
if roots.contains(&pid) || unproven.contains(&pid) {
continue;
}
match claim_on(pid, &started) {
Claim::Proved => roots.push(pid),
Claim::Gone => {}
Claim::Reissued => {
eprintln!(
"onepipeline: run '{}': the {named_by} names pid {pid}, which this host has \
since given to another process, so it was not signalled",
paths.run
);
declined.push(pid);
}
Claim::Unstamped => {
left_alone(
&paths.run,
named_by,
pid,
"its record carries no start token",
);
unproven.push(pid);
}
Claim::HostSilent => {
left_alone(
&paths.run,
named_by,
pid,
"this host will not say when it started",
);
unproven.push(pid);
}
}
}
if !on_this_host {
return Ok(Aim::Elsewhere);
}
Ok(Aim::Here {
roots,
unproven,
declined,
})
}
fn left_alone(run: &str, named_by: &str, pid: u32, why: &str) {
eprintln!(
"onepipeline: run '{run}': the {named_by} names pid {pid}, which is running on this \
host — {why}, so nothing says it is still this run's process and it was not signalled"
);
}
const RECORDED_DRIVER: &str = "launch record";
const LOCK_HOLDER: &str = "ownership lock";
const REGISTERED_DISPATCH: &str = "dispatch registry";
#[derive(Debug, PartialEq, Eq)]
enum Claim {
Proved,
Gone,
Reissued,
Unstamped,
HostSilent,
}
fn claim_on(pid: u32, started: &str) -> Claim {
let reading = sys::process_start_token(pid);
match reading {
Some(ref token) if token.matches(started) => Claim::Proved,
_ if !sys::process_may_be_live(pid) => Claim::Gone,
Some(_) if !started.is_empty() => Claim::Reissued,
Some(_) => Claim::Unstamped,
None => Claim::HostSilent,
}
}
fn lock_held_on(paths: &RunPaths) -> Option<ledger::LockRecord> {
let path = paths.lock();
match ledger::read_json::<ledger::LockRecord>(&path) {
Ok(held) => Some(held),
Err(Error::Ledger { source, .. }) if source.kind() == std::io::ErrorKind::NotFound => None,
Err(error) => {
eprintln!(
"onepipeline: the ownership lock of run '{}' cannot be read, so this stop aims \
only at the driver the launch record names: {error}",
paths.run
);
None
}
}
}
fn next(args: &ReadArgs) -> Result<i32> {
let paths = resolve(&args.run)?;
let view = RunView::open(&paths)?;
let filter = read_filter(&view, args)?;
let events = views::shaped(&view, &filter);
let channel = ChannelState::new(&paths);
let Some(surface) = channel.claim()? else {
let settled = view.liveness().is_undriven();
let status = if settled { "finished" } else { "running" };
println!(
"{}",
json!({"status": status, "surface": null, "events": events})
);
return Ok(EXIT_SUCCESS);
};
let mut journal = Journal::open(&paths);
journal.emit(
journal::PipelineKind::PlannerSurfaced,
journal::labels(&paths.run, surface.workstream.as_deref()),
journal::payload(&[
("kind", json!(surface.kind)),
("message", json!(surface.message)),
("source", json!(surface.source)),
("blocking", json!(surface.blocking)),
]),
)?;
if let Err(error) = agentgraph::recorded_graph_run(&view.launch.graph_run, &paths.run)
.and_then(|graph_run| agentgraph::reset_timer(&graph_run, agentgraph::CHECK_IN_MEMBER))
{
eprintln!("onepipeline: could not reset the check-in pacemaker: {error}");
}
println!(
"{}",
json!({"status": "surface", "surface": surface, "events": events})
);
Ok(EXIT_SUCCESS)
}
fn surface_message(args: &SurfaceArgs) -> Result<String> {
let (body, whence) = match (&args.message, &args.file) {
(Some(message), _) => (message.clone(), "`--message`"),
(None, Some(path)) => (
std::fs::read_to_string(path).map_err(|e| Error::Ledger {
path: path.clone(),
source: e,
})?,
"the file",
),
(None, None) => {
let mut buffer = String::new();
std::io::stdin()
.read_to_string_compat(&mut buffer)
.map_err(|e| Error::Refused(format!("cannot read the message from stdin: {e}")))?;
(buffer, "stdin")
}
};
let body = body.trim();
if body.is_empty() {
return Err(Error::Refused(format!(
"a surface carries what it has to say and {whence} carried nothing; \
give the message on stdin, as a file argument, or with `--message TEXT`"
)));
}
Ok(body.to_string())
}
fn surface(args: &SurfaceArgs) -> Result<i32> {
let paths = resolve(&args.run)?;
let message = surface_message(args)?;
let source = match args.kind {
SurfaceKind::CheckIn => crate::channel::source::CHECK_IN,
SurfaceKind::Finding => crate::channel::source::PROPOSAL,
};
let queued = ChannelState::new(&paths).push(Surface {
id: 0,
kind: args.kind.as_str().to_string(),
message,
source: source.to_string(),
blocking: false,
queued_at: sys::now_millis(),
abandoned: false,
asker: None,
workstream: None,
})?;
let mut journal = Journal::open(&paths);
journal.emit(
journal::PipelineKind::PlannerSurfaceQueued,
journal::labels(&paths.run, None),
journal::payload(&[
("kind", json!(queued.kind)),
("message", json!(queued.message)),
("source", json!(queued.source)),
("blocking", json!(false)),
]),
)?;
println!("{}", json!({"surface": queued.id, "state": "queued"}));
Ok(EXIT_SUCCESS)
}
fn attest(args: &AttestArgs) -> Result<i32> {
submit(
&resolve(&args.run)?,
&Reply {
version: Some(crate::channel::REPLY_ENVELOPE_VERSION),
author: Author::Planner,
commands: vec![Command::Attest {
reference: args.reference.clone(),
}],
..Reply::default()
},
)
}
fn reply(args: &ReplyArgs) -> Result<i32> {
let paths = resolve(&args.run)?;
let text = match &args.file {
Some(path) => std::fs::read_to_string(path).map_err(|e| Error::Ledger {
path: path.clone(),
source: e,
})?,
None => {
let mut buffer = String::new();
std::io::stdin()
.read_to_string_compat(&mut buffer)
.map_err(|e| Error::Refused(format!("cannot read the reply from stdin: {e}")))?;
buffer
}
};
let envelope: Reply = serde_json::from_str(text.trim()).map_err(|e| {
let why = serde_json::from_str::<serde_json::Value>(text.trim())
.ok()
.as_ref()
.and_then(crate::plan::retired_field_refusal)
.unwrap_or_else(|| e.to_string());
Error::Refused(format!("the reply is malformed: {why}"))
})?;
submit(&paths, &envelope)
}
enum Submitted {
Answered {
reply: u64,
},
AppliedHere {
operations: Vec<edits::Operation>,
},
AppliedByRun {
reply: u64,
},
Queued {
reply: u64,
},
}
fn submit(paths: &RunPaths, envelope: &Reply) -> Result<i32> {
match submit_envelope(paths, envelope)? {
Submitted::Answered { reply } => {
println!("{}", json!({"reply": reply, "state": "delivered"}));
Ok(EXIT_SUCCESS)
}
Submitted::AppliedHere { .. } => {
println!("{}", json!({"reply": 0, "state": "applied"}));
Ok(EXIT_SUCCESS)
}
Submitted::AppliedByRun { reply } => {
println!("{}", json!({"reply": reply, "state": "applied"}));
Ok(EXIT_SUCCESS)
}
Submitted::Queued { reply } => {
println!("{}", json!({"reply": reply, "state": "queued"}));
Ok(EXIT_QUEUED)
}
}
}
pub(crate) fn deliver_note_envelope(
paths: &RunPaths,
envelope: &Reply,
) -> Result<crate::note::Delivered> {
let [Command::Note { id, text, .. }] = &envelope.commands[..] else {
return Err(Error::Refused(
"a note is delivered one at a time, and this envelope carries something else"
.to_string(),
));
};
let (id, text) = (id.clone(), text.clone());
match submit_envelope(paths, envelope)? {
Submitted::AppliedHere { operations } => reached_in(&operations)
.map(crate::note::Delivered::To)
.ok_or_else(|| {
Error::Refused(format!(
"note: node '{id}': the delivery recorded no disposition"
))
}),
Submitted::AppliedByRun { .. } => last_note_delivered(paths, &id, &text)?
.map(crate::note::Delivered::To)
.ok_or_else(|| {
Error::Refused(format!(
"note: node '{id}': the run applied the note and recorded no disposition \
for it"
))
}),
Submitted::Queued { .. } => Ok(crate::note::Delivered::Queued),
Submitted::Answered { .. } => Err(Error::Refused(
"a note carries a command, and this envelope was answered as a verdict".to_string(),
)),
}
}
fn reached_in(operations: &[edits::Operation]) -> Option<crate::note::Reached> {
operations.iter().find_map(|operation| match operation {
edits::Operation::NoteDelivered { reached, .. } => Some(reached.clone()),
_ => None,
})
}
fn last_note_delivered(
paths: &RunPaths,
node: &str,
text: &crate::note::NoteText,
) -> Result<Option<crate::note::Reached>> {
for envelope in journal::read(&paths.journal()).into_iter().rev() {
if envelope.kind.0 != journal::PipelineKind::EditCommitted.as_str() {
continue;
}
let Some(recorded) = envelope.payload.get("operations") else {
continue;
};
let operations: Vec<edits::Operation> =
serde_json::from_value(recorded.clone()).map_err(|why| {
Error::Invalid(format!(
"run '{}': a committed edit at seq {} records operations this build does \
not read, so what it did with a note cannot be said: {why}",
paths.run, envelope.seq
))
})?;
let reached = operations.iter().find_map(|operation| match operation {
edits::Operation::NoteDelivered {
node: written,
text: said,
reached,
..
} if written == node && said == text => Some(reached.clone()),
_ => None,
});
if reached.is_some() {
return Ok(reached);
}
}
Ok(None)
}
fn submit_envelope(paths: &RunPaths, envelope: &Reply) -> Result<Submitted> {
let view = RunView::open(paths)?;
let channel = ChannelState::new(paths);
if envelope.commands.is_empty() {
crate::channel::allows_completion(envelope.author, envelope.completion)?;
if channel.pending().is_none() && crate::views::has_settled(&view) {
return Err(Error::Refused(format!(
"run '{}' has settled, so nothing will ever read a reply to it; \
no reply was queued",
paths.run
)));
}
let id = channel.answer(envelope)?;
let mut journal = Journal::open(paths);
journal.emit(
journal::PipelineKind::PlannerReplied,
journal::labels(&paths.run, None),
journal::payload(&[
("author", json!(envelope.author)),
("completion", json!(envelope.completion)),
("reason", json!(envelope.reason)),
]),
)?;
if let Some(reason) = &envelope.reason {
if envelope.completion == Some(true) {
journal.emit(
journal::PipelineKind::CompletionRequested,
journal::labels(&paths.run, None),
journal::payload(&[("reason", json!(reason))]),
)?;
}
}
return Ok(Submitted::Answered { reply: id });
}
if envelope.version != Some(crate::channel::REPLY_ENVELOPE_VERSION) {
return Err(Error::Refused(format!(
"an edit envelope requires version {}",
crate::channel::REPLY_ENVELOPE_VERSION
)));
}
for command in &envelope.commands {
crate::channel::allows(envelope.author, command)?;
}
let mut projected = view.state.graph.clone();
let frontier = Frontier {
node_validator: view.launch.node_validator().map(str::to_owned),
..view.state.frontier()
};
let mut checking = frontier.clone();
for command in &envelope.commands {
let operations = edits::compile(&mut projected, &checking, envelope.author, command)?;
edits::advance(&mut checking, &operations);
}
edits::offer_envelope_to_reviewer(
view.launch.envelope_reviewer(),
&envelope.commands,
&projected,
view.state.plan.as_ref(),
)?;
match ledger::OwnershipLock::acquire(paths, "reply") {
Ok(lock) => {
let mut journal = Journal::open(paths);
let mut graph = view.state.graph.clone();
let mut compiled: Vec<edits::Operation> = Vec::new();
let mut frontier = frontier.clone();
for command in &envelope.commands {
let operations = match apply_here(
paths,
&mut journal,
&mut graph,
&frontier,
envelope.author,
command,
) {
Ok(operations) => operations,
Err(error) => {
lock.release();
return Err(error);
}
};
edits::advance(&mut frontier, &operations);
compiled.extend(operations.iter().cloned());
journal.emit(
journal::PipelineKind::EditCommitted,
journal::labels(&paths.run, None),
journal::payload(&[
("author", json!(envelope.author)),
("command", json!(command)),
("operations", json!(operations)),
]),
)?;
engine::record_operation_facts(paths, &mut journal, envelope.author, &operations)?;
if envelope.author == Author::Monitor {
if let Some(raised) = engine::monitor_edit(command) {
engine::raise(paths, &mut journal, raised)?;
}
}
}
lock.release();
channel.answer_if_verdict(envelope)?;
Ok(Submitted::AppliedHere {
operations: compiled,
})
}
Err(Error::Locked { .. }) => {
let id = channel.submit(envelope.author, &envelope.commands)?;
let deadline = Instant::now() + Duration::from_secs(reply_timeout_seconds());
while Instant::now() < deadline {
if let Some(outcome) = channel.outcome_of(id) {
channel.answer_if_verdict(envelope)?;
if outcome.applied {
return Ok(Submitted::AppliedByRun { reply: id });
}
return Err(Error::Refused(
outcome
.reason
.unwrap_or_else(|| "the reconciler rejected the edit".into()),
));
}
std::thread::sleep(ATTACH_POLL);
}
channel.answer_if_verdict(envelope)?;
Ok(Submitted::Queued { reply: id })
}
Err(other) => Err(other),
}
}
fn apply_here(
paths: &RunPaths,
journal: &mut Journal,
graph: &mut crate::graph::Graph,
frontier: &Frontier,
author: Author,
command: &Command,
) -> Result<Vec<edits::Operation>> {
let operations = edits::compile(graph, frontier, author, command)?;
let Command::Note {
id,
addressee,
text,
criterion,
deliver,
persist,
} = command
else {
return Ok(operations);
};
let offered = engine::Offered {
id,
addressee: *addressee,
text,
criterion: criterion.as_ref(),
reach: crate::note::Reach::of(id, *deliver, *persist)?,
dispatchable: frontier.recorded.get(id) != Some(&crate::graph::NodeStatus::Done),
};
match engine::deliver_manager_note(paths, &offered, None) {
Ok(operations) => Ok(operations),
Err(error) => {
engine::record_rejection(paths, journal, author, command, &error)?;
Err(error)
}
}
}
fn reply_timeout_seconds() -> u64 {
std::env::var(crate::channel::REPLY_TIMEOUT_ENV)
.ok()
.and_then(|value| value.parse().ok())
.filter(|seconds| *seconds > 0)
.unwrap_or(crate::channel::DEFAULT_REPLY_TIMEOUT_SECONDS)
}
fn serve_session_deadline() -> Result<Option<Instant>> {
let key = crate::channel::SERVE_SESSION_ENV;
let Some(value) = std::env::var_os(key) else {
return Ok(None);
};
let value = value.to_string_lossy().into_owned();
let seconds = value
.trim()
.parse::<u64>()
.ok()
.filter(|seconds| *seconds > 0)
.ok_or_else(|| {
Error::Refused(format!(
"{key} is a whole number of seconds greater than zero, and this session was \
given '{value}'; leave it unset for a session that serves until its member's \
frame stream ends"
))
})?;
Instant::now()
.checked_add(Duration::from_secs(seconds))
.map(Some)
.ok_or_else(|| {
Error::Refused(format!(
"{key} of {seconds} seconds is further ahead than this host's clock can name"
))
})
}
fn serve_asker() -> Result<Option<crate::channel::Asker>> {
std::env::var_os(crate::channel::ASKER_ENV)
.map(|value| crate::channel::Asker::named(&value))
.transpose()
}
enum Served {
StreamEnded,
Completed,
SessionOver,
}
fn serve(args: &RunArgs) -> Result<i32> {
let paths = resolve(&args.run)?;
let channel = ChannelState::new(&paths);
let mut raised: Vec<u64> = Vec::new();
let asker = serve_asker()?;
let session_deadline = serve_session_deadline()?;
if let Some(asker) = &asker {
raised.extend(channel.attend(asker)?.into_iter().map(|surface| surface.id));
}
let mut ending = Served::StreamEnded;
let (frames, arriving) = std::sync::mpsc::channel();
std::thread::spawn(move || {
for line in std::io::stdin().lock().lines() {
if frames.send(line).is_err() {
break;
}
}
});
loop {
let waiting_for = match session_deadline {
Some(deadline) => {
let left = deadline.saturating_duration_since(Instant::now());
if left.is_zero() {
ending = Served::SessionOver;
break;
}
Some(left)
}
None => None,
};
let line = match waiting_for {
Some(left) => match arriving.recv_timeout(left) {
Ok(line) => line,
Err(std::sync::mpsc::RecvTimeoutError::Timeout) => {
ending = Served::SessionOver;
break;
}
Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => break,
},
None => match arriving.recv() {
Ok(line) => line,
Err(_) => break,
},
};
let line = line.map_err(|e| Error::Sibling {
tool: "oneagentgraph",
message: format!("the observer's frame stream could not be read: {e}"),
})?;
if line.trim().is_empty() {
continue;
}
let frame: ObserverFrame = serde_json::from_str(line.trim())
.map_err(|e| Error::Refused(format!("the observer emitted a bad frame: {e}")))?;
if let Some(node) = &frame.node {
let graph = RunView::open(&paths)?.state.graph;
if !graph.contains(node) {
return Err(Error::Refused(format!(
"the observer raised a frame about node '{node}', which run '{}' does not \
have; it has: {}",
paths.run,
graph.ids().cloned().collect::<Vec<_>>().join(", ")
)));
}
}
let queued = channel.push(Surface {
id: 0,
kind: frame.kind,
message: frame.message,
source: crate::channel::source::PROPOSAL.to_string(),
blocking: frame.blocking,
queued_at: sys::now_millis(),
abandoned: false,
asker: asker.clone(),
workstream: frame.node,
})?;
let mut journal = Journal::open(&paths);
journal.emit(
journal::PipelineKind::PlannerSurfaceQueued,
journal::labels(&paths.run, queued.workstream.as_deref()),
journal::payload(&[
("kind", json!(queued.kind)),
("message", json!(queued.message)),
("source", json!(queued.source)),
("blocking", json!(queued.blocking)),
]),
)?;
raised.push(queued.id);
let answer = wait_for_reply(&channel)?;
println!(
"{}",
serde_json::to_string(&answer).map_err(|e| Error::Invalid(format!("verdict: {e}")))?
);
std::io::stdout()
.flush()
.map_err(|e| Error::Refused(format!("cannot write the verdict: {e}")))?;
if answer.completion == Some(true) {
ending = Served::Completed;
break;
}
}
if matches!(ending, Served::SessionOver) {
eprintln!(
"onepipeline: this channel session reached its {} bound with the observer's stream \
still open; {}",
crate::channel::SERVE_SESSION_ENV,
match raised.len() {
0 => "it had raised nothing".to_owned(),
left => format!(
"the {left} surface(s) it raised stay in the queue, still waiting for an \
answer"
),
}
);
}
if matches!(ending, Served::StreamEnded | Served::Completed) {
channel.abandon(&raised)?;
}
Ok(EXIT_SUCCESS)
}
#[derive(Debug, serde::Deserialize)]
#[serde(deny_unknown_fields)]
struct ObserverFrame {
kind: String,
message: String,
#[serde(default = "blocking_by_default")]
blocking: bool,
#[serde(default)]
node: Option<String>,
}
fn blocking_by_default() -> bool {
true
}
fn wait_for_reply(channel: &ChannelState) -> Result<Reply> {
let deadline = Instant::now() + Duration::from_secs(reply_timeout_seconds());
while Instant::now() < deadline {
if let Some(claimed) = channel.claim_reply()? {
return Ok(claimed.reply);
}
std::thread::sleep(ATTACH_POLL);
}
Ok(Reply {
completion: Some(false),
message: Some("no planner reply within the timeout; continue".into()),
reason: Some("the channel timed out waiting for a verdict".into()),
..Reply::default()
})
}
fn runs(args: &RunsArgs) -> Result<i32> {
print!(
"{}",
views::runs(&ledger::runs_root(), args.mine, &sys::launching_session())
);
Ok(EXIT_SUCCESS)
}
fn report(args: &OptionalRunArgs, render: fn(&views::Survey) -> String) -> Result<i32> {
let survey = match &args.run {
Some(run) => views::Survey::of_one(RunView::open(&resolve(run)?)?),
None => views::Survey::of(&ledger::runs_root()),
};
print!("{}", render(&survey));
Ok(EXIT_SUCCESS)
}
fn transcript(args: &TranscriptArgs) -> Result<i32> {
let view = RunView::open(&resolve(&args.run)?)?;
if let Some(node) = &args.node {
if views::nodes_with_agent_records(&view, Some(node)).is_empty() {
let recorded = views::nodes_with_agent_records(&view, None);
return Err(Error::Refused(format!(
"run '{}' has recorded nothing for node '{node}'; it has records for: {}",
args.run,
if recorded.is_empty() {
"nothing yet".to_string()
} else {
recorded.join(", ")
}
)));
}
}
print!("{}", views::transcript(&view, args.node.as_deref()));
Ok(EXIT_SUCCESS)
}
fn report_telemetry(args: &TelemetryArgs) -> Result<i32> {
let survey = match &args.run {
Some(run) => views::Survey::of_one(RunView::open(&resolve(run)?)?),
None => views::Survey::of(&ledger::runs_root()),
};
for view in &survey.views {
let aggregated = telemetry::of_run(&view.paths, &view.events);
if args.breakdown {
print!("{}", telemetry::render_breakdown(&aggregated));
} else {
println!(
"{}",
serde_json::to_string(&aggregated)
.map_err(|e| Error::Invalid(format!("telemetry: {e}")))?
);
}
}
Ok(EXIT_SUCCESS)
}
trait ReadToStringCompat {
fn read_to_string_compat(&self, buffer: &mut String) -> std::io::Result<usize>;
}
impl ReadToStringCompat for std::io::Stdin {
fn read_to_string_compat(&self, buffer: &mut String) -> std::io::Result<usize> {
use std::io::Read;
self.lock().read_to_string(buffer)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::plan::{Node, PLAN_SCHEMA_VERSION};
use crate::views::DriverLiveness;
use std::path::PathBuf;
fn plan(name: Option<&str>) -> Plan {
Plan {
schema_version: PLAN_SCHEMA_VERSION,
goal: None,
name: name.map(str::to_string),
concurrency: 4,
tasks: vec![Node {
id: "build".into(),
persona: Some("engineer".into()),
task: Some("## What\ndo it".into()),
..Node::default()
}],
}
}
const ROLE_PROSE: &[&str] = &[
"Drive",
"drive",
"to settlement",
"observe",
"Observe",
"nothing else",
"run state",
"you",
"You",
];
fn assert_role_neutral(task: &str) {
for prose in ROLE_PROSE {
assert!(
!task.contains(prose),
"the launched graph's task tells a member what to do ({prose:?}): {task}"
);
}
}
#[test]
fn the_launched_graphs_task_names_the_run_and_its_goal_and_no_role() {
let task = run_description("tracked-release", Some("close the coverage gap"));
assert!(
task.contains("tracked-release"),
"the task does not name the run: {task}"
);
assert!(
task.contains("close the coverage gap"),
"the task does not say what the run is for: {task}"
);
assert_role_neutral(&task);
}
#[test]
fn a_run_whose_plan_states_no_goal_says_so_in_the_same_shape() {
let task = run_description("nameless", None);
assert!(
task.contains("nameless") && task.contains(crate::plan::NO_GOAL),
"the task for a goalless run reads: {task}"
);
assert_role_neutral(&task);
}
fn scratch(name: &str) -> PathBuf {
let dir = std::env::temp_dir().join(format!("onepipeline-driver-{name}-{}", sys::pid()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).expect("a scratch root");
dir
}
#[test]
fn a_run_id_comes_from_the_plans_name_and_is_made_unique() {
let root = scratch("mint");
let project = "plans:release";
assert_eq!(
mint_run_id(&plan(Some("tracked-release")), project, &root),
"tracked-release"
);
std::fs::create_dir_all(root.join("tracked-release")).expect("an existing run");
assert_eq!(
mint_run_id(&plan(Some("tracked-release")), project, &root),
"tracked-release-2"
);
std::fs::remove_dir_all(&root).ok();
}
#[test]
fn a_nameless_plan_takes_its_run_id_from_the_project_it_was_launched_by() {
let root = scratch("mint-project");
assert_eq!(mint_run_id(&plan(None), "release", &root), "release");
assert_eq!(mint_run_id(&plan(None), "odd name!", &root), "odd-name-");
std::fs::remove_dir_all(&root).ok();
}
#[test]
fn every_settlement_carries_the_exit_code_the_contract_assigns() {
assert_eq!(Settlement::Complete.exit_code(), EXIT_SUCCESS);
assert_eq!(Settlement::AwaitingPlanner.exit_code(), EXIT_SUCCESS);
assert_eq!(Settlement::Unattended.exit_code(), EXIT_NOTHING_DRIVING);
assert_eq!(Settlement::Complete.as_str(), "complete");
assert_eq!(Settlement::AwaitingPlanner.as_str(), "awaiting-planner");
assert_eq!(Settlement::Unattended.as_str(), "unattended");
}
#[test]
fn the_launched_graphs_task_never_asks_a_member_to_drive_the_engine() {
let task = run_description("tracked-release", Some("close the coverage gap"));
for verb in ["round run", "round next", "drive-run"] {
assert!(
!task.contains(verb),
"the observer's task names {verb}: {task}"
);
}
}
#[test]
fn an_undriven_run_is_the_settlement_a_planner_must_intervene_in() {
assert!(DriverLiveness::DriverDead.is_undriven());
assert!(DriverLiveness::Parked.is_undriven());
assert!(!DriverLiveness::Driving.is_undriven());
}
#[test]
fn the_reply_timeout_falls_back_when_the_environment_is_unusable() {
assert!(reply_timeout_seconds() > 0);
}
fn launched_by(pid: u32, host: &str, started: &str) -> LaunchRecord {
LaunchRecord {
run_id: "stopped".into(),
project: "plans:demo".into(),
dir: PathBuf::from("/tmp/launch"),
graph: String::new(),
graph_run: String::new(),
observer_runs: Vec::new(),
observer_ending: String::new(),
node_graph: "graphs/node-scope.yaml".into(),
pr_author_graph: String::new(),
node_validator: String::new(),
envelope_reviewer: String::new(),
launcher: "e2e".into(),
session: "session-a".into(),
pid,
host: host.to_string(),
started: started.to_string(),
started_at: sys::now_rfc3339(),
heartbeat_interval: 1_800,
dag_sets: Vec::new(),
node_sets: Vec::new(),
adoptions: 0,
filters: crate::filter::Filters::default(),
}
}
#[test]
fn a_stop_aims_at_every_stamped_claim_the_run_holds_and_never_at_a_pid_nothing_proves() {
let root = scratch("roots");
let paths = RunPaths::under(&root, "stopped");
paths.create().expect("the run directory");
let here = sys::hostname();
let dead = sys::reaped_pid();
let stamp = |started: &str| ledger::LockRecord {
pid: sys::pid(),
host: here.clone(),
acquired_at: sys::now_rfc3339(),
verb: "drive".into(),
started: started.to_string(),
};
let held = |record: &ledger::LockRecord| {
ledger::write_json(&paths.lock(), record).expect("a held lock");
};
let proven = sys::process_start_token(sys::pid())
.expect("this host says when a process started")
.recorded()
.to_string();
let aimed_at =
|record: &LaunchRecord| roots_to_stop(&paths, record).expect("this run's records read");
let roots = |record: &LaunchRecord| match aimed_at(record) {
Aim::Here { roots, .. } => roots,
Aim::Elsewhere => panic!("a run this host's own records name read as another host's"),
};
assert_eq!(
aimed_at(&launched_by(dead, &here, &proven)),
Aim::Here {
roots: Vec::new(),
unproven: Vec::new(),
declined: Vec::new(),
}
);
assert_eq!(
roots(&launched_by(sys::pid(), &here, &proven)),
vec![sys::pid()]
);
assert_eq!(
aimed_at(&launched_by(
sys::pid(),
&here,
"the driver it named, which is not this process",
)),
Aim::Here {
roots: Vec::new(),
unproven: Vec::new(),
declined: vec![sys::pid()],
},
"a stop aimed at a pid the host has since given to another process"
);
assert_eq!(
aimed_at(&launched_by(sys::pid(), &here, "")),
Aim::Here {
roots: Vec::new(),
unproven: vec![sys::pid()],
declined: Vec::new(),
}
);
held(&stamp(&proven));
assert_eq!(
roots(&launched_by(dead, &here, &proven)),
vec![sys::pid()],
"a stop did not aim at the process the lock stamps as driving the run"
);
held(&stamp("the process that took it, which is not this one"));
assert!(
roots(&launched_by(dead, &here, &proven)).is_empty(),
"a stop aimed at a pid the lock's own stamp disowns"
);
held(&stamp(""));
assert_eq!(
aimed_at(&launched_by(dead, &here, &proven)),
Aim::Here {
roots: Vec::new(),
unproven: vec![sys::pid()],
declined: Vec::new(),
}
);
held(&ledger::LockRecord {
host: "a-host-this-is-not".into(),
..stamp(&proven)
});
assert!(roots(&launched_by(dead, &here, &proven)).is_empty());
std::fs::write(paths.lock(), "not json at all").expect("a lock nobody can read");
assert!(roots(&launched_by(dead, &here, &proven)).is_empty());
assert_eq!(
aimed_at(&launched_by(dead, "a-host-this-is-not", &proven)),
Aim::Elsewhere
);
std::fs::remove_file(paths.lock()).expect("the lock is given up");
let running = |pid: u32, host: &str, started: &str| ledger::DispatchRecord {
node: "build".into(),
pid,
host: host.to_string(),
dispatched_at: sys::now_rfc3339(),
started: started.to_string(),
};
let recorded = |record: &ledger::DispatchRecord| {
ledger::write_json(&paths.dispatch(record.pid, 0), record)
.expect("a recorded dispatch");
};
recorded(&running(sys::pid(), &here, &proven));
assert_eq!(
roots(&launched_by(dead, &here, &proven)),
vec![sys::pid()],
"a stop did not aim at the process the registry says the run's work is in"
);
for disowned in [
running(
sys::pid(),
&here,
"the process that took it, which is not this one",
),
running(sys::pid(), "a-host-this-is-not", &proven),
] {
recorded(&disowned);
assert!(
roots(&launched_by(dead, &here, &proven)).is_empty(),
"a stop aimed at a pid the registry cannot prove: {disowned:?}"
);
}
recorded(&running(sys::pid(), &here, &proven));
held(&stamp(&proven));
assert_eq!(
roots(&launched_by(sys::pid(), &here, &proven)),
vec![sys::pid()]
);
std::fs::remove_dir_all(&root).ok();
}
#[test]
fn a_registry_this_build_cannot_read_refuses_the_stop_rather_than_narrowing_it() {
let root = scratch("roots-unreadable");
let paths = RunPaths::under(&root, "stopped");
paths.create().expect("the run directory");
let here = sys::hostname();
let launch = launched_by(sys::reaped_pid(), &here, "a driver that has since died");
let usable = ledger::DispatchRecord {
node: "build".into(),
pid: sys::pid(),
host: here.clone(),
dispatched_at: sys::now_rfc3339(),
started: sys::process_start_token(sys::pid())
.expect("this host says when a process started")
.recorded()
.to_string(),
};
assert!(roots_to_stop(&paths, &launch).is_ok());
std::fs::write(
paths.dispatch(usable.pid, 0),
serde_json::to_string(&serde_json::json!({
"node": "build",
"pid": sys::pid(),
"host": here,
"dispatched_at": sys::now_rfc3339(),
"started": usable.started,
"reaped_by": "a build that came later",
}))
.expect("an entry from a newer writer"),
)
.expect("an entry");
assert_eq!(
roots_to_stop(&paths, &launch).expect("an entry from a newer writer reads"),
Aim::Here {
roots: vec![usable.pid],
unproven: Vec::new(),
declined: Vec::new(),
},
"a dispatch recorded with a field this build does not know was left running"
);
for (what, entry) in [
(
"a record that is not JSON at all",
"not an entry".to_string(),
),
(
"a record whose stamp proves nothing",
serde_json::to_string(&ledger::DispatchRecord {
started: String::new(),
..usable.clone()
})
.expect("an unstamped entry"),
),
] {
std::fs::write(paths.dispatch(usable.pid, 0), entry).expect("an entry");
let refused = roots_to_stop(&paths, &launch)
.expect_err(&format!("{what} was read as a registry to act on"));
assert!(
refused.to_string().contains(&usable.pid.to_string()),
"the refusal does not name the entry that caused it: {refused}"
);
}
std::fs::remove_dir_all(paths.dispatches()).expect("the registry is taken away");
let refused = roots_to_stop(&paths, &launch)
.expect_err("a run with no registry at all was read as a run with nothing running");
assert!(
refused
.to_string()
.contains(&paths.dispatches().display().to_string()),
"the refusal does not name the registry it could not read: {refused}"
);
std::fs::remove_dir_all(&root).ok();
}
}