use serde::{Deserialize, Serialize};
use std::fs;
use std::io;
use std::num::NonZeroU32;
use std::path::{Path, PathBuf};
use crate::error::{Error, Result};
use crate::filter::Filters;
use crate::sys;
#[cfg(test)]
pub(crate) fn bytes_read() -> u64 {
crate::loopstats::store_bytes()
}
fn counted<T>(bytes: usize, read: T) -> T {
crate::loopstats::store_read(bytes as u64);
crate::rendercost::store_read(bytes as u64);
read
}
pub const RUNS_DIR_ENV: &str = "ONEPIPELINE_RUNS_DIR";
pub const DEFAULT_RUNS_DIR: &str = "runs";
pub fn runs_root() -> PathBuf {
std::env::var_os(RUNS_DIR_ENV)
.map(PathBuf::from)
.filter(|p| !p.as_os_str().is_empty())
.unwrap_or_else(|| PathBuf::from(DEFAULT_RUNS_DIR))
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RunPaths {
pub run: String,
pub dir: PathBuf,
}
pub fn is_valid_run_id(run: &str) -> bool {
!run.is_empty()
&& run != "."
&& run != ".."
&& !run.contains('/')
&& !run.contains('\\')
&& !Path::new(run).is_absolute()
&& Path::new(run).components().count() == 1
}
fn path_segment(name: &str) -> String {
let mapped: String = name
.chars()
.map(|c| {
if c.is_ascii_alphanumeric() || c == '-' || c == '_' || c == '.' {
c
} else {
'-'
}
})
.collect();
if mapped.is_empty() || mapped.chars().all(|c| c == '.') {
return "unnamed".to_string();
}
mapped
}
impl RunPaths {
pub fn new(run: &str) -> Self {
Self::under(&runs_root(), run)
}
pub fn under(root: &Path, run: &str) -> Self {
Self {
run: run.to_string(),
dir: root.join(run),
}
}
pub fn exists(&self) -> bool {
self.dir.is_dir()
}
pub fn create(&self) -> Result<()> {
for dir in [self.channel_dir(), self.dispatches()] {
fs::create_dir_all(&dir).map_err(|e| Error::Ledger {
path: dir,
source: e,
})?;
}
Ok(())
}
pub fn journal(&self) -> PathBuf {
self.dir.join("events.jsonl")
}
pub fn launch(&self) -> PathBuf {
self.dir.join("launch.json")
}
pub fn plan(&self) -> PathBuf {
self.dir.join("plan.json")
}
pub fn summary(&self) -> PathBuf {
self.dir.join("summary.json")
}
pub fn lock(&self) -> PathBuf {
self.dir.join("owner.lock")
}
pub fn driver_log(&self) -> PathBuf {
self.dir.join("driver.log")
}
pub fn channel_dir(&self) -> PathBuf {
self.dir.join("channel")
}
pub fn reports_dir(&self) -> PathBuf {
self.dir.join("reports")
}
pub fn report_for(&self, stream: &str, seq: u64) -> PathBuf {
self.reports_dir()
.join(format!("{}-{seq}.json", path_segment(stream)))
}
pub fn channel(&self, name: &str) -> PathBuf {
self.channel_dir().join(name)
}
pub fn result(&self) -> PathBuf {
self.dir.join("result.json")
}
pub fn dispatches(&self) -> PathBuf {
self.dir.join("dispatches")
}
pub fn dispatch(&self, pid: u32, claim: u64) -> PathBuf {
self.dispatches().join(format!("{pid}-{claim}.json"))
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Skipped {
pub path: PathBuf,
pub reason: String,
}
#[derive(Debug, Default, PartialEq, Eq)]
pub struct RunIndex {
pub runs: Vec<RunPaths>,
pub skipped: Vec<Skipped>,
}
pub fn all_runs(root: &Path) -> RunIndex {
let mut index = RunIndex::default();
let entries = match fs::read_dir(root) {
Ok(entries) => entries,
Err(error) if error.kind() == io::ErrorKind::NotFound => return index,
Err(error) => {
index.skipped.push(Skipped {
path: root.to_path_buf(),
reason: format!("the runs root cannot be read: {error}"),
});
return index;
}
};
for entry in entries {
let entry = match entry {
Ok(entry) => entry,
Err(error) => {
index.skipped.push(Skipped {
path: root.to_path_buf(),
reason: format!("an entry under the runs root cannot be read: {error}"),
});
continue;
}
};
let path = entry.path();
let about = match fs::metadata(&path) {
Ok(about) => about,
Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
Err(error) => {
index.skipped.push(Skipped {
path,
reason: format!("this host will not describe it: {error}"),
});
continue;
}
};
if !about.is_dir() {
continue;
}
let Ok(name) = entry.file_name().into_string() else {
index.skipped.push(Skipped {
path,
reason: "its name is not text this host can read, so no run id names it".into(),
});
continue;
};
let paths = RunPaths::under(root, &name);
let launch = paths.launch();
let named = launch
.file_name()
.map_or_else(|| "launch record".into(), |name| name.to_string_lossy());
let refused = match fs::metadata(&launch) {
Ok(about) if about.is_file() => None,
Ok(_) => Some(format!(
"its {named} is not a file, so it records no launch"
)),
Err(error) if error.kind() == io::ErrorKind::NotFound => Some(format!(
"no {named}: a run root records the launch that owns it"
)),
Err(error) => Some(format!("its {named} cannot be read: {error}")),
};
if let Some(reason) = refused {
index.skipped.push(Skipped {
path: paths.dir,
reason,
});
continue;
}
index.runs.push(paths);
}
index.runs.sort_by(|a, b| a.run.cmp(&b.run));
index.skipped.sort_by(|a, b| a.path.cmp(&b.path));
index
}
fn is_unset(path: &Path) -> bool {
path.as_os_str().is_empty()
}
fn unattributed_launcher() -> String {
sys::UNKNOWN_LAUNCHER.to_string()
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct LaunchRecord {
pub run_id: String,
#[serde(default, skip_serializing_if = "String::is_empty")]
pub project: String,
#[serde(default, skip_serializing_if = "is_unset")]
pub dir: PathBuf,
#[serde(default, skip_serializing_if = "String::is_empty")]
pub graph: String,
#[serde(default, skip_serializing_if = "String::is_empty")]
pub graph_run: String,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub observer_runs: Vec<String>,
#[serde(default, skip_serializing_if = "String::is_empty")]
pub observer_ending: String,
#[serde(default, skip_serializing_if = "String::is_empty")]
pub node_graph: String,
#[serde(default, skip_serializing_if = "String::is_empty")]
pub pr_author_graph: String,
#[serde(default, skip_serializing_if = "String::is_empty")]
pub node_validator: String,
#[serde(default, skip_serializing_if = "String::is_empty")]
pub envelope_reviewer: String,
#[serde(default = "unattributed_launcher")]
pub launcher: String,
#[serde(default)]
pub session: String,
#[serde(default)]
pub pid: u32,
#[serde(default)]
pub host: String,
#[serde(default, skip_serializing_if = "String::is_empty")]
pub started: String,
#[serde(default)]
pub started_at: String,
#[serde(default)]
pub heartbeat_interval: u64,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub dag_sets: Vec<String>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub node_sets: Vec<String>,
#[serde(default)]
pub adoptions: u32,
#[serde(default, skip_serializing_if = "Filters::is_empty")]
pub filters: Filters,
}
impl LaunchRecord {
pub fn driven_by_this_process(&mut self) {
self.pid = sys::pid();
self.host = sys::hostname();
self.started = sys::process_start_token(self.pid)
.map(|token| token.recorded().to_string())
.unwrap_or_default();
}
pub fn watched_by(&mut self, graph_run: String) {
self.graph_run = graph_run;
if !self.graph_run.is_empty() {
self.observer_runs.push(self.graph_run.clone());
}
self.observer_ending.clear();
}
pub fn observer_graph(&self) -> Option<&str> {
(!self.graph.is_empty()).then_some(self.graph.as_str())
}
pub fn pr_author_graph(&self) -> Option<&str> {
(!self.pr_author_graph.is_empty()).then_some(self.pr_author_graph.as_str())
}
pub fn node_validator(&self) -> Option<&str> {
(!self.node_validator.is_empty()).then_some(self.node_validator.as_str())
}
pub fn envelope_reviewer(&self) -> Option<&str> {
(!self.envelope_reviewer.is_empty()).then_some(self.envelope_reviewer.as_str())
}
fn attributed(&self) -> bool {
!self.session.is_empty() && self.session != sys::UNKNOWN_LAUNCHER
}
pub fn owned_by(&self, session: &str) -> bool {
self.attributed() && self.session == session
}
pub fn owner_label(&self, session: &str) -> String {
if !self.attributed() {
"[unknown]".to_string()
} else if self.session == session {
"[mine]".to_string()
} else {
format!("[{}:{}]", self.launcher, sys::session_digest(&self.session))
}
}
pub fn driver_pid(&self) -> Option<NonZeroU32> {
NonZeroU32::new(self.pid)
}
pub fn recorded_host(&self) -> Option<&str> {
(!self.host.is_empty()).then_some(self.host.as_str())
}
pub fn launched_at(&self) -> Option<&str> {
(!self.started_at.is_empty()).then_some(self.started_at.as_str())
}
pub fn pacemaker_interval(&self) -> Option<u64> {
(self.heartbeat_interval > 0).then_some(self.heartbeat_interval)
}
}
pub fn read_json<T: serde::de::DeserializeOwned>(path: &Path) -> Result<T> {
let text = fs::read_to_string(path).map_err(|e| Error::Ledger {
path: path.to_path_buf(),
source: e,
})?;
counted(text.len(), ());
serde_json::from_str(&text).map_err(|e| Error::Invalid(format!("{}: {e}", path.display())))
}
pub fn read_json_opt<T: serde::de::DeserializeOwned>(path: &Path) -> Option<T> {
fs::read_to_string(path)
.ok()
.map(|text| counted(text.len(), text))
.and_then(|text| serde_json::from_str(&text).ok())
}
pub fn write_json<T: Serialize>(path: &Path, value: &T) -> Result<()> {
let body = serde_json::to_string_pretty(value)
.map_err(|e| Error::Invalid(format!("{}: {e}", path.display())))?;
write_atomic(path, body.as_bytes())
}
pub fn write_atomic(path: &Path, bytes: &[u8]) -> Result<()> {
let ledger = |e: io::Error| Error::Ledger {
path: path.to_path_buf(),
source: e,
};
if let Some(parent) = path.parent() {
fs::create_dir_all(parent).map_err(ledger)?;
}
let temp = path.with_extension(format!("tmp.{}", sys::pid()));
fs::write(&temp, bytes).map_err(ledger)?;
fs::rename(&temp, path).map_err(ledger)
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct TornTail {
pub at: String,
pub offset: u64,
pub bytes: u64,
pub healed_by: u32,
}
pub fn torn_tail_log(path: &Path) -> PathBuf {
let mut name = path.file_name().map_or_else(
|| std::ffi::OsString::from("torn"),
std::ffi::OsStr::to_os_string,
);
name.push(".torn");
path.with_file_name(name)
}
pub fn torn_tails(path: &Path) -> Vec<TornTail> {
read_lines(&torn_tail_log(path))
.iter()
.filter_map(|line| serde_json::from_str(line).ok())
.collect()
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Record {
pub line: usize,
pub offset: u64,
pub text: String,
pub bytes: u64,
pub terminated: bool,
}
pub fn append_line(path: &Path, line: &str) -> Result<()> {
append_line_healed(path, line).map(|_| ())
}
pub fn append_line_healed(path: &Path, line: &str) -> Result<u64> {
let (healed, appended) = append_line_locked(path, line);
if let Some(torn) = &healed {
report_torn_tail(path, torn);
}
appended.map(|()| healed.map_or(0, |torn| torn.bytes))
}
fn append_line_locked(path: &Path, line: &str) -> (Option<TornTail>, Result<()>) {
use std::io::{Seek, SeekFrom, Write};
let ledger = |e: io::Error| Error::Ledger {
path: path.to_path_buf(),
source: e,
};
if let Some(parent) = path.parent() {
if let Err(e) = fs::create_dir_all(parent) {
return (None, Err(ledger(e)));
}
}
let mut file = match sys::open_locked_append(path) {
Ok(file) => file,
Err(e) => return (None, Err(ledger(e))),
};
let torn = match heal_tail(&mut file) {
Ok(torn) => torn,
Err(e) => return (None, Err(ledger(e))),
};
let boundary = match file.seek(SeekFrom::End(0)) {
Ok(length) => length,
Err(e) => return (torn, Err(ledger(e))),
};
let written = file
.write_all(format!("{line}\n").as_bytes())
.and_then(|()| file.flush());
match written {
Ok(()) => (torn, Ok(())),
Err(e) => {
let _ = file.set_len(boundary);
(torn, Err(ledger(e)))
}
}
}
fn heal_tail(file: &mut fs::File) -> io::Result<Option<TornTail>> {
use std::io::{Read, Seek, SeekFrom};
let length = file.metadata()?.len();
if length == 0 {
return Ok(None);
}
let mut last = [0u8; 1];
file.seek(SeekFrom::Start(length - 1))?;
file.read_exact(&mut last)?;
if last[0] == b'\n' {
return Ok(None);
}
const CHUNK: u64 = 64 * 1024;
let mut buffer = vec![0u8; CHUNK as usize];
let mut end = length;
let mut boundary = 0;
while end > 0 {
let start = end.saturating_sub(CHUNK);
let size = (end - start) as usize;
file.seek(SeekFrom::Start(start))?;
file.read_exact(&mut buffer[..size])?;
if let Some(at) = buffer[..size].iter().rposition(|byte| *byte == b'\n') {
boundary = start + at as u64 + 1;
break;
}
end = start;
}
file.set_len(boundary)?;
Ok(Some(TornTail {
at: sys::now_rfc3339(),
offset: boundary,
bytes: length - boundary,
healed_by: sys::pid(),
}))
}
fn report_torn_tail(path: &Path, torn: &TornTail) {
eprintln!(
"onepipeline: {}: discarded a {}-byte record fragment at byte {}, left by a writer \
that did not finish it; the record it was is lost",
path.display(),
torn.bytes,
torn.offset
);
let Ok(line) = serde_json::to_string(torn) else {
return;
};
let (healed, _) = append_line_locked(&torn_tail_log(path), &line);
if let Some(torn) = healed {
eprintln!(
"onepipeline: {}: discarded a {}-byte fragment of the loss log itself",
torn_tail_log(path).display(),
torn.bytes
);
}
}
pub fn read_records(path: &Path) -> Vec<Record> {
let Ok(bytes) = fs::read(path) else {
return Vec::new();
};
counted(bytes.len(), ());
records_of(&bytes, 0)
}
fn records_of(bytes: &[u8], base: u64) -> Vec<Record> {
let mut records = Vec::new();
let mut offset = base;
for (index, line) in bytes.split_inclusive(|byte| *byte == b'\n').enumerate() {
let terminated = line.ends_with(b"\n");
let text = String::from_utf8_lossy(line)
.trim_end_matches('\n')
.trim_end_matches('\r')
.to_string();
let bytes = line.len() as u64 - u64::from(terminated);
records.push(Record {
line: index + 1,
offset,
text,
bytes,
terminated,
});
offset += line.len() as u64;
}
records
}
pub fn read_records_from(path: &Path, from: u64) -> Vec<Record> {
let Ok(mut file) = fs::File::open(path) else {
return Vec::new();
};
use std::io::{Read, Seek, SeekFrom};
if file.seek(SeekFrom::Start(from)).is_err() {
return Vec::new();
}
let mut bytes = Vec::new();
if file.read_to_end(&mut bytes).is_err() {
return Vec::new();
}
counted(bytes.len(), ());
records_of(&bytes, from)
}
pub fn read_lines(path: &Path) -> Vec<String> {
read_records(path)
.into_iter()
.filter(|record| !record.text.trim().is_empty())
.map(|record| record.text)
.collect()
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct LockRecord {
pub pid: u32,
pub host: String,
pub acquired_at: String,
pub verb: String,
#[serde(default, skip_serializing_if = "String::is_empty")]
pub started: String,
}
#[derive(Debug)]
pub struct OwnershipLock {
path: PathBuf,
held: bool,
}
impl OwnershipLock {
pub fn acquire(paths: &RunPaths, verb: &str) -> Result<Self> {
let path = paths.lock();
if let Some(parent) = path.parent() {
fs::create_dir_all(parent).map_err(|e| Error::Ledger {
path: parent.to_path_buf(),
source: e,
})?;
}
let record = LockRecord {
pid: sys::pid(),
host: sys::hostname(),
acquired_at: sys::now_rfc3339(),
verb: verb.to_string(),
started: sys::process_start_token(sys::pid())
.map(|token| token.recorded().to_string())
.unwrap_or_default(),
};
let body = serde_json::to_string(&record)
.map_err(|e| Error::Invalid(format!("{}: {e}", path.display())))?;
match fs::OpenOptions::new()
.write(true)
.create_new(true)
.open(&path)
{
Ok(mut file) => {
use std::io::Write;
file.write_all(body.as_bytes()).map_err(|e| Error::Ledger {
path: path.clone(),
source: e,
})?;
Ok(Self { path, held: true })
}
Err(e) if e.kind() == io::ErrorKind::AlreadyExists => {
let held_by: Option<LockRecord> = read_json_opt(&path);
match held_by {
Some(held)
if held.host == sys::hostname() && !sys::process_may_be_live(held.pid) =>
{
write_atomic(&path, body.as_bytes())?;
Ok(Self { path, held: true })
}
Some(held) => Err(Error::Locked {
run: paths.run.clone(),
pid: held.pid,
host: held.host,
verb: held.verb,
}),
None => Err(Error::Locked {
run: paths.run.clone(),
pid: 0,
host: sys::hostname(),
verb: "an unreadable lock".to_string(),
}),
}
}
Err(e) => Err(Error::Ledger { path, source: e }),
}
}
pub fn release(mut self) {
self.remove();
}
fn remove(&mut self) {
if self.held {
let _ = fs::remove_file(&self.path);
self.held = false;
}
}
}
impl Drop for OwnershipLock {
fn drop(&mut self) {
self.remove();
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct DispatchRecord {
pub node: String,
pub pid: u32,
pub host: String,
pub dispatched_at: String,
pub started: String,
}
impl DispatchRecord {
fn is_usable(&self) -> bool {
!self.started.trim().is_empty()
}
}
#[derive(Debug)]
pub struct DispatchClaim {
path: PathBuf,
started: String,
}
impl Drop for DispatchClaim {
fn drop(&mut self) {
let ours = read_json_opt::<DispatchRecord>(&self.path)
.is_some_and(|held| held.started == self.started);
if ours {
let _ = fs::remove_file(&self.path);
}
}
}
pub fn claim_dispatch(paths: &RunPaths, node: &str, pid: u32) -> Result<DispatchClaim> {
static CLAIMED: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
let claim = CLAIMED.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
let Some(started) = sys::process_start_token(pid) else {
return Err(Error::Refused(format!(
"run '{}': node '{node}': this host will not say when pid {pid} started, so its \
dispatch cannot be recorded as running there and nothing could prove that pid is \
still this run's work",
paths.run
)));
};
let record = DispatchRecord {
node: node.to_string(),
pid,
host: sys::hostname(),
dispatched_at: sys::now_rfc3339(),
started: started.recorded().to_string(),
};
let path = paths.dispatch(pid, claim);
write_dispatch(paths, &path, claim, &record)?;
match read_json::<DispatchRecord>(&path) {
Ok(held) if held == record => Ok(DispatchClaim {
path,
started: record.started,
}),
Ok(_) | Err(_) => Err(Error::Refused(format!(
"run '{}': node '{node}': its dispatch in pid {pid} was written to {} and did not \
read back as itself, so the run cannot say where that work is",
paths.run,
path.display()
))),
} }
fn write_dispatch(
paths: &RunPaths,
path: &Path,
claim: u64,
record: &DispatchRecord,
) -> Result<()> {
let body = serde_json::to_string_pretty(record)
.map_err(|e| Error::Invalid(format!("{}: {e}", path.display())))?;
let ledger = |at: &Path| {
let at = at.to_path_buf();
move |source: io::Error| Error::Ledger { path: at, source }
};
fs::create_dir_all(paths.dispatches()).map_err(ledger(&paths.dispatches()))?;
let temp = paths
.dir
.join(format!("dispatch-{}-{claim}.tmp", record.pid));
fs::write(&temp, body.as_bytes()).map_err(ledger(&temp))?;
fs::rename(&temp, path).map_err(ledger(path))
}
pub fn dispatches_of(paths: &RunPaths) -> Result<Vec<DispatchRecord>> {
let registry = paths.dispatches();
let listed = fs::read_dir(®istry).map_err(|source| Error::Ledger {
path: registry.clone(),
source,
})?;
let mut found = Vec::new();
for entry in listed {
let entry = entry.map_err(|source| Error::Ledger {
path: registry.clone(),
source,
})?; let held: DispatchRecord = read_json(&entry.path())?;
if !held.is_usable() {
return Err(Error::Invalid(format!(
"{}: the dispatch it records carries no start token, so nothing says pid {} is \
still this run's work",
entry.path().display(),
held.pid
)));
}
found.push(held);
}
found.sort_by_key(|held| held.pid);
Ok(found)
}
#[cfg(test)]
mod tests {
use super::*;
fn scratch(name: &str) -> PathBuf {
let dir = std::env::temp_dir().join(format!("onepipeline-ledger-{name}-{}", sys::pid()));
let _ = fs::remove_dir_all(&dir);
fs::create_dir_all(&dir).expect("a scratch root");
dir
}
#[test]
fn a_launch_that_declared_no_filters_writes_the_record_it_always_wrote() {
let record = LaunchRecord {
filters: Filters::default(),
..a_record()
};
let text = serde_json::to_string(&record).expect("it serialises");
assert!(
!text.contains("filters"),
"an empty filters block reached the record: {text}"
);
assert_eq!(
serde_json::from_str::<LaunchRecord>(&text).expect("it re-parses"),
record
);
let older = serde_json::json!({
"run_id": "demo",
"plan": "plan.json",
"node_graph": "graphs/node-scope.yaml",
"launcher": "claude-code",
"session": "a-session",
"pid": 1,
"host": "h",
"started_at": "2026-08-15T00:00:00.000Z",
"heartbeat_interval": 1800,
});
let read: LaunchRecord =
serde_json::from_value(older).expect("a record predating the field still reads");
assert!(read.filters.is_empty());
}
#[test]
fn a_launchs_filters_survive_the_record_they_are_retained_in() {
let declared = Filters {
agentgraph: Some(
crate::filter::EventFilter::parse(r#"{"exclude": [{"kind": "turn-*"}]}"#)
.expect("a filter"),
),
vcs: Some(
crate::filter::EventFilter::parse(r#"{"include": [{"kind": "gate-*"}]}"#)
.expect("a filter"),
),
profiles: [(
"planner".to_string(),
crate::filter::EventFilter::parse(r#"{"include": [{"source": "pipeline"}]}"#)
.expect("a filter"),
)]
.into_iter()
.collect(),
};
let record = LaunchRecord {
filters: declared.clone(),
..a_record()
};
let text = serde_json::to_string(&record).expect("it serialises");
let read: LaunchRecord = serde_json::from_str(&text).expect("it re-parses");
assert_eq!(read.filters, declared);
assert_eq!(read, record);
}
fn a_record() -> LaunchRecord {
LaunchRecord {
run_id: "demo".into(),
project: "plans:demo".into(),
dir: PathBuf::from("/tmp/launch"),
graph: String::new(),
graph_run: String::new(),
observer_runs: Vec::new(),
observer_ending: String::new(),
node_graph: "graphs/node-scope.yaml".into(),
pr_author_graph: String::new(),
node_validator: String::new(),
envelope_reviewer: String::new(),
launcher: "claude-code".into(),
session: "a-session".into(),
pid: 1,
host: "h".into(),
started: "Fri Aug 15 00:00:00 2026".into(),
started_at: sys::now_rfc3339(),
heartbeat_interval: 1_800,
dag_sets: Vec::new(),
node_sets: Vec::new(),
adoptions: 0,
filters: Filters::default(),
}
}
#[test]
fn claiming_a_run_records_the_stamp_that_proves_its_pid_and_an_older_record_carries_none() {
let mut record = a_record();
record.driven_by_this_process();
assert_eq!(record.pid, sys::pid());
assert_eq!(record.host, sys::hostname());
assert!(
sys::process_start_token(sys::pid())
.expect("this host says when a process started")
.matches(&record.started),
"a run claimed by this process recorded a stamp that does not prove it"
);
let text = serde_json::to_string(&record).expect("it serialises");
assert_eq!(
serde_json::from_str::<LaunchRecord>(&text).expect("it re-parses"),
record
);
let older = serde_json::json!({
"run_id": "demo",
"plan": "plan.json",
"node_graph": "graphs/node-scope.yaml",
"launcher": "claude-code",
"session": "a-session",
"pid": sys::pid(),
"host": sys::hostname(),
"started_at": "2026-08-15T00:00:00.000Z",
"heartbeat_interval": 1800,
});
let read: LaunchRecord =
serde_json::from_value(older).expect("a record predating the stamp still reads");
assert!(read.started.is_empty());
assert!(
!sys::process_start_token(sys::pid())
.expect("this host says when a process started")
.matches(&read.started),
"a record carrying no stamp proved a live pid"
);
let text = serde_json::to_string(&LaunchRecord {
started: String::new(),
..a_record()
})
.expect("it serialises");
assert!(
!text.contains("started\""),
"an empty stamp reached the record: {text}"
);
}
#[test]
fn a_launch_record_written_before_any_of_these_five_keys_still_reads() {
const HISTORICAL: [&str; 5] =
["session", "pid", "host", "started_at", "heartbeat_interval"];
let root = scratch("historical-launch");
let whole = serde_json::to_value(a_record()).expect("a record this build writes");
let read_one = |name: &str, without: &[&str]| -> LaunchRecord {
let mut document = whole.clone();
let fields = document.as_object_mut().expect("a launch record");
for key in without {
assert!(
fields.remove(*key).is_some(),
"the record this build writes carries no `{key}` to take away"
);
}
let dir = root.join(name);
fs::create_dir_all(&dir).expect("a run root");
let launch = dir.join("launch.json");
fs::write(&launch, document.to_string()).expect("a record an older build wrote");
read_json::<LaunchRecord>(&launch).unwrap_or_else(|error| {
panic!("a record written before `{without:?}` existed was refused: {error}")
})
};
for key in HISTORICAL {
let read = read_one(key, &[key]);
assert_eq!(read.run_id, "demo");
match key {
"session" => {
assert!(read.session.is_empty(), "a session was invented: {read:?}");
assert!(
!read.owned_by(""),
"a record naming no session was owned by a reader that names none either"
);
assert_eq!(read.owner_label("a-session"), "[unknown]");
}
"pid" => {
assert_eq!(read.pid, 0);
assert_eq!(
read.driver_pid(),
None,
"a pid nobody wrote was served as one a reader may act on"
);
}
"host" => {
assert!(read.host.is_empty());
assert_eq!(
read.recorded_host(),
None,
"a host nobody named was served as one"
);
}
"started_at" => {
assert_eq!(
read.launched_at(),
None,
"a launch instant nobody recorded was served as an instant"
);
}
"heartbeat_interval" => {
assert_eq!(read.heartbeat_interval, 0);
assert_eq!(
read.pacemaker_interval(),
None,
"a record naming no interval produced a pacemaker interval"
);
assert_ne!(
read.pacemaker_interval(),
Some(0),
"a zero-second pacemaker was served as an interval"
);
}
other => unreachable!("{other} is not one of the five"),
}
}
let oldest = read_one("all-five", &HISTORICAL);
assert!(oldest.session.is_empty());
assert_eq!(oldest.owner_label("a-session"), "[unknown]");
assert!(!oldest.owned_by(sys::UNKNOWN_LAUNCHER));
assert_eq!(oldest.driver_pid(), None);
assert_eq!(oldest.recorded_host(), None);
assert_eq!(oldest.launched_at(), None);
assert_eq!(oldest.pacemaker_interval(), None);
assert_eq!(oldest.run_id, "demo");
assert_eq!(oldest.launcher, "claude-code");
assert_eq!(oldest.node_graph, "graphs/node-scope.yaml");
let whole = read_one("whole", &[]);
assert_eq!(whole.session, "a-session");
assert_eq!(whole.driver_pid(), NonZeroU32::new(1));
assert_eq!(whole.recorded_host(), Some("h"));
assert!(whole.launched_at().is_some());
assert_eq!(whole.pacemaker_interval(), Some(1_800));
}
#[test]
fn concurrent_appenders_each_land_a_whole_line() {
let root = scratch("append");
let path = root.join("events.jsonl");
const WRITERS: usize = 8;
const EACH: usize = 60;
std::thread::scope(|scope| {
for writer in 0..WRITERS {
let path = path.clone();
scope.spawn(move || {
for n in 0..EACH {
let line = serde_json::json!({
"writer": writer,
"seq": n,
"payload": "x".repeat(512),
})
.to_string();
append_line(&path, &line).expect("the line is appended");
}
});
}
});
let lines = read_lines(&path);
assert_eq!(lines.len(), WRITERS * EACH, "a record was torn or lost");
for line in &lines {
serde_json::from_str::<serde_json::Value>(line)
.unwrap_or_else(|e| panic!("a torn record reached the file: {e}: {line}"));
}
fs::remove_dir_all(&root).ok();
}
#[test]
fn a_lock_refuses_a_second_writer_and_names_the_first() {
let root = scratch("lock");
let paths = RunPaths::under(&root, "demo");
paths.create().expect("the run directory");
let first = OwnershipLock::acquire(&paths, "start").expect("the first writer wins");
let second = OwnershipLock::acquire(&paths, "adopt");
match second {
Err(Error::Locked { run, pid, verb, .. }) => {
assert_eq!(run, "demo");
assert_eq!(pid, sys::pid());
assert_eq!(verb, "start");
}
other => panic!("a second writer was not refused: {other:?}"),
}
first.release();
OwnershipLock::acquire(&paths, "adopt").expect("the lock was released");
fs::remove_dir_all(&root).ok();
}
#[test]
fn a_run_id_names_one_directory_and_never_a_path() {
for good in ["demo", "run-2", "a_b", "tracked-release", "R1"] {
assert!(is_valid_run_id(good), "{good} was refused");
}
for bad in [
"",
".",
"..",
"../elsewhere",
"../../elsewhere",
"a/b",
"a\\b",
"/absolute",
"./here",
] {
assert!(!is_valid_run_id(bad), "{bad:?} was accepted");
}
}
#[test]
fn a_lock_whose_holder_is_proved_gone_is_reclaimed() {
let root = scratch("stale");
let paths = RunPaths::under(&root, "demo");
paths.create().expect("the run directory");
let dead = sys::reaped_pid();
write_json(
&paths.lock(),
&LockRecord {
pid: dead,
host: sys::hostname(),
acquired_at: sys::now_rfc3339(),
verb: "start".to_string(),
started: String::new(),
},
)
.expect("a stale lock");
OwnershipLock::acquire(&paths, "start").expect("a dead holder's lock is reclaimed");
fs::remove_dir_all(&root).ok();
}
#[test]
fn an_unreadable_lock_is_still_a_claim() {
let root = scratch("unreadable");
let paths = RunPaths::under(&root, "demo");
paths.create().expect("the run directory");
fs::write(paths.lock(), "not json at all").expect("a corrupt lock");
assert!(matches!(
OwnershipLock::acquire(&paths, "start"),
Err(Error::Locked { .. })
));
fs::remove_dir_all(&root).ok();
}
#[test]
fn an_unknown_launch_is_nobodys_run() {
let record = LaunchRecord {
run_id: "demo".into(),
project: "plans:demo".into(),
dir: PathBuf::from("/tmp/launch"),
graph: "graphs/dag-scope.yaml".into(),
graph_run: String::new(),
observer_runs: Vec::new(),
observer_ending: String::new(),
node_graph: String::new(),
pr_author_graph: String::new(),
node_validator: String::new(),
envelope_reviewer: String::new(),
launcher: sys::UNKNOWN_LAUNCHER.into(),
session: sys::UNKNOWN_LAUNCHER.into(),
pid: 1,
host: "h".into(),
started: String::new(),
started_at: sys::now_rfc3339(),
heartbeat_interval: 1,
dag_sets: Vec::new(),
node_sets: Vec::new(),
adoptions: 0,
filters: Filters::default(),
};
assert!(!record.owned_by(sys::UNKNOWN_LAUNCHER));
assert_eq!(record.owner_label("anyone"), "[unknown]");
}
#[test]
fn a_foreign_owner_is_labelled_without_naming_the_session() {
let record = LaunchRecord {
run_id: "demo".into(),
project: "plans:demo".into(),
dir: PathBuf::from("/tmp/launch"),
graph: "graphs/dag-scope.yaml".into(),
graph_run: String::new(),
observer_runs: Vec::new(),
observer_ending: String::new(),
node_graph: String::new(),
pr_author_graph: String::new(),
node_validator: String::new(),
envelope_reviewer: String::new(),
launcher: "claude-code".into(),
session: "secret-session-id".into(),
pid: 1,
host: "h".into(),
started: String::new(),
started_at: sys::now_rfc3339(),
heartbeat_interval: 1,
dag_sets: Vec::new(),
node_sets: Vec::new(),
adoptions: 0,
filters: Filters::default(),
};
let label = record.owner_label("mine");
assert!(!label.contains("secret-session-id"), "{label} leaks the id");
assert!(label.starts_with("[claude-code:"));
assert_eq!(record.owner_label("secret-session-id"), "[mine]");
assert!(record.owned_by("secret-session-id"));
}
#[test]
fn an_atomic_write_leaves_no_temporary_behind() {
let root = scratch("atomic");
let target = root.join("nested").join("record.json");
write_json(&target, &serde_json::json!({"ok": true})).expect("written");
let value: serde_json::Value = read_json(&target).expect("read back");
assert_eq!(value["ok"], serde_json::json!(true));
let leftovers: Vec<_> = fs::read_dir(root.join("nested"))
.expect("the directory")
.flatten()
.filter(|e| e.file_name().to_string_lossy().contains("tmp"))
.collect();
assert!(leftovers.is_empty(), "a temporary survived the rename");
fs::remove_dir_all(&root).ok();
}
#[test]
fn appended_lines_read_back_in_order_and_skip_blanks() {
let root = scratch("append-order");
let path = root.join("queue.jsonl");
assert!(read_lines(&path).is_empty());
append_line(&path, "first").expect("appended");
append_line(&path, "").expect("appended");
append_line(&path, "second").expect("appended");
assert_eq!(read_lines(&path), vec!["first", "second"]);
fs::remove_dir_all(&root).ok();
}
#[test]
fn a_record_carries_where_it_is_and_whether_its_writer_finished_it() {
let root = scratch("records");
let path = root.join("events.jsonl");
fs::write(&path, "first\n\nsecond\nhalf").expect("the file is written");
let records = read_records(&path);
assert_eq!(records.len(), 4);
assert_eq!(records[0].offset, 0);
assert_eq!(records[2].text, "second");
assert_eq!(records[2].offset, 7);
assert!(records[2].terminated);
assert_eq!(records[3].text, "half");
assert_eq!(records[3].offset, 14);
assert!(
!records[3].terminated,
"a fragment read as a record its writer had finished"
);
assert_eq!(read_lines(&path), vec!["first", "second", "half"]);
fs::remove_dir_all(&root).ok();
}
#[test]
fn a_file_torn_mid_character_still_reads_back_the_records_before_the_tear() {
let root = scratch("lossy");
let path = root.join("queue.jsonl");
fs::create_dir_all(&root).ok();
let mut bytes = b"{\"first\":1}\n{\"second\":2}\n{\"third\":".to_vec();
bytes.push(0xE2);
fs::write(&path, &bytes).expect("the file is written");
let records = read_records(&path);
assert_eq!(
records.len(),
3,
"the tear took the file with it: {records:?}"
);
assert_eq!(records[0].text, "{\"first\":1}");
assert_eq!(records[1].text, "{\"second\":2}");
assert_eq!(records[2].offset, 25);
assert_eq!(
records[2].bytes, 10,
"the loss was measured in the reading rather than in the file"
);
assert!(!records[2].terminated);
assert_eq!(read_lines(&path).len(), 3);
fs::remove_dir_all(&root).ok();
}
#[test]
fn an_append_heals_a_fragment_and_records_what_it_discarded() {
let root = scratch("heal");
let path = root.join("events.jsonl");
append_line(&path, "first").expect("appended");
let whole = fs::read_to_string(&path).expect("the file reads");
fs::OpenOptions::new()
.append(true)
.open(&path)
.and_then(|mut file| std::io::Write::write_all(&mut file, b"{\"half\":"))
.expect("the fragment is written");
append_line(&path, "second").expect("appended");
assert_eq!(
fs::read_to_string(&path).expect("the file reads"),
format!("{whole}second\n"),
"the heal cut into a whole record, or left the fragment in"
);
let recorded = torn_tails(&path);
assert_eq!(recorded.len(), 1, "{recorded:?}");
assert_eq!(recorded[0].offset, whole.len() as u64);
assert_eq!(recorded[0].bytes, 8);
assert_eq!(recorded[0].healed_by, sys::pid());
append_line(&path, "third").expect("appended");
assert_eq!(torn_tails(&path).len(), 1);
fs::remove_dir_all(&root).ok();
}
#[test]
fn a_store_that_is_nothing_but_a_fragment_heals_to_empty_and_says_so() {
let root = scratch("heal-whole");
let path = root.join("events.jsonl");
fs::write(&path, "x".repeat(70 * 1024)).expect("the fragment is written");
append_line(&path, "first").expect("appended");
assert_eq!(
fs::read_to_string(&path).expect("the file reads"),
"first\n"
);
let recorded = torn_tails(&path);
assert_eq!(recorded.len(), 1, "{recorded:?}");
assert_eq!(recorded[0].offset, 0);
assert_eq!(recorded[0].bytes, 70 * 1024);
fs::remove_dir_all(&root).ok();
}
#[test]
fn a_store_that_lost_nothing_records_nothing() {
let root = scratch("torn-absent");
let path = root.join("events.jsonl");
assert_eq!(
torn_tail_log(&path),
root.join("events.jsonl.torn"),
"the loss log is not beside the store it is about"
);
assert!(torn_tails(&path).is_empty());
append_line(&path, "first").expect("appended");
assert!(
torn_tails(&path).is_empty(),
"an append that healed nothing reported a loss"
);
fs::remove_dir_all(&root).ok();
}
#[test]
fn only_directories_with_a_launch_record_are_runs_and_the_rest_are_named() {
let root = scratch("index");
for name in ["b-run", "a-run"] {
let paths = RunPaths::under(&root, name);
paths.create().expect("a run directory");
write_json(&paths.launch(), &serde_json::json!({})).expect("a launch record");
}
fs::create_dir_all(root.join("scratch")).expect("a directory that records no run");
fs::write(root.join("notes.txt"), "not a run root").expect("a file beside the runs");
fs::create_dir_all(RunPaths::under(&root, "impostor").launch())
.expect("a launch record that is a directory");
let index = all_runs(&root);
let ids: Vec<String> = index.runs.iter().map(|r| r.run.clone()).collect();
assert_eq!(ids, vec!["a-run".to_string(), "b-run".to_string()]);
let refused: Vec<(PathBuf, String)> = index
.skipped
.iter()
.map(|root| (root.path.clone(), root.reason.clone()))
.collect();
assert_eq!(refused.len(), 2, "{refused:?}");
assert_eq!(refused[0].0, root.join("impostor"));
assert!(refused[0].1.contains("is not a file"), "{refused:?}");
assert_eq!(refused[1].0, root.join("scratch"));
assert!(refused[1].1.contains("no launch.json"), "{refused:?}");
assert_eq!(all_runs(&root.join("missing")), RunIndex::default());
fs::remove_dir_all(&root).ok();
}
#[test]
fn a_lock_records_the_holders_start_token_beside_its_pid() {
let root = scratch("stamp");
let paths = RunPaths::under(&root, "demo");
paths.create().expect("the run directory");
let held = OwnershipLock::acquire(&paths, "drive").expect("the lock is taken");
let record: LockRecord = read_json(&paths.lock()).expect("the lock reads back");
assert_eq!(record.pid, sys::pid());
assert!(
sys::process_start_token(sys::pid())
.expect("this host says when a process started")
.matches(&record.started),
"the lock's stamp is not this process's own start"
);
assert!(!record.started.is_empty());
held.release();
fs::remove_dir_all(&root).ok();
}
#[test]
fn a_dispatch_claims_the_process_it_runs_in_and_gives_it_up_when_it_ends() {
let root = scratch("dispatches");
let paths = RunPaths::under(&root, "demo");
paths.create().expect("the run directory");
assert!(
dispatches_of(&paths)
.expect("a run that has dispatched nothing has an empty registry")
.is_empty(),
"a run that has dispatched nothing claimed a process"
);
let claim = claim_dispatch(&paths, "build", sys::pid()).expect("the dispatch is recorded");
let recorded = dispatches_of(&paths).expect("the registry reads");
assert_eq!(recorded.len(), 1, "{recorded:?}");
assert_eq!(recorded[0].node, "build");
assert_eq!(recorded[0].pid, sys::pid());
assert_eq!(recorded[0].host, sys::hostname());
assert!(
sys::process_start_token(sys::pid())
.expect("this host says when a process started")
.matches(&recorded[0].started),
"the entry's stamp is not this process's own start: {recorded:?}"
);
drop(claim);
assert!(
dispatches_of(&paths)
.expect("the registry reads")
.is_empty(),
"a dispatch that ended left the run claiming its process"
);
fs::remove_dir_all(&root).ok();
}
#[test]
fn two_dispatches_in_one_process_are_two_entries_and_each_ends_alone() {
let root = scratch("dispatches-shared-process");
let paths = RunPaths::under(&root, "demo");
paths.create().expect("the run directory");
let first = claim_dispatch(&paths, "first", sys::pid()).expect("the first is recorded");
let second = claim_dispatch(&paths, "second", sys::pid()).expect("the second is recorded");
let nodes = |paths: &RunPaths| {
let mut named: Vec<String> = dispatches_of(paths)
.expect("the registry reads")
.into_iter()
.map(|held| held.node)
.collect();
named.sort();
named
};
assert_eq!(
nodes(&paths),
vec!["first".to_string(), "second".to_string()],
"two dispatches in one process did not record two entries"
);
drop(first);
assert_eq!(
nodes(&paths),
vec!["second".to_string()],
"a dispatch that ended took a live one's registration with it"
);
drop(second);
assert!(dispatches_of(&paths)
.expect("the registry reads")
.is_empty());
fs::remove_dir_all(&root).ok();
}
#[test]
fn a_claim_that_ends_leaves_a_later_dispatchs_entry_alone() {
let root = scratch("dispatches-reused");
let paths = RunPaths::under(&root, "demo");
paths.create().expect("the run directory");
let first = claim_dispatch(&paths, "build", sys::pid()).expect("the dispatch is recorded");
write_json(
&paths.dispatch(sys::pid(), 0),
&DispatchRecord {
started: "the process that took it, which is not the first one".into(),
..dispatches_of(&paths).expect("the registry reads")[0].clone()
},
)
.expect("the entry is rewritten");
drop(first);
let recorded = dispatches_of(&paths).expect("the registry reads");
assert_eq!(
recorded.len(),
1,
"a dispatch that ended removed an entry it did not write: {recorded:?}"
);
fs::remove_dir_all(&root).ok();
}
#[test]
fn the_registry_reads_in_pid_order() {
let root = scratch("dispatches-order");
let paths = RunPaths::under(&root, "demo");
paths.create().expect("the run directory");
for (node, pid) in [("later", 900_u32), ("earlier", 90), ("middle", 300)] {
write_json(
&paths.dispatch(pid, 0),
&DispatchRecord {
node: node.to_string(),
pid,
host: sys::hostname(),
dispatched_at: sys::now_rfc3339(),
started: "a start this host once reported".into(),
},
)
.expect("an entry");
}
let read: Vec<u32> = dispatches_of(&paths)
.expect("the registry reads")
.iter()
.map(|held| held.pid)
.collect();
assert_eq!(read, vec![90, 300, 900]);
fs::remove_dir_all(&root).ok();
}
#[test]
fn a_registry_this_build_cannot_read_is_reported_and_never_read_as_an_empty_one() {
let root = scratch("dispatches-unreadable");
let paths = RunPaths::under(&root, "demo");
paths.create().expect("the run directory");
let usable = DispatchRecord {
node: "build".into(),
pid: 4_242,
host: sys::hostname(),
dispatched_at: sys::now_rfc3339(),
started: "a start this host once reported".into(),
};
fs::write(
paths.dispatch(usable.pid, 0),
serde_json::to_string(&serde_json::json!({
"node": usable.node,
"pid": usable.pid,
"host": usable.host,
"dispatched_at": usable.dispatched_at,
"started": usable.started,
"reaped_by": "a build that came later",
}))
.expect("an entry from a newer writer"),
)
.expect("an entry");
assert_eq!(
dispatches_of(&paths).expect("an entry from a newer writer reads"),
vec![usable.clone()],
"an entry carrying a field this build does not know took the whole registry with it"
);
for (what, entry) in [
(
"a record that is not JSON at all",
"not an entry".to_string(),
),
(
"a record missing the stamp entirely",
serde_json::to_string(&serde_json::json!({
"node": usable.node,
"pid": usable.pid,
"host": usable.host,
"dispatched_at": usable.dispatched_at,
}))
.expect("an entry from a writer that recorded no stamp"),
),
(
"a record whose stamp proves nothing",
serde_json::to_string(&DispatchRecord {
started: String::new(),
..usable.clone()
})
.expect("an unstamped entry"),
),
] {
fs::write(paths.dispatch(usable.pid, 0), entry).expect("an entry");
let refused =
dispatches_of(&paths).expect_err(&format!("{what} was read as a registry"));
assert!(
refused.to_string().contains(&usable.pid.to_string()),
"the refusal over {what} does not name what caused it: {refused}"
);
}
fs::remove_dir_all(paths.dispatches()).expect("the registry is taken away");
let refused = dispatches_of(&paths)
.expect_err("a registry that is not there was read as a run with nothing running");
assert!(
refused
.to_string()
.contains(&paths.dispatches().display().to_string()),
"the refusal does not name the registry it could not read: {refused}"
);
fs::remove_dir_all(&root).ok();
}
#[test]
fn a_dispatch_the_registry_cannot_record_is_refused() {
let root = scratch("dispatches-unwritable");
let paths = RunPaths::under(&root, "demo");
paths.create().expect("the run directory");
let reaped = sys::reaped_pid();
let refused = claim_dispatch(&paths, "build", reaped)
.expect_err("a dispatch nothing can stamp was recorded anyway");
assert!(
refused.to_string().contains(&reaped.to_string()),
"the refusal does not name the process it could not stamp: {refused}"
);
fs::remove_dir_all(paths.dispatches()).expect("the registry is taken away");
fs::write(paths.dispatches(), "not a directory").expect("something in the way");
let refused = claim_dispatch(&paths, "build", sys::pid())
.expect_err("a claim that could not be written was reported as held");
assert!(
refused
.to_string()
.contains(&paths.dispatches().display().to_string()),
"the refusal does not name what it could not write: {refused}"
);
fs::remove_dir_all(&root).ok();
}
#[test]
fn a_lock_without_a_start_token_reads_and_is_written_back_without_one() {
let record: LockRecord = serde_json::from_str(
r#"{"pid":1,"host":"h","acquired_at":"2026-01-01T00:00:00.000Z","verb":"drive"}"#,
)
.expect("a lock from a build that predates the stamp");
assert!(record.started.is_empty());
let written = serde_json::to_string(&record).expect("it serializes");
assert!(!written.contains("started"), "{written}");
}
}