use serde::{Deserialize, Serialize};
use std::ffi::OsStr;
use std::fs;
use std::io;
use std::num::{NonZeroU32, NonZeroU64};
use std::path::{Path, PathBuf};
use crate::error::{Error, Result};
use crate::filter::Filters;
use crate::sys;
#[cfg(test)]
pub(crate) fn bytes_read() -> u64 {
crate::loopstats::store_bytes()
}
fn counted<T>(bytes: usize, read: T) -> T {
crate::loopstats::store_read(bytes as u64);
crate::rendercost::store_read(bytes as u64);
read
}
pub const RUNS_DIR_ENV: &str = "ONEPIPELINE_RUNS_DIR";
pub const DEFAULT_RUNS_DIR: &str = "runs";
pub fn runs_root() -> PathBuf {
std::env::var_os(RUNS_DIR_ENV)
.map(PathBuf::from)
.filter(|p| !p.as_os_str().is_empty())
.unwrap_or_else(|| PathBuf::from(DEFAULT_RUNS_DIR))
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RunPaths {
pub run: String,
pub dir: PathBuf,
}
pub fn is_valid_run_id(run: &str) -> bool {
!run.is_empty()
&& run != "."
&& run != ".."
&& !run.contains('/')
&& !run.contains('\\')
&& !Path::new(run).is_absolute()
&& Path::new(run).components().count() == 1
}
fn path_segment(name: &str) -> String {
let mapped: String = name
.chars()
.map(|c| {
if c.is_ascii_alphanumeric() || c == '-' || c == '_' || c == '.' {
c
} else {
'-'
}
})
.collect();
if mapped.is_empty() || mapped.chars().all(|c| c == '.') {
return "unnamed".to_string();
}
mapped
}
impl RunPaths {
pub fn new(run: &str) -> Self {
Self::under(&runs_root(), run)
}
pub fn under(root: &Path, run: &str) -> Self {
Self {
run: run.to_string(),
dir: root.join(run),
}
}
pub fn exists(&self) -> bool {
self.dir.is_dir()
}
pub fn create(&self) -> Result<()> {
for dir in [self.channel_dir(), self.dispatches()] {
fs::create_dir_all(&dir).map_err(|e| Error::Ledger {
path: dir,
source: e,
})?;
}
Ok(())
}
pub fn journal(&self) -> PathBuf {
self.dir.join("events.jsonl")
}
pub fn launch(&self) -> PathBuf {
self.dir.join("launch.json")
}
pub fn plan(&self) -> PathBuf {
self.dir.join("plan.json")
}
pub fn summary(&self) -> PathBuf {
self.dir.join("summary.json")
}
pub(crate) fn maintenance(&self) -> PathBuf {
self.dir.join("maintenance.json")
}
pub(crate) fn checkpoint(&self) -> PathBuf {
self.dir.join("checkpoint.json")
}
pub(crate) fn watchers(&self) -> PathBuf {
self.dir.join("watchers")
}
pub(crate) fn watcher(&self, pid: u32, nonce: &str) -> PathBuf {
self.watchers().join(format!("{pid}-{nonce}.json"))
}
pub fn lock(&self) -> PathBuf {
self.dir.join("owner.lock")
}
pub fn driver_log(&self) -> PathBuf {
self.dir.join("driver.log")
}
pub fn channel_dir(&self) -> PathBuf {
self.dir.join("channel")
}
pub fn reports_dir(&self) -> PathBuf {
self.dir.join("reports")
}
pub fn report_for(&self, stream: &str, seq: u64) -> PathBuf {
self.reports_dir()
.join(format!("{}-{seq}.json", path_segment(stream)))
}
pub fn channel(&self, name: &str) -> PathBuf {
self.channel_dir().join(name)
}
pub fn result(&self) -> PathBuf {
self.dir.join("result.json")
}
pub fn dispatches(&self) -> PathBuf {
self.dir.join("dispatches")
}
pub fn dispatch(&self, pid: u32, claim: u64) -> PathBuf {
self.dispatches().join(format!("{pid}-{claim}.json"))
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Skipped {
pub path: PathBuf,
pub reason: String,
}
#[derive(Debug, Default, PartialEq, Eq)]
pub struct RunIndex {
pub runs: Vec<RunPaths>,
pub skipped: Vec<Skipped>,
}
pub fn all_runs(root: &Path) -> RunIndex {
let mut index = RunIndex::default();
let entries = match fs::read_dir(root) {
Ok(entries) => entries,
Err(error) if error.kind() == io::ErrorKind::NotFound => return index,
Err(error) => {
index.skipped.push(Skipped {
path: root.to_path_buf(),
reason: format!("the runs root cannot be read: {error}"),
});
return index;
}
};
for entry in entries {
let entry = match entry {
Ok(entry) => entry,
Err(error) => {
index.skipped.push(Skipped {
path: root.to_path_buf(),
reason: format!("an entry under the runs root cannot be read: {error}"),
});
continue;
}
};
let path = entry.path();
let about = match fs::metadata(&path) {
Ok(about) => about,
Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
Err(error) => {
index.skipped.push(Skipped {
path,
reason: format!("this host will not describe it: {error}"),
});
continue;
}
};
if !about.is_dir() {
continue;
}
let Ok(name) = entry.file_name().into_string() else {
index.skipped.push(Skipped {
path,
reason: "its name is not text this host can read, so no run id names it".into(),
});
continue;
};
let paths = RunPaths::under(root, &name);
let launch = paths.launch();
let named = launch
.file_name()
.map_or_else(|| "launch record".into(), |name| name.to_string_lossy());
let refused = match fs::metadata(&launch) {
Ok(about) if about.is_file() => None,
Ok(_) => Some(format!(
"its {named} is not a file, so it records no launch"
)),
Err(error) if error.kind() == io::ErrorKind::NotFound => Some(format!(
"no {named}: a run root records the launch that owns it"
)),
Err(error) => Some(format!("its {named} cannot be read: {error}")),
};
if let Some(reason) = refused {
index.skipped.push(Skipped {
path: paths.dir,
reason,
});
continue;
}
index.runs.push(paths);
}
index.runs.sort_by(|a, b| a.run.cmp(&b.run));
index.skipped.sort_by(|a, b| a.path.cmp(&b.path));
index
}
fn is_unset(path: &Path) -> bool {
path.as_os_str().is_empty()
}
fn unattributed_launcher() -> String {
sys::UNKNOWN_LAUNCHER.to_string()
}
#[derive(Debug, Clone, PartialEq)]
pub struct RecordedBusConfig {
recorded: serde_json::Value,
config: onemessagebus::Config,
}
impl RecordedBusConfig {
#[must_use]
pub fn config(&self) -> &onemessagebus::Config {
&self.config
}
}
impl From<onemessagebus::Config> for RecordedBusConfig {
fn from(config: onemessagebus::Config) -> Self {
let recorded = serde_json::to_value(&config)
.unwrap_or_else(|_| unreachable!("a bus configuration serializes"));
Self { recorded, config }
}
}
impl Serialize for RecordedBusConfig {
fn serialize<S: serde::Serializer>(
&self,
serializer: S,
) -> std::result::Result<S::Ok, S::Error> {
self.recorded.serialize(serializer)
}
}
impl<'de> Deserialize<'de> for RecordedBusConfig {
fn deserialize<D: serde::Deserializer<'de>>(
deserializer: D,
) -> std::result::Result<Self, D::Error> {
let recorded = serde_json::Value::deserialize(deserializer)?;
let mut completed = recorded.clone();
if let Some(codecs) = completed
.get_mut("codecs")
.and_then(serde_json::Value::as_object_mut)
{
for codec in codecs
.values_mut()
.filter_map(serde_json::Value::as_object_mut)
{
for (field, empty) in codec_required_empties() {
codec.entry(field.clone()).or_insert_with(|| empty.clone());
}
}
}
let config = serde_json::from_value(completed).map_err(serde::de::Error::custom)?;
Ok(Self { recorded, config })
}
}
fn codec_required_empties() -> &'static serde_json::Map<String, serde_json::Value> {
static EMPTIES: std::sync::OnceLock<serde_json::Map<String, serde_json::Value>> =
std::sync::OnceLock::new();
EMPTIES.get_or_init(|| {
let schema = schemars::schema_for!(onemessagebus::CodecConfig).to_value();
let resolved = |property: &serde_json::Value| -> Option<serde_json::Value> {
match property.get("$ref").and_then(serde_json::Value::as_str) {
Some(reference) => schema.pointer(reference.trim_start_matches('#')).cloned(),
None => Some(property.clone()),
}
};
let empty_of = |kind: &str| match kind {
"string" => Some(serde_json::Value::String(String::new())),
"object" => Some(serde_json::Value::Object(serde_json::Map::new())),
"array" => Some(serde_json::Value::Array(Vec::new())),
_ => None,
};
schema
.get("required")
.and_then(serde_json::Value::as_array)
.into_iter()
.flatten()
.filter_map(serde_json::Value::as_str)
.filter_map(|field| {
let property = resolved(schema.pointer(&format!("/properties/{field}"))?)?;
let empty = empty_of(property.get("type")?.as_str()?)?;
Some((field.to_owned(), empty))
})
.collect()
})
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct LaunchRecord {
pub run_id: String,
#[serde(default, skip_serializing_if = "String::is_empty")]
pub project: String,
#[serde(default, skip_serializing_if = "is_unset")]
pub dir: PathBuf,
#[serde(default, skip_serializing_if = "String::is_empty")]
pub graph: String,
#[serde(default, skip_serializing_if = "String::is_empty")]
pub graph_run: String,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub observer_runs: Vec<String>,
#[serde(default, skip_serializing_if = "String::is_empty")]
pub observer_ending: String,
#[serde(default, skip_serializing_if = "String::is_empty")]
pub node_graph: String,
#[serde(default, skip_serializing_if = "String::is_empty")]
pub pr_author_graph: String,
#[serde(default, skip_serializing_if = "String::is_empty")]
pub node_validator: String,
#[serde(default, skip_serializing_if = "String::is_empty")]
pub envelope_reviewer: String,
#[serde(default, skip_serializing_if = "String::is_empty")]
pub envelope_reviewer_bar: String,
#[serde(default = "unattributed_launcher")]
pub launcher: String,
#[serde(default)]
pub session: String,
#[serde(default)]
pub pid: u32,
#[serde(default)]
pub host: String,
#[serde(default, skip_serializing_if = "String::is_empty")]
pub started: String,
#[serde(default)]
pub started_at: String,
#[serde(default)]
pub heartbeat_interval: u64,
#[serde(default)]
pub writeback_item_budget: u64,
#[serde(default, skip_serializing_if = "String::is_empty")]
pub success_hook: String,
#[serde(default, skip_serializing_if = "String::is_empty")]
pub failure_hook: String,
#[serde(default, skip_serializing_if = "is_zero")]
pub hook_timeout: u64,
#[serde(default, skip_serializing_if = "String::is_empty")]
pub dispatch_env_hook: String,
#[serde(default, skip_serializing_if = "is_zero")]
pub dispatch_env_hook_timeout: u64,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub dag_sets: Vec<String>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub node_sets: Vec<String>,
#[serde(default)]
pub adoptions: u32,
#[serde(default, skip_serializing_if = "Filters::is_empty")]
pub filters: Filters,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub bus_config: Option<RecordedBusConfig>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub maintenance_config: Option<crate::maintenance::MaintenanceConfig>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub oneharness_sessions: Option<PathBuf>,
}
impl LaunchRecord {
pub fn driven_by_this_process(&mut self) {
self.pid = sys::pid();
self.host = sys::hostname();
self.started = sys::process_start_token(self.pid)
.map(|token| token.recorded().to_string())
.unwrap_or_default();
}
pub fn watched_by(&mut self, graph_run: String) {
self.graph_run = graph_run;
if !self.graph_run.is_empty() {
self.observer_runs.push(self.graph_run.clone());
}
self.observer_ending.clear();
}
pub fn observer_graph(&self) -> Option<&str> {
(!self.graph.is_empty()).then_some(self.graph.as_str())
}
pub fn pr_author_graph(&self) -> Option<&str> {
(!self.pr_author_graph.is_empty()).then_some(self.pr_author_graph.as_str())
}
pub fn node_validator(&self) -> Option<&str> {
(!self.node_validator.is_empty()).then_some(self.node_validator.as_str())
}
pub fn envelope_reviewer(&self) -> Option<&str> {
(!self.envelope_reviewer.is_empty()).then_some(self.envelope_reviewer.as_str())
}
pub fn envelope_reviewer_bar(&self) -> Option<&str> {
(!self.envelope_reviewer_bar.is_empty()).then_some(self.envelope_reviewer_bar.as_str())
}
pub fn owned_by(&self, session: &str) -> bool {
owned_by(&self.session, session)
}
pub fn owner_label(&self, session: &str) -> String {
owner_label(&self.launcher, &self.session, session)
}
pub fn driver_pid(&self) -> Option<NonZeroU32> {
NonZeroU32::new(self.pid)
}
pub fn driver_stamp(&self) -> Option<&str> {
(!self.started.is_empty()).then_some(self.started.as_str())
}
pub fn recorded_host(&self) -> Option<&str> {
(!self.host.is_empty()).then_some(self.host.as_str())
}
pub fn launched_at(&self) -> Option<&str> {
(!self.started_at.is_empty()).then_some(self.started_at.as_str())
}
pub fn pacemaker_interval(&self) -> Option<u64> {
(self.heartbeat_interval > 0).then_some(self.heartbeat_interval)
}
pub fn item_budget(&self) -> Option<NonZeroU64> {
NonZeroU64::new(self.writeback_item_budget)
}
pub fn success_hook(&self) -> Option<&str> {
(!self.success_hook.is_empty()).then_some(self.success_hook.as_str())
}
pub fn failure_hook(&self) -> Option<&str> {
(!self.failure_hook.is_empty()).then_some(self.failure_hook.as_str())
}
pub fn hook_timeout(&self) -> NonZeroU64 {
NonZeroU64::new(self.hook_timeout).unwrap_or(crate::cli::DEFAULT_HOOK_TIMEOUT_SECONDS)
}
pub fn dispatch_env_hook(&self) -> Option<&str> {
(!self.dispatch_env_hook.is_empty()).then_some(self.dispatch_env_hook.as_str())
}
pub fn dispatch_env_hook_timeout(&self) -> NonZeroU64 {
NonZeroU64::new(self.dispatch_env_hook_timeout)
.unwrap_or(crate::cli::DEFAULT_DISPATCH_ENV_HOOK_TIMEOUT_SECONDS)
}
}
fn is_zero(value: &u64) -> bool {
*value == 0
}
fn attributed(recorded: &str) -> bool {
!recorded.is_empty() && recorded != sys::UNKNOWN_LAUNCHER
}
pub(crate) fn owned_by(recorded: &str, reader: &str) -> bool {
attributed(recorded) && recorded == reader
}
pub(crate) fn owner_label(launcher: &str, recorded: &str, reader: &str) -> String {
if !attributed(recorded) {
"[unknown]".to_string()
} else if recorded == reader {
"[mine]".to_string()
} else {
format!("[{launcher}:{}]", sys::session_digest(recorded))
}
}
pub fn read_json<T: serde::de::DeserializeOwned>(path: &Path) -> Result<T> {
let text = fs::read_to_string(path).map_err(|e| Error::Ledger {
path: path.to_path_buf(),
source: e,
})?;
counted(text.len(), ());
serde_json::from_str(&text).map_err(|e| Error::Invalid(format!("{}: {e}", path.display())))
}
pub fn read_json_opt<T: serde::de::DeserializeOwned>(path: &Path) -> Option<T> {
fs::read_to_string(path)
.ok()
.map(|text| counted(text.len(), text))
.and_then(|text| serde_json::from_str(&text).ok())
}
pub fn write_json<T: Serialize>(path: &Path, value: &T) -> Result<()> {
let body = serde_json::to_string_pretty(value)
.map_err(|e| Error::Invalid(format!("{}: {e}", path.display())))?;
write_atomic(path, body.as_bytes())
}
pub fn write_atomic(path: &Path, bytes: &[u8]) -> Result<()> {
let ledger = |e: io::Error| Error::Ledger {
path: path.to_path_buf(),
source: e,
};
if let Some(parent) = path.parent() {
fs::create_dir_all(parent).map_err(ledger)?;
}
let temp = path.with_extension(format!("tmp.{}", sys::pid()));
fs::write(&temp, bytes).map_err(ledger)?;
fs::rename(&temp, path).map_err(ledger)
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct TornTail {
pub at: String,
pub offset: u64,
pub bytes: u64,
pub healed_by: u32,
}
pub fn torn_tail_log(path: &Path) -> PathBuf {
let mut name = path.file_name().map_or_else(
|| std::ffi::OsString::from("torn"),
std::ffi::OsStr::to_os_string,
);
name.push(".torn");
path.with_file_name(name)
}
pub fn torn_tails(path: &Path) -> Vec<TornTail> {
read_lines(&torn_tail_log(path))
.iter()
.filter_map(|line| serde_json::from_str(line).ok())
.collect()
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Record {
pub line: usize,
pub offset: u64,
pub text: String,
pub bytes: u64,
pub terminated: bool,
}
#[cfg(test)]
pub fn append_line(path: &Path, line: &str) -> Result<()> {
append_line_healed(path, line).map(|_| ())
}
pub fn append_line_healed(path: &Path, line: &str) -> Result<u64> {
let (healed, appended) = append_line_locked(path, line);
if let Some(torn) = &healed {
report_torn_tail(path, torn);
}
appended.map(|()| healed.map_or(0, |torn| torn.bytes))
}
fn append_line_locked(path: &Path, line: &str) -> (Option<TornTail>, Result<()>) {
let (torn, opened) = open_healed(path);
let mut file = match opened {
Ok(file) => file,
Err(e) => return (torn, Err(e)),
};
let appended = write_record(path, &mut file, line);
(torn, appended)
}
fn open_healed(path: &Path) -> (Option<TornTail>, Result<fs::File>) {
let ledger = |e: io::Error| Error::Ledger {
path: path.to_path_buf(),
source: e,
};
if let Some(parent) = path.parent() {
if let Err(e) = fs::create_dir_all(parent) {
return (None, Err(ledger(e)));
}
}
let mut file = match sys::open_locked_append(path) {
Ok(file) => file,
Err(e) => return (None, Err(ledger(e))),
};
match heal_tail(&mut file) {
Ok(torn) => (torn, Ok(file)),
Err(e) => (None, Err(ledger(e))),
}
}
fn write_record(path: &Path, file: &mut fs::File, line: &str) -> Result<()> {
use std::io::{Seek, SeekFrom, Write};
let ledger = |e: io::Error| Error::Ledger {
path: path.to_path_buf(),
source: e,
};
let boundary = file.seek(SeekFrom::End(0)).map_err(ledger)?;
let written = file
.write_all(format!("{line}\n").as_bytes())
.and_then(|()| file.flush());
match written {
Ok(()) => Ok(()),
Err(e) => {
let _ = file.set_len(boundary);
Err(ledger(e))
}
}
}
fn heal_tail(file: &mut fs::File) -> io::Result<Option<TornTail>> {
use std::io::{Read, Seek, SeekFrom};
let length = file.metadata()?.len();
if length == 0 {
return Ok(None);
}
let mut last = [0u8; 1];
file.seek(SeekFrom::Start(length - 1))?;
file.read_exact(&mut last)?;
if last[0] == b'\n' {
return Ok(None);
}
const CHUNK: u64 = 64 * 1024;
let mut buffer = vec![0u8; CHUNK as usize];
let mut end = length;
let mut boundary = 0;
while end > 0 {
let start = end.saturating_sub(CHUNK);
let size = (end - start) as usize;
file.seek(SeekFrom::Start(start))?;
file.read_exact(&mut buffer[..size])?;
if let Some(at) = buffer[..size].iter().rposition(|byte| *byte == b'\n') {
boundary = start + at as u64 + 1;
break;
}
end = start;
}
file.set_len(boundary)?;
Ok(Some(TornTail {
at: sys::now_rfc3339(),
offset: boundary,
bytes: length - boundary,
healed_by: sys::pid(),
}))
}
fn report_torn_tail(path: &Path, torn: &TornTail) {
eprintln!(
"onepipeline: {}: discarded a {}-byte record fragment at byte {}, left by a writer \
that did not finish it; the record it was is lost",
path.display(),
torn.bytes,
torn.offset
);
let Ok(line) = serde_json::to_string(torn) else {
return;
};
let (healed, _) = append_line_locked(&torn_tail_log(path), &line);
if let Some(torn) = healed {
eprintln!(
"onepipeline: {}: discarded a {}-byte fragment of the loss log itself",
torn_tail_log(path).display(),
torn.bytes
);
}
}
pub fn read_records(path: &Path) -> Vec<Record> {
let Ok(bytes) = fs::read(path) else {
return Vec::new();
};
counted(bytes.len(), ());
records_of(&bytes, 0)
}
fn records_of(bytes: &[u8], base: u64) -> Vec<Record> {
let mut records = Vec::new();
let mut offset = base;
for (index, line) in bytes.split_inclusive(|byte| *byte == b'\n').enumerate() {
let terminated = line.ends_with(b"\n");
let text = String::from_utf8_lossy(line)
.trim_end_matches('\n')
.trim_end_matches('\r')
.to_string();
let bytes = line.len() as u64 - u64::from(terminated);
records.push(Record {
line: index + 1,
offset,
text,
bytes,
terminated,
});
offset += line.len() as u64;
}
records
}
pub(crate) struct EnvelopeLine {
pub(crate) line: usize,
pub(crate) offset: u64,
pub(crate) bytes: u64,
pub(crate) terminated: bool,
pub(crate) envelope: Option<crate::event::Envelope>,
}
impl EnvelopeLine {
pub(crate) fn text(&self, path: &Path) -> Option<String> {
read_range(path, self.offset, self.bytes).map(|bytes| {
String::from_utf8_lossy(&bytes)
.trim_end_matches('\r')
.to_string()
})
}
}
pub(crate) fn read_envelope_lines(path: &Path, from: u64) -> Vec<EnvelopeLine> {
let Ok(reader) = onemessagebus_agent::Reader::open_at(path, from) else {
return Vec::new();
};
let mut lines = Vec::new();
let mut start = from;
for (index, reading) in reader.enumerate() {
let line = index + 1;
match reading {
onemessagebus::Reading::Record(record) => {
lines.push(EnvelopeLine {
line,
offset: start,
bytes: record.position - start - 1,
terminated: true,
envelope: Some(record.envelope),
});
start = record.position;
}
onemessagebus::Reading::Refused(refused) => {
lines.push(EnvelopeLine {
line,
offset: refused.at,
bytes: refused.position - refused.at - 1,
terminated: true,
envelope: None,
});
start = refused.position;
}
onemessagebus::Reading::Torn(torn) => {
lines.push(EnvelopeLine {
line,
offset: torn.at,
bytes: torn.bytes,
terminated: false,
envelope: None,
});
start = torn.at + torn.bytes;
}
}
}
counted(usize::try_from(start - from).unwrap_or(usize::MAX), lines)
}
pub(crate) fn read_range(path: &Path, from: u64, len: u64) -> Option<Vec<u8>> {
use std::io::{Read, Seek, SeekFrom};
let mut file = fs::File::open(path).ok()?;
file.seek(SeekFrom::Start(from)).ok()?;
let want = usize::try_from(len).ok()?;
let mut bytes = vec![0u8; want];
file.read_exact(&mut bytes).ok()?;
Some(counted(bytes.len(), bytes))
}
pub fn read_lines(path: &Path) -> Vec<String> {
read_records(path)
.into_iter()
.filter(|record| !record.text.trim().is_empty())
.map(|record| record.text)
.collect()
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct LockRecord {
pub pid: u32,
pub host: String,
pub acquired_at: String,
pub verb: String,
#[serde(default, skip_serializing_if = "String::is_empty")]
pub started: String,
}
fn claim_or_report_the_holder(path: &Path, run: &str, verb: &str) -> Result<()> {
if let Some(parent) = path.parent() {
fs::create_dir_all(parent).map_err(|e| Error::Ledger {
path: parent.to_path_buf(),
source: e,
})?;
}
let record = LockRecord {
pid: sys::pid(),
host: sys::hostname(),
acquired_at: sys::now_rfc3339(),
verb: verb.to_string(),
started: sys::process_start_token(sys::pid())
.map(|token| token.recorded().to_string())
.unwrap_or_default(),
};
let body = serde_json::to_string(&record)
.map_err(|e| Error::Invalid(format!("{}: {e}", path.display())))?;
loop {
if create_exclusively_filled(path, &body)? {
return Ok(());
}
let held = match read_lock_file(path)? {
LockFile::Record(held) => held,
LockFile::Absent => continue,
LockFile::Unreadable => return Err(unreadable_lock(path, run)),
};
if !(held.host == sys::hostname() && !sys::process_may_be_live(held.pid)) {
return Err(locked_by(run, &held));
}
match reclaim(path, &held, &body)? {
Reclaimed::Won => return Ok(()),
Reclaimed::HeldBy(holder) => return Err(locked_by(run, &holder)),
Reclaimed::Unreadable(at) => return Err(unreadable_lock(&at, run)),
Reclaimed::Released => {}
}
}
}
enum Reclaimed {
Won,
HeldBy(LockRecord),
Unreadable(PathBuf),
Released,
}
enum LockFile {
Record(LockRecord),
Absent,
Unreadable,
}
const LOCK_READ_PATIENCE: std::time::Duration = std::time::Duration::from_secs(1);
fn read_lock_file(path: &Path) -> Result<LockFile> {
let deadline = std::time::Instant::now() + LOCK_READ_PATIENCE;
loop {
match fs::read_to_string(path) {
Ok(text) => {
return Ok(match serde_json::from_str(&counted(text.len(), text)) {
Ok(record) => LockFile::Record(record),
Err(_) => LockFile::Unreadable,
})
}
Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(LockFile::Absent),
Err(_) if std::time::Instant::now() < deadline => {
std::thread::sleep(std::time::Duration::from_millis(1));
}
Err(e) => {
return Err(Error::Ledger {
path: path.to_path_buf(),
source: e,
})
}
}
}
}
fn reclaim(path: &Path, dead: &LockRecord, body: &str) -> Result<Reclaimed> {
let key = reclaim_key(dead);
let mut number = 1u64;
loop {
let entry = reclaim_entry(path, &key, number);
if create_exclusively_filled(&entry, body)? {
let outcome = match read_lock_file(path) {
Ok(LockFile::Record(now)) if now == *dead => {
write_atomic(path, body.as_bytes()).map(|()| Reclaimed::Won)
}
Ok(LockFile::Record(now)) => Ok(Reclaimed::HeldBy(now)),
Ok(LockFile::Absent) => Ok(Reclaimed::Released),
Ok(LockFile::Unreadable) => Ok(Reclaimed::Unreadable(path.to_path_buf())),
Err(refused) => Err(refused),
};
for done in 1..=number {
let _ = fs::remove_file(reclaim_entry(path, &key, done));
}
return outcome;
}
match read_lock_file(path)? {
LockFile::Record(now) if now == *dead => {}
LockFile::Record(now) => return Ok(Reclaimed::HeldBy(now)),
LockFile::Absent => return Ok(Reclaimed::Released),
LockFile::Unreadable => return Ok(Reclaimed::Unreadable(path.to_path_buf())),
}
match read_lock_file(&entry)? {
LockFile::Record(reclaimer)
if reclaimer.host == sys::hostname()
&& !sys::process_may_be_live(reclaimer.pid) =>
{
number += 1;
}
LockFile::Record(reclaimer) => return Ok(Reclaimed::HeldBy(reclaimer)),
LockFile::Absent => {}
LockFile::Unreadable => return Ok(Reclaimed::Unreadable(entry)),
}
}
}
fn create_exclusively_filled(path: &Path, body: &str) -> Result<bool> {
static NONCE: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
let ledger = |e: io::Error| Error::Ledger {
path: path.to_path_buf(),
source: e,
};
let mut name = path
.file_name()
.map(OsStr::to_os_string)
.unwrap_or_default();
name.push(format!(
".tmp.{}.{}",
sys::pid(),
NONCE.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
));
let temp = path.with_file_name(name);
if let Err(e) = fs::write(&temp, body) {
let _ = fs::remove_file(&temp);
return Err(ledger(e));
}
let linked = match fs::hard_link(&temp, path) {
Ok(()) => Ok(true),
Err(e) if e.kind() == io::ErrorKind::AlreadyExists => Ok(false),
Err(e) => Err(ledger(e)),
};
let _ = fs::remove_file(&temp);
linked
}
fn reclaim_key(dead: &LockRecord) -> String {
let plain =
|text: &str| -> String { text.chars().filter(char::is_ascii_alphanumeric).collect() };
format!(
"{}-{}-{}",
dead.pid,
plain(&dead.acquired_at),
plain(&dead.started)
)
}
fn reclaim_entry(path: &Path, key: &str, number: u64) -> PathBuf {
let mut name = path
.file_name()
.map(OsStr::to_os_string)
.unwrap_or_default();
name.push(format!(".reclaim.{key}.{number}"));
path.with_file_name(name)
}
fn locked_by(run: &str, holder: &LockRecord) -> Error {
Error::Locked {
run: run.to_string(),
pid: holder.pid,
host: holder.host.clone(),
verb: holder.verb.clone(),
}
}
#[derive(Debug)]
struct UnreadableLock {
run: String,
}
impl std::fmt::Display for UnreadableLock {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"run '{}' is claimed by an unreadable lock: what it holds is not a record this \
build can read, so no process can be named as its holder",
self.run
)
}
}
impl std::error::Error for UnreadableLock {}
fn unreadable_lock(path: &Path, run: &str) -> Error {
Error::Ledger {
path: path.to_path_buf(),
source: io::Error::new(
io::ErrorKind::InvalidData,
UnreadableLock {
run: run.to_string(),
},
),
}
}
pub(crate) fn is_unreadable_lock(error: &Error) -> bool {
matches!(
error,
Error::Ledger { source, .. }
if source.get_ref().is_some_and(|inner| inner.is::<UnreadableLock>())
)
}
#[derive(Debug)]
pub(crate) struct Handover {
entry: PathBuf,
}
fn handover_entries(paths: &RunPaths) -> PathBuf {
paths.channel("handover")
}
pub(crate) const REPLY_VERB: &str = "reply";
pub(crate) const DRIVE_VERB: &str = "drive";
const HANDOVER_PATIENCE: std::time::Duration = std::time::Duration::from_secs(30);
const NUMBERS_TRIED: usize = 1_000;
impl Handover {
pub(crate) fn hold(paths: &RunPaths) -> Result<Self> {
Self::hold_within(paths, HANDOVER_PATIENCE)
}
pub(crate) fn hold_within(paths: &RunPaths, patience: std::time::Duration) -> Result<Self> {
let dir = handover_entries(paths);
fs::create_dir_all(&dir).map_err(|e| not_taken(&paths.run, &e.to_string()))?;
let host = sys::hostname();
let deadline = std::time::Instant::now() + patience;
for _ in 0..NUMBERS_TRIED {
let top = highest_entry_in(&dir).map_err(|e| not_taken(&paths.run, &e.to_string()))?;
if top.is_some_and(|top| !holder_is_gone(&dir, top, &host)) {
if std::time::Instant::now() >= deadline {
return Err(not_taken(
&paths.run,
&format!(
"it has been held for {}s by a party this process cannot show \
has gone",
patience.as_secs()
),
));
}
std::thread::sleep(std::time::Duration::from_millis(5));
continue;
}
let Some(below) = top.unwrap_or(0).checked_add(1).map(|_| top.unwrap_or(0)) else {
return Err(not_taken(
&paths.run,
"its order is at the highest number this build can write",
));
};
match Self::take_above(paths, &dir, below, sys::pid(), &host)? {
Some(held) => return Ok(held),
None => continue,
}
}
Err(not_taken(
&paths.run,
&format!("{NUMBERS_TRIED} numbers in its order were taken while this process looked"),
))
}
fn take_above(
paths: &RunPaths,
dir: &Path,
observed: u64,
pid: u32,
host: &str,
) -> Result<Option<Self>> {
let entry = dir.join(entry_named(observed + 1));
let mut file = match fs::OpenOptions::new()
.write(true)
.create_new(true)
.open(&entry)
{
Ok(file) => file,
Err(e) if e.kind() == io::ErrorKind::AlreadyExists => return Ok(None),
Err(e) => return Err(not_taken(&paths.run, &e.to_string())),
};
use std::io::Write;
if let Err(e) = file.write_all(body_of_entry(pid, host).as_bytes()) {
drop(file);
let _ = fs::remove_file(&entry);
return Err(not_taken(&paths.run, &e.to_string()));
}
Ok(Some(Self { entry }))
}
}
fn highest_entry_in(dir: &Path) -> Result<Option<u64>> {
let listing = fs::read_dir(dir).map_err(|e| Error::Ledger {
path: dir.to_path_buf(),
source: e,
})?;
let mut highest: Option<u64> = None;
let mut a_name_this_build_did_not_write = false;
for entry in listing {
let name = entry
.map_err(|e| Error::Ledger {
path: dir.to_path_buf(),
source: e,
})?
.file_name();
match number_of_entry(Path::new(&name)) {
Some(number) => highest = Some(highest.map_or(number, |high: u64| high.max(number))),
None => a_name_this_build_did_not_write = true,
}
}
match (highest, a_name_this_build_did_not_write) {
(_, true) => Ok(Some(u64::MAX)),
(highest, false) => Ok(highest),
}
}
fn holder_is_gone(dir: &Path, number: u64, this_host: &str) -> bool {
let Ok(body) = fs::read_to_string(dir.join(entry_named(number))) else {
return false;
};
let Some((pid, host)) = identity_of_body(&body) else {
return false;
};
host == this_host && !sys::process_may_be_live(pid)
}
fn entry_named(number: u64) -> String {
format!("{number:020}")
}
fn number_of_entry(entry: &Path) -> Option<u64> {
let name = entry.file_name()?.to_str()?;
(name.len() == 20 && name.bytes().all(|byte| byte.is_ascii_digit()))
.then(|| name.parse().ok())
.flatten()
}
fn body_of_entry(pid: u32, host: &str) -> String {
format!("{pid} {host}")
}
fn identity_of_body(body: &str) -> Option<(u32, &str)> {
let (pid, host) = body.trim().split_once(' ')?;
Some((pid.parse().ok()?, host))
}
fn not_taken(run: &str, because: &str) -> Error {
Error::Refused(format!(
"the handover gate of run '{run}' could not be taken, so nothing was accepted onto \
its command queue and nothing was released: {because}. This process is not inside \
the gate, and going on without it is what would let an edit be accepted by a run \
whose owner has already left"
))
}
impl Drop for Handover {
fn drop(&mut self) {
let _ = fs::remove_file(&self.entry);
}
}
#[derive(Debug)]
pub struct OwnershipLock {
path: PathBuf,
held: bool,
}
impl OwnershipLock {
pub fn acquire(paths: &RunPaths, verb: &str) -> Result<Self> {
let path = paths.lock();
claim_or_report_the_holder(&path, &paths.run, verb)?;
Ok(Self { path, held: true })
}
pub fn release(mut self) {
self.remove();
}
pub(crate) fn abandon(mut self) {
self.held = false;
}
fn remove(&mut self) {
if self.held {
let _ = fs::remove_file(&self.path);
self.held = false;
}
}
}
impl Drop for OwnershipLock {
fn drop(&mut self) {
self.remove();
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct DispatchRecord {
pub node: String,
pub pid: u32,
pub host: String,
pub dispatched_at: String,
pub started: String,
}
impl DispatchRecord {
fn is_usable(&self) -> bool {
!self.started.trim().is_empty()
}
}
#[derive(Debug)]
pub struct DispatchClaim {
path: PathBuf,
started: String,
}
impl Drop for DispatchClaim {
fn drop(&mut self) {
let ours = read_json_opt::<DispatchRecord>(&self.path)
.is_some_and(|held| held.started == self.started);
if ours {
let _ = fs::remove_file(&self.path);
}
}
}
pub fn claim_dispatch(paths: &RunPaths, node: &str, pid: u32) -> Result<DispatchClaim> {
static CLAIMED: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
let claim = CLAIMED.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
let Some(started) = sys::process_start_token(pid) else {
return Err(Error::Refused(format!(
"run '{}': node '{node}': this host will not say when pid {pid} started, so its \
dispatch cannot be recorded as running there and nothing could prove that pid is \
still this run's work",
paths.run
)));
};
let record = DispatchRecord {
node: node.to_string(),
pid,
host: sys::hostname(),
dispatched_at: sys::now_rfc3339(),
started: started.recorded().to_string(),
};
let path = paths.dispatch(pid, claim);
write_dispatch(paths, &path, claim, &record)?;
match read_json::<DispatchRecord>(&path) {
Ok(held) if held == record => Ok(DispatchClaim {
path,
started: record.started,
}),
Ok(_) | Err(_) => Err(Error::Refused(format!(
"run '{}': node '{node}': its dispatch in pid {pid} was written to {} and did not \
read back as itself, so the run cannot say where that work is",
paths.run,
path.display()
))),
} }
fn write_dispatch(
paths: &RunPaths,
path: &Path,
claim: u64,
record: &DispatchRecord,
) -> Result<()> {
let body = serde_json::to_string_pretty(record)
.map_err(|e| Error::Invalid(format!("{}: {e}", path.display())))?;
let ledger = |at: &Path| {
let at = at.to_path_buf();
move |source: io::Error| Error::Ledger { path: at, source }
};
fs::create_dir_all(paths.dispatches()).map_err(ledger(&paths.dispatches()))?;
let temp = paths
.dir
.join(format!("dispatch-{}-{claim}.tmp", record.pid));
fs::write(&temp, body.as_bytes()).map_err(ledger(&temp))?;
fs::rename(&temp, path).map_err(ledger(path))
}
pub fn dispatches_of(paths: &RunPaths) -> Result<Vec<DispatchRecord>> {
let registry = paths.dispatches();
let listed = fs::read_dir(®istry).map_err(|source| Error::Ledger {
path: registry.clone(),
source,
})?;
let mut found = Vec::new();
for entry in listed {
let entry = entry.map_err(|source| Error::Ledger {
path: registry.clone(),
source,
})?; let held: DispatchRecord = read_json(&entry.path())?;
if !held.is_usable() {
return Err(Error::Invalid(format!(
"{}: the dispatch it records carries no start token, so nothing says pid {} is \
still this run's work",
entry.path().display(),
held.pid
)));
}
found.push(held);
}
found.sort_by_key(|held| held.pid);
Ok(found)
}
#[cfg(test)]
mod tests {
use super::{body_of_entry, entry_named, identity_of_body, number_of_entry, Handover};
use std::time::Duration;
fn gate_scratch(name: &str) -> RunPaths {
let dir = std::env::temp_dir().join(format!("onepipeline-gate-{name}-{}", sys::pid()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(dir.join("channel")).expect("a run directory");
RunPaths {
run: name.to_owned(),
dir,
}
}
fn entries_in(paths: &RunPaths) -> Vec<u64> {
let mut numbers: Vec<u64> = std::fs::read_dir(paths.channel("handover"))
.expect("the gate's entries")
.filter_map(|entry| number_of_entry(std::path::Path::new(&entry.ok()?.file_name())))
.collect();
numbers.sort_unstable();
numbers
}
#[test]
fn two_parties_that_looked_at_the_same_gate_do_not_both_enter_it() {
let paths = gate_scratch("looked-together");
let dir = paths.channel("handover");
std::fs::create_dir_all(&dir).expect("the gate's entries");
let the_first_looked = super::highest_entry_in(&dir).expect("the gate reads");
let the_second_looked = super::highest_entry_in(&dir).expect("the gate reads");
assert_eq!((the_first_looked, the_second_looked), (None, None));
let first = Handover::take_above(&paths, &dir, 0, 200, "this-host")
.expect("the first party's write is answered")
.expect("the first party takes the gate");
let second = Handover::take_above(&paths, &dir, 0, 100, "this-host")
.expect("the second party's write is answered");
assert!(
second.is_none(),
"two parties took the same number, so both are inside the gate: {:?}",
entries_in(&paths)
);
assert_eq!(
entries_in(&paths),
vec![1],
"the gate holds an entry no party is accountable for"
);
Handover::hold_within(&paths, Duration::from_millis(50))
.expect_err("the gate another party is inside is not taken");
drop(first);
std::fs::remove_dir_all(&paths.dir).ok();
}
#[test]
fn a_party_arriving_while_another_is_inside_does_not_get_in() {
let paths = gate_scratch("late-arrival");
let inside = Handover::hold(&paths).expect("this thread is inside the gate");
assert_eq!(entries_in(&paths), vec![1]);
Handover::hold_within(&paths, Duration::from_millis(50))
.expect_err("a gate another party is inside is not taken");
assert_eq!(
entries_in(&paths),
vec![1],
"an arrival numbered itself over a holder that is alive"
);
drop(inside);
let next = Handover::hold(&paths).expect("the gate is free once its holder lets go");
drop(next);
std::fs::remove_dir_all(&paths.dir).ok();
}
#[test]
fn a_holder_this_host_knows_is_gone_is_stepped_over_rather_than_removed() {
let paths = gate_scratch("holder-gone");
let dir = paths.channel("handover");
std::fs::create_dir_all(&dir).expect("the gate's entries");
std::fs::write(dir.join(entry_named(1)), body_of_entry(0, &sys::hostname()))
.expect("the entry a holder that died left behind");
let held = Handover::hold_within(&paths, Duration::from_millis(50))
.expect("a gate whose holder is gone is taken");
assert_eq!(
entries_in(&paths),
vec![1, 2],
"the entry of the holder that is gone was removed rather than stepped over"
);
Handover::hold_within(&paths, Duration::from_millis(50))
.expect_err("the gate is held, so a second party is refused");
drop(held);
std::fs::remove_dir_all(&paths.dir).ok();
}
#[test]
fn a_holder_this_host_cannot_account_for_is_waited_on_rather_than_stepped_over() {
for (what, name, body) in [
(
"another host's",
entry_named(1),
body_of_entry(0, "somewhere-else"),
),
(
"an unreadable body",
entry_named(1),
"not an identity".to_owned(),
),
(
"a name from outside this build",
"handover.lock".to_owned(),
String::new(),
),
] {
let paths = gate_scratch("holder-unknown");
let dir = paths.channel("handover");
std::fs::create_dir_all(&dir).expect("the gate's entries");
std::fs::write(dir.join(&name), &body).expect("the entry is written");
Handover::hold_within(&paths, Duration::from_millis(50))
.err()
.unwrap_or_else(|| {
panic!("a gate held by {what} was taken, so two parties are inside it")
});
std::fs::remove_dir_all(&paths.dir).ok();
}
}
#[test]
fn a_gate_entrys_name_carries_its_number_and_its_body_the_holder() {
assert_eq!(
number_of_entry(std::path::Path::new(&entry_named(1))),
Some(1)
);
assert_eq!(
identity_of_body(&body_of_entry(4242, "a-host")),
Some((4242, "a-host"))
);
assert!(entry_named(2) > entry_named(1));
assert!(entry_named(10) > entry_named(2));
assert!(entry_named(u64::MAX) > entry_named(1_000_000));
for stranger in [
"handover.lock",
"1",
"0000000000000000000x",
"-0000000000000000001",
] {
assert_eq!(
number_of_entry(std::path::Path::new(stranger)),
None,
"'{stranger}' was read as a name this build wrote"
);
}
for stranger in ["", "notanumber a-host", "4242"] {
assert_eq!(
identity_of_body(stranger),
None,
"'{stranger}' was read as an identity this build wrote"
);
}
}
use super::*;
fn scratch(name: &str) -> PathBuf {
let dir = std::env::temp_dir().join(format!("onepipeline-ledger-{name}-{}", sys::pid()));
let _ = fs::remove_dir_all(&dir);
fs::create_dir_all(&dir).expect("a scratch root");
dir
}
#[test]
fn a_launch_that_declared_no_filters_writes_the_record_it_always_wrote() {
let record = LaunchRecord {
filters: Filters::default(),
..a_record()
};
let text = serde_json::to_string(&record).expect("it serialises");
assert!(
!text.contains("filters"),
"an empty filters block reached the record: {text}"
);
assert_eq!(
serde_json::from_str::<LaunchRecord>(&text).expect("it re-parses"),
record
);
let older = serde_json::json!({
"run_id": "demo",
"plan": "plan.json",
"node_graph": "graphs/node-scope.yaml",
"launcher": "claude-code",
"session": "a-session",
"pid": 1,
"host": "h",
"started_at": "2026-08-15T00:00:00.000Z",
"heartbeat_interval": 1800,
});
let read: LaunchRecord =
serde_json::from_value(older).expect("a record predating the field still reads");
assert!(read.filters.is_empty());
}
#[test]
fn a_launchs_filters_survive_the_record_they_are_retained_in() {
let declared = Filters {
agentgraph: Some(
crate::filter::EventFilter::parse(r#"{"exclude": [{"kind": "turn-*"}]}"#)
.expect("a filter"),
),
vcs: Some(
crate::filter::EventFilter::parse(r#"{"include": [{"kind": "gate-*"}]}"#)
.expect("a filter"),
),
profiles: [(
"planner".to_string(),
crate::filter::EventFilter::parse(r#"{"include": [{"source": "pipeline"}]}"#)
.expect("a filter"),
)]
.into_iter()
.collect(),
};
let record = LaunchRecord {
filters: declared.clone(),
..a_record()
};
let text = serde_json::to_string(&record).expect("it serialises");
let read: LaunchRecord = serde_json::from_str(&text).expect("it re-parses");
assert_eq!(read.filters, declared);
assert_eq!(read, record);
}
fn a_record() -> LaunchRecord {
LaunchRecord {
run_id: "demo".into(),
project: "plans:demo".into(),
dir: PathBuf::from("/tmp/launch"),
graph: String::new(),
graph_run: String::new(),
observer_runs: Vec::new(),
observer_ending: String::new(),
node_graph: "graphs/node-scope.yaml".into(),
pr_author_graph: String::new(),
node_validator: String::new(),
envelope_reviewer: String::new(),
launcher: "claude-code".into(),
session: "a-session".into(),
pid: 1,
host: "h".into(),
started: "Fri Aug 15 00:00:00 2026".into(),
started_at: sys::now_rfc3339(),
heartbeat_interval: 1_800,
writeback_item_budget: 10,
success_hook: "./scripts/follow-up.sh".into(),
failure_hook: "./scripts/report-failure.sh".into(),
hook_timeout: 45,
dispatch_env_hook: "./scripts/dispatch-env.sh".into(),
dispatch_env_hook_timeout: 20,
dag_sets: Vec::new(),
node_sets: Vec::new(),
adoptions: 0,
filters: Filters::default(),
bus_config: Default::default(),
maintenance_config: None,
oneharness_sessions: None,
envelope_reviewer_bar: Default::default(),
}
}
#[test]
fn claiming_a_run_records_the_stamp_that_proves_its_pid_and_an_older_record_carries_none() {
let mut record = a_record();
record.driven_by_this_process();
assert_eq!(record.pid, sys::pid());
assert_eq!(record.host, sys::hostname());
assert!(
sys::process_start_token(sys::pid())
.expect("this host says when a process started")
.matches(&record.started),
"a run claimed by this process recorded a stamp that does not prove it"
);
let text = serde_json::to_string(&record).expect("it serialises");
assert_eq!(
serde_json::from_str::<LaunchRecord>(&text).expect("it re-parses"),
record
);
let older = serde_json::json!({
"run_id": "demo",
"plan": "plan.json",
"node_graph": "graphs/node-scope.yaml",
"launcher": "claude-code",
"session": "a-session",
"pid": sys::pid(),
"host": sys::hostname(),
"started_at": "2026-08-15T00:00:00.000Z",
"heartbeat_interval": 1800,
});
let read: LaunchRecord =
serde_json::from_value(older).expect("a record predating the stamp still reads");
assert!(read.started.is_empty());
assert!(
!sys::process_start_token(sys::pid())
.expect("this host says when a process started")
.matches(&read.started),
"a record carrying no stamp proved a live pid"
);
let text = serde_json::to_string(&LaunchRecord {
started: String::new(),
..a_record()
})
.expect("it serialises");
assert!(
!text.contains("started\""),
"an empty stamp reached the record: {text}"
);
}
#[test]
fn a_launch_record_written_before_any_of_these_eleven_keys_still_reads() {
const HISTORICAL: [&str; 11] = [
"session",
"pid",
"host",
"started_at",
"heartbeat_interval",
"writeback_item_budget",
"success_hook",
"failure_hook",
"hook_timeout",
"dispatch_env_hook",
"dispatch_env_hook_timeout",
];
let root = scratch("historical-launch");
let whole = serde_json::to_value(a_record()).expect("a record this build writes");
let read_one = |name: &str, without: &[&str]| -> LaunchRecord {
let mut document = whole.clone();
let fields = document.as_object_mut().expect("a launch record");
for key in without {
assert!(
fields.remove(*key).is_some(),
"the record this build writes carries no `{key}` to take away"
);
}
let dir = root.join(name);
fs::create_dir_all(&dir).expect("a run root");
let launch = dir.join("launch.json");
fs::write(&launch, document.to_string()).expect("a record an older build wrote");
read_json::<LaunchRecord>(&launch).unwrap_or_else(|error| {
panic!("a record written before `{without:?}` existed was refused: {error}")
})
};
for key in HISTORICAL {
let read = read_one(key, &[key]);
assert_eq!(read.run_id, "demo");
match key {
"session" => {
assert!(read.session.is_empty(), "a session was invented: {read:?}");
assert!(
!read.owned_by(""),
"a record naming no session was owned by a reader that names none either"
);
assert_eq!(read.owner_label("a-session"), "[unknown]");
}
"pid" => {
assert_eq!(read.pid, 0);
assert_eq!(
read.driver_pid(),
None,
"a pid nobody wrote was served as one a reader may act on"
);
}
"host" => {
assert!(read.host.is_empty());
assert_eq!(
read.recorded_host(),
None,
"a host nobody named was served as one"
);
}
"started_at" => {
assert_eq!(
read.launched_at(),
None,
"a launch instant nobody recorded was served as an instant"
);
}
"heartbeat_interval" => {
assert_eq!(read.heartbeat_interval, 0);
assert_eq!(
read.pacemaker_interval(),
None,
"a record naming no interval produced a check-in interval"
);
assert_ne!(
read.pacemaker_interval(),
Some(0),
"a zero-second check-in was served as an interval"
);
}
"writeback_item_budget" => {
assert_eq!(read.writeback_item_budget, 0);
assert_eq!(
read.item_budget(),
None,
"a record naming no budget produced a per-item budget"
);
assert_eq!(
read.item_budget().map(NonZeroU64::get),
None,
"a zero-second budget was served as one"
);
}
"success_hook" => {
assert_eq!(read.success_hook(), None, "a success hook was invented");
assert_eq!(read.failure_hook(), Some("./scripts/report-failure.sh"));
}
"failure_hook" => {
assert_eq!(read.failure_hook(), None, "a failure hook was invented");
assert_eq!(read.success_hook(), Some("./scripts/follow-up.sh"));
}
"hook_timeout" => {
assert_eq!(read.hook_timeout, 0);
assert_eq!(
read.hook_timeout(),
crate::cli::DEFAULT_HOOK_TIMEOUT_SECONDS,
"a record naming no timeout was not given the shipped one"
);
}
"dispatch_env_hook" => {
assert_eq!(
read.dispatch_env_hook(),
None,
"a dispatch-env hook was invented"
);
assert_eq!(read.success_hook(), Some("./scripts/follow-up.sh"));
}
"dispatch_env_hook_timeout" => {
assert_eq!(read.dispatch_env_hook_timeout, 0);
assert_eq!(
read.dispatch_env_hook_timeout(),
crate::cli::DEFAULT_DISPATCH_ENV_HOOK_TIMEOUT_SECONDS,
"a record naming no dispatch-env timeout was not given the shipped one"
);
}
other => unreachable!("{other} is not one of the eleven"),
}
}
let oldest = read_one("all-eleven", &HISTORICAL);
assert_eq!(oldest.success_hook(), None);
assert_eq!(oldest.failure_hook(), None);
assert_eq!(
oldest.hook_timeout(),
crate::cli::DEFAULT_HOOK_TIMEOUT_SECONDS
);
assert_eq!(oldest.dispatch_env_hook(), None);
assert_eq!(
oldest.dispatch_env_hook_timeout(),
crate::cli::DEFAULT_DISPATCH_ENV_HOOK_TIMEOUT_SECONDS
);
assert!(oldest.session.is_empty());
assert_eq!(oldest.owner_label("a-session"), "[unknown]");
assert!(!oldest.owned_by(sys::UNKNOWN_LAUNCHER));
assert_eq!(oldest.driver_pid(), None);
assert_eq!(oldest.recorded_host(), None);
assert_eq!(oldest.launched_at(), None);
assert_eq!(oldest.pacemaker_interval(), None);
assert_eq!(oldest.item_budget(), None);
assert_eq!(oldest.run_id, "demo");
assert_eq!(oldest.launcher, "claude-code");
assert_eq!(oldest.node_graph, "graphs/node-scope.yaml");
let whole = read_one("whole", &[]);
assert_eq!(whole.success_hook(), Some("./scripts/follow-up.sh"));
assert_eq!(whole.failure_hook(), Some("./scripts/report-failure.sh"));
assert_eq!(whole.hook_timeout(), NonZeroU64::new(45).expect("nonzero"));
assert_eq!(whole.dispatch_env_hook(), Some("./scripts/dispatch-env.sh"));
assert_eq!(
whole.dispatch_env_hook_timeout(),
NonZeroU64::new(20).expect("nonzero")
);
assert_eq!(whole.session, "a-session");
assert_eq!(whole.driver_pid(), NonZeroU32::new(1));
assert_eq!(whole.recorded_host(), Some("h"));
assert!(whole.launched_at().is_some());
assert_eq!(whole.pacemaker_interval(), Some(1_800));
assert_eq!(whole.item_budget(), NonZeroU64::new(10));
}
#[test]
fn concurrent_appenders_each_land_a_whole_line() {
let root = scratch("append");
let path = root.join("events.jsonl");
const WRITERS: usize = 8;
const EACH: usize = 60;
std::thread::scope(|scope| {
for writer in 0..WRITERS {
let path = path.clone();
scope.spawn(move || {
for n in 0..EACH {
let line = serde_json::json!({
"writer": writer,
"seq": n,
"payload": "x".repeat(512),
})
.to_string();
append_line(&path, &line).expect("the line is appended");
}
});
}
});
let lines = read_lines(&path);
assert_eq!(lines.len(), WRITERS * EACH, "a record was torn or lost");
for line in &lines {
serde_json::from_str::<serde_json::Value>(line)
.unwrap_or_else(|e| panic!("a torn record reached the file: {e}: {line}"));
}
fs::remove_dir_all(&root).ok();
}
#[test]
fn a_lock_refuses_a_second_writer_and_names_the_first() {
let root = scratch("lock");
let paths = RunPaths::under(&root, "demo");
paths.create().expect("the run directory");
let first = OwnershipLock::acquire(&paths, "start").expect("the first writer wins");
let second = OwnershipLock::acquire(&paths, "adopt");
match second {
Err(Error::Locked { run, pid, verb, .. }) => {
assert_eq!(run, "demo");
assert_eq!(pid, sys::pid());
assert_eq!(verb, "start");
}
other => panic!("a second writer was not refused: {other:?}"),
}
first.release();
OwnershipLock::acquire(&paths, "adopt").expect("the lock was released");
fs::remove_dir_all(&root).ok();
}
#[test]
fn a_run_id_names_one_directory_and_never_a_path() {
for good in ["demo", "run-2", "a_b", "tracked-release", "R1"] {
assert!(is_valid_run_id(good), "{good} was refused");
}
for bad in [
"",
".",
"..",
"../elsewhere",
"../../elsewhere",
"a/b",
"a\\b",
"/absolute",
"./here",
] {
assert!(!is_valid_run_id(bad), "{bad:?} was accepted");
}
}
#[test]
fn a_lock_whose_holder_is_proved_gone_is_reclaimed() {
let root = scratch("stale");
let paths = RunPaths::under(&root, "demo");
paths.create().expect("the run directory");
let dead = sys::reaped_pid();
write_json(
&paths.lock(),
&LockRecord {
pid: dead,
host: sys::hostname(),
acquired_at: sys::now_rfc3339(),
verb: "start".to_string(),
started: String::new(),
},
)
.expect("a stale lock");
OwnershipLock::acquire(&paths, "start").expect("a dead holder's lock is reclaimed");
fs::remove_dir_all(&root).ok();
}
fn a_dead_holders_lock(paths: &RunPaths) -> LockRecord {
let dead = LockRecord {
pid: sys::reaped_pid(),
host: sys::hostname(),
acquired_at: sys::now_rfc3339(),
verb: "drive".to_string(),
started: String::new(),
};
write_json(&paths.lock(), &dead).expect("a dead holder's lock");
dead
}
fn reclaim_entries_beside(paths: &RunPaths) -> Vec<String> {
let mut names: Vec<String> = fs::read_dir(&paths.dir)
.expect("the run directory lists")
.map(|entry| {
entry
.expect("an entry")
.file_name()
.to_string_lossy()
.into_owned()
})
.filter(|name| name.contains(".reclaim."))
.collect();
names.sort();
names
}
#[test]
fn racing_reclaimers_of_one_dead_lock_leave_exactly_one_holder() {
const CONTENDERS: usize = 16;
const ROUNDS: usize = 25;
let root = scratch("reclaim-race");
let paths = RunPaths::under(&root, "demo");
paths.create().expect("the run directory");
for round in 0..ROUNDS {
a_dead_holders_lock(&paths);
let gate = std::sync::Arc::new(std::sync::Barrier::new(CONTENDERS));
let outcomes: Vec<Result<OwnershipLock>> = (0..CONTENDERS)
.map(|_| {
let gate = gate.clone();
let paths = paths.clone();
std::thread::spawn(move || {
gate.wait();
OwnershipLock::acquire(&paths, "reply")
})
})
.collect::<Vec<_>>()
.into_iter()
.map(|contender| contender.join().expect("a contender finishes"))
.collect();
let winners = outcomes.iter().filter(|outcome| outcome.is_ok()).count();
assert_eq!(
winners, 1,
"round {round}: {winners} contenders each believe they hold the run"
);
for lost in outcomes.iter().filter_map(|outcome| outcome.as_ref().err()) {
match lost {
Error::Locked {
pid, host, verb, ..
} => {
assert_eq!(*pid, sys::pid(), "round {round}: {lost}");
assert_eq!(*host, sys::hostname(), "round {round}: {lost}");
assert_eq!(verb, "reply", "round {round}: {lost}");
}
other => {
panic!("round {round}: a loser was not told who holds the run: {other}")
}
}
}
let held: LockRecord = read_json(&paths.lock()).expect("the lock reads back");
assert_eq!(
held.pid,
sys::pid(),
"round {round}: the lock does not name the winner"
);
assert_eq!(
reclaim_entries_beside(&paths),
Vec::<String>::new(),
"round {round}: the reclaim left its entries behind"
);
drop(outcomes);
}
fs::remove_dir_all(&root).ok();
}
#[test]
fn a_lock_is_never_observable_before_its_record_is_complete() {
const CLAIMS: usize = 500;
let root = scratch("lock-observed");
let paths = RunPaths::under(&root, "demo");
paths.create().expect("the run directory");
let taking = std::sync::atomic::AtomicBool::new(true);
let incomplete = std::thread::scope(|scope| {
let observer = scope.spawn(|| {
let mut incomplete = Vec::new();
while taking.load(std::sync::atomic::Ordering::Relaxed) {
if let Ok(text) = fs::read_to_string(paths.lock()) {
if serde_json::from_str::<LockRecord>(&text).is_err() {
incomplete.push(text);
}
}
}
incomplete
});
for _ in 0..CLAIMS {
OwnershipLock::acquire(&paths, "drive")
.expect("a lock nobody holds is taken")
.release();
}
taking.store(false, std::sync::atomic::Ordering::Relaxed);
observer.join().expect("the observer finishes")
});
assert!(
incomplete.is_empty(),
"a reader found the lock's name before its record was complete, {} times: {:?}",
incomplete.len(),
incomplete.iter().take(3).collect::<Vec<_>>()
);
fs::remove_dir_all(&root).ok();
}
#[test]
fn a_dead_reclaimers_entry_is_stepped_over_and_taken_away() {
let root = scratch("reclaim-stale");
let paths = RunPaths::under(&root, "demo");
paths.create().expect("the run directory");
let dead = a_dead_holders_lock(&paths);
let key = reclaim_key(&dead);
let abandoned = reclaim_entry(&paths.lock(), &key, 1);
write_json(
&abandoned,
&LockRecord {
pid: sys::reaped_pid(),
host: sys::hostname(),
acquired_at: sys::now_rfc3339(),
verb: "reply".to_string(),
started: String::new(),
},
)
.expect("an entry a reclaimer died holding");
let held = OwnershipLock::acquire(&paths, "adopt")
.expect("a dead reclaimer's entry does not keep the run from being reclaimed");
let record: LockRecord = read_json(&paths.lock()).expect("the lock reads back");
assert_eq!(record.pid, sys::pid());
assert_eq!(record.verb, "adopt");
assert!(
!abandoned.exists(),
"the dead reclaimer's entry was left behind"
);
assert_eq!(reclaim_entries_beside(&paths), Vec::<String>::new());
held.release();
fs::remove_dir_all(&root).ok();
}
#[test]
fn a_live_reclaimers_entry_names_it_as_the_holder() {
let root = scratch("reclaim-live");
let paths = RunPaths::under(&root, "demo");
paths.create().expect("the run directory");
let dead = a_dead_holders_lock(&paths);
let key = reclaim_key(&dead);
let taking_over = reclaim_entry(&paths.lock(), &key, 1);
write_json(
&taking_over,
&LockRecord {
pid: sys::pid(),
host: sys::hostname(),
acquired_at: sys::now_rfc3339(),
verb: "reply".to_string(),
started: String::new(),
},
)
.expect("an entry a live reclaimer holds");
match OwnershipLock::acquire(&paths, "adopt") {
Err(Error::Locked { run, pid, verb, .. }) => {
assert_eq!(run, "demo");
assert_eq!(pid, sys::pid());
assert_eq!(verb, "reply");
}
other => panic!("a run being taken over was not reported as held: {other:?}"),
}
let untouched: LockRecord = read_json(&paths.lock()).expect("the lock reads back");
assert_eq!(untouched, dead, "the loser wrote the lock");
assert!(
taking_over.exists(),
"the loser took away an entry it did not create"
);
fs::remove_dir_all(&root).ok();
}
#[test]
fn an_unreadable_reclaim_entry_is_still_a_claim() {
let root = scratch("reclaim-unreadable");
let paths = RunPaths::under(&root, "demo");
paths.create().expect("the run directory");
let dead = a_dead_holders_lock(&paths);
let entry = reclaim_entry(&paths.lock(), &reclaim_key(&dead), 1);
fs::write(&entry, "not json at all").expect("a corrupt entry");
match OwnershipLock::acquire(&paths, "adopt") {
Err(unreadable) if is_unreadable_lock(&unreadable) => {
assert!(
matches!(&unreadable, Error::Ledger { path, .. } if *path == entry),
"the refusal did not name the entry nobody can read: {unreadable}"
);
}
other => panic!("an unreadable entry was not reported as one: {other:?}"),
}
let untouched: LockRecord = read_json(&paths.lock()).expect("the lock reads back");
assert_eq!(untouched, dead);
fs::remove_dir_all(&root).ok();
}
#[test]
fn an_unreadable_lock_is_still_a_claim() {
let root = scratch("unreadable");
let paths = RunPaths::under(&root, "demo");
paths.create().expect("the run directory");
fs::write(paths.lock(), "not json at all").expect("a corrupt lock");
match OwnershipLock::acquire(&paths, "start") {
Err(unreadable) if is_unreadable_lock(&unreadable) => {
let said = unreadable.to_string();
assert!(said.contains("an unreadable lock"), "{said}");
assert!(said.contains("'demo'"), "{said}");
assert!(
!said.contains("pid 0"),
"a holder nobody wrote was named: {said}"
);
assert!(
matches!(&unreadable, Error::Ledger { path, .. } if *path == paths.lock()),
"the refusal did not name the lock nobody can read: {said}"
);
}
other => panic!("an unreadable lock was not reported as one: {other:?}"),
}
assert_eq!(
fs::read_to_string(paths.lock()).expect("the lock is still there"),
"not json at all",
"a claim nobody can read was overwritten"
);
fs::remove_dir_all(&root).ok();
}
#[cfg(unix)]
#[test]
fn a_lock_the_filesystem_refuses_for_a_moment_names_its_holder_once_it_opens() {
use std::os::unix::fs::PermissionsExt;
let root = scratch("lock-refused-briefly");
let paths = RunPaths::under(&root, "demo");
paths.create().expect("the run directory");
let holder = LockRecord {
pid: sys::pid(),
host: sys::hostname(),
acquired_at: sys::now_rfc3339(),
verb: "drive".to_string(),
started: String::new(),
};
write_json(&paths.lock(), &holder).expect("a live holder's lock");
fs::set_permissions(paths.lock(), fs::Permissions::from_mode(0o000))
.expect("the lock's mode is taken away");
assert!(
fs::read(paths.lock()).is_err(),
"this process reads a file whose mode refuses it, so the refusal this is about \
cannot be arranged here"
);
let refused = std::thread::scope(|scope| {
scope.spawn(|| {
std::thread::sleep(std::time::Duration::from_millis(100));
fs::set_permissions(paths.lock(), fs::Permissions::from_mode(0o644))
.expect("the lock's mode is given back");
});
OwnershipLock::acquire(&paths, "reply")
});
match refused {
Err(Error::Locked {
pid, host, verb, ..
}) => {
assert_eq!(pid, holder.pid);
assert_eq!(host, holder.host);
assert_eq!(verb, "drive");
}
other => {
panic!("a lock refused for a moment was not reported by its holder: {other:?}")
}
}
fs::remove_dir_all(&root).ok();
}
#[test]
fn a_lock_the_filesystem_goes_on_refusing_is_reported_as_the_refusal() {
let root = scratch("lock-refused");
let paths = RunPaths::under(&root, "demo");
paths.create().expect("the run directory");
fs::create_dir(paths.lock()).expect("something that is not a file holds the name");
match OwnershipLock::acquire(&paths, "start") {
Err(refused @ Error::Ledger { .. }) if !is_unreadable_lock(&refused) => {
assert!(
matches!(&refused, Error::Ledger { path, .. } if *path == paths.lock()),
"the refusal did not name the lock: {refused}"
);
}
other => panic!("a lock the filesystem refuses was not reported as that: {other:?}"),
}
assert!(paths.lock().is_dir(), "what held the name was replaced");
fs::remove_dir_all(&root).ok();
}
#[test]
fn an_unknown_launch_is_nobodys_run() {
let record = LaunchRecord {
run_id: "demo".into(),
project: "plans:demo".into(),
dir: PathBuf::from("/tmp/launch"),
graph: "graphs/dag-scope.yaml".into(),
graph_run: String::new(),
observer_runs: Vec::new(),
observer_ending: String::new(),
node_graph: String::new(),
pr_author_graph: String::new(),
node_validator: String::new(),
envelope_reviewer: String::new(),
launcher: sys::UNKNOWN_LAUNCHER.into(),
session: sys::UNKNOWN_LAUNCHER.into(),
pid: 1,
host: "h".into(),
started: String::new(),
started_at: sys::now_rfc3339(),
heartbeat_interval: 1,
writeback_item_budget: 0,
success_hook: String::new(),
failure_hook: String::new(),
hook_timeout: 0,
dispatch_env_hook: String::new(),
dispatch_env_hook_timeout: 0,
dag_sets: Vec::new(),
node_sets: Vec::new(),
adoptions: 0,
filters: Filters::default(),
bus_config: Default::default(),
maintenance_config: None,
oneharness_sessions: None,
envelope_reviewer_bar: Default::default(),
};
assert!(!record.owned_by(sys::UNKNOWN_LAUNCHER));
assert_eq!(record.owner_label("anyone"), "[unknown]");
}
#[test]
fn a_foreign_owner_is_labelled_without_naming_the_session() {
let record = LaunchRecord {
run_id: "demo".into(),
project: "plans:demo".into(),
dir: PathBuf::from("/tmp/launch"),
graph: "graphs/dag-scope.yaml".into(),
graph_run: String::new(),
observer_runs: Vec::new(),
observer_ending: String::new(),
node_graph: String::new(),
pr_author_graph: String::new(),
node_validator: String::new(),
envelope_reviewer: String::new(),
launcher: "claude-code".into(),
session: "secret-session-id".into(),
pid: 1,
host: "h".into(),
started: String::new(),
started_at: sys::now_rfc3339(),
heartbeat_interval: 1,
writeback_item_budget: 0,
success_hook: String::new(),
failure_hook: String::new(),
hook_timeout: 0,
dispatch_env_hook: String::new(),
dispatch_env_hook_timeout: 0,
dag_sets: Vec::new(),
node_sets: Vec::new(),
adoptions: 0,
filters: Filters::default(),
bus_config: Default::default(),
maintenance_config: None,
oneharness_sessions: None,
envelope_reviewer_bar: Default::default(),
};
let label = record.owner_label("mine");
assert!(!label.contains("secret-session-id"), "{label} leaks the id");
assert!(label.starts_with("[claude-code:"));
assert_eq!(record.owner_label("secret-session-id"), "[mine]");
assert!(record.owned_by("secret-session-id"));
}
#[test]
fn an_atomic_write_leaves_no_temporary_behind() {
let root = scratch("atomic");
let target = root.join("nested").join("record.json");
write_json(&target, &serde_json::json!({"ok": true})).expect("written");
let value: serde_json::Value = read_json(&target).expect("read back");
assert_eq!(value["ok"], serde_json::json!(true));
let leftovers: Vec<_> = fs::read_dir(root.join("nested"))
.expect("the directory")
.flatten()
.filter(|e| e.file_name().to_string_lossy().contains("tmp"))
.collect();
assert!(leftovers.is_empty(), "a temporary survived the rename");
fs::remove_dir_all(&root).ok();
}
#[test]
fn appended_lines_read_back_in_order_and_skip_blanks() {
let root = scratch("append-order");
let path = root.join("queue.jsonl");
assert!(read_lines(&path).is_empty());
append_line(&path, "first").expect("appended");
append_line(&path, "").expect("appended");
append_line(&path, "second").expect("appended");
assert_eq!(read_lines(&path), vec!["first", "second"]);
fs::remove_dir_all(&root).ok();
}
#[test]
fn a_record_carries_where_it_is_and_whether_its_writer_finished_it() {
let root = scratch("records");
let path = root.join("events.jsonl");
fs::write(&path, "first\n\nsecond\nhalf").expect("the file is written");
let records = read_records(&path);
assert_eq!(records.len(), 4);
assert_eq!(records[0].offset, 0);
assert_eq!(records[2].text, "second");
assert_eq!(records[2].offset, 7);
assert!(records[2].terminated);
assert_eq!(records[3].text, "half");
assert_eq!(records[3].offset, 14);
assert!(
!records[3].terminated,
"a fragment read as a record its writer had finished"
);
assert_eq!(read_lines(&path), vec!["first", "second", "half"]);
fs::remove_dir_all(&root).ok();
}
#[test]
fn a_file_torn_mid_character_still_reads_back_the_records_before_the_tear() {
let root = scratch("lossy");
let path = root.join("queue.jsonl");
fs::create_dir_all(&root).ok();
let mut bytes = b"{\"first\":1}\n{\"second\":2}\n{\"third\":".to_vec();
bytes.push(0xE2);
fs::write(&path, &bytes).expect("the file is written");
let records = read_records(&path);
assert_eq!(
records.len(),
3,
"the tear took the file with it: {records:?}"
);
assert_eq!(records[0].text, "{\"first\":1}");
assert_eq!(records[1].text, "{\"second\":2}");
assert_eq!(records[2].offset, 25);
assert_eq!(
records[2].bytes, 10,
"the loss was measured in the reading rather than in the file"
);
assert!(!records[2].terminated);
assert_eq!(read_lines(&path).len(), 3);
fs::remove_dir_all(&root).ok();
}
#[test]
fn an_append_heals_a_fragment_and_records_what_it_discarded() {
let root = scratch("heal");
let path = root.join("events.jsonl");
append_line(&path, "first").expect("appended");
let whole = fs::read_to_string(&path).expect("the file reads");
fs::OpenOptions::new()
.append(true)
.open(&path)
.and_then(|mut file| std::io::Write::write_all(&mut file, b"{\"half\":"))
.expect("the fragment is written");
append_line(&path, "second").expect("appended");
assert_eq!(
fs::read_to_string(&path).expect("the file reads"),
format!("{whole}second\n"),
"the heal cut into a whole record, or left the fragment in"
);
let recorded = torn_tails(&path);
assert_eq!(recorded.len(), 1, "{recorded:?}");
assert_eq!(recorded[0].offset, whole.len() as u64);
assert_eq!(recorded[0].bytes, 8);
assert_eq!(recorded[0].healed_by, sys::pid());
append_line(&path, "third").expect("appended");
assert_eq!(torn_tails(&path).len(), 1);
fs::remove_dir_all(&root).ok();
}
#[test]
fn a_store_that_is_nothing_but_a_fragment_heals_to_empty_and_says_so() {
let root = scratch("heal-whole");
let path = root.join("events.jsonl");
fs::write(&path, "x".repeat(70 * 1024)).expect("the fragment is written");
append_line(&path, "first").expect("appended");
assert_eq!(
fs::read_to_string(&path).expect("the file reads"),
"first\n"
);
let recorded = torn_tails(&path);
assert_eq!(recorded.len(), 1, "{recorded:?}");
assert_eq!(recorded[0].offset, 0);
assert_eq!(recorded[0].bytes, 70 * 1024);
fs::remove_dir_all(&root).ok();
}
#[test]
fn a_store_that_lost_nothing_records_nothing() {
let root = scratch("torn-absent");
let path = root.join("events.jsonl");
assert_eq!(
torn_tail_log(&path),
root.join("events.jsonl.torn"),
"the loss log is not beside the store it is about"
);
assert!(torn_tails(&path).is_empty());
append_line(&path, "first").expect("appended");
assert!(
torn_tails(&path).is_empty(),
"an append that healed nothing reported a loss"
);
fs::remove_dir_all(&root).ok();
}
#[test]
fn only_directories_with_a_launch_record_are_runs_and_the_rest_are_named() {
let root = scratch("index");
for name in ["b-run", "a-run"] {
let paths = RunPaths::under(&root, name);
paths.create().expect("a run directory");
write_json(&paths.launch(), &serde_json::json!({})).expect("a launch record");
}
fs::create_dir_all(root.join("scratch")).expect("a directory that records no run");
fs::write(root.join("notes.txt"), "not a run root").expect("a file beside the runs");
fs::create_dir_all(RunPaths::under(&root, "impostor").launch())
.expect("a launch record that is a directory");
let index = all_runs(&root);
let ids: Vec<String> = index.runs.iter().map(|r| r.run.clone()).collect();
assert_eq!(ids, vec!["a-run".to_string(), "b-run".to_string()]);
let refused: Vec<(PathBuf, String)> = index
.skipped
.iter()
.map(|root| (root.path.clone(), root.reason.clone()))
.collect();
assert_eq!(refused.len(), 2, "{refused:?}");
assert_eq!(refused[0].0, root.join("impostor"));
assert!(refused[0].1.contains("is not a file"), "{refused:?}");
assert_eq!(refused[1].0, root.join("scratch"));
assert!(refused[1].1.contains("no launch.json"), "{refused:?}");
assert_eq!(all_runs(&root.join("missing")), RunIndex::default());
fs::remove_dir_all(&root).ok();
}
#[test]
fn a_lock_records_the_holders_start_token_beside_its_pid() {
let root = scratch("stamp");
let paths = RunPaths::under(&root, "demo");
paths.create().expect("the run directory");
let held = OwnershipLock::acquire(&paths, "drive").expect("the lock is taken");
let record: LockRecord = read_json(&paths.lock()).expect("the lock reads back");
assert_eq!(record.pid, sys::pid());
assert!(
sys::process_start_token(sys::pid())
.expect("this host says when a process started")
.matches(&record.started),
"the lock's stamp is not this process's own start"
);
assert!(!record.started.is_empty());
held.release();
fs::remove_dir_all(&root).ok();
}
#[test]
fn a_dispatch_claims_the_process_it_runs_in_and_gives_it_up_when_it_ends() {
let root = scratch("dispatches");
let paths = RunPaths::under(&root, "demo");
paths.create().expect("the run directory");
assert!(
dispatches_of(&paths)
.expect("a run that has dispatched nothing has an empty registry")
.is_empty(),
"a run that has dispatched nothing claimed a process"
);
let claim = claim_dispatch(&paths, "build", sys::pid()).expect("the dispatch is recorded");
let recorded = dispatches_of(&paths).expect("the registry reads");
assert_eq!(recorded.len(), 1, "{recorded:?}");
assert_eq!(recorded[0].node, "build");
assert_eq!(recorded[0].pid, sys::pid());
assert_eq!(recorded[0].host, sys::hostname());
assert!(
sys::process_start_token(sys::pid())
.expect("this host says when a process started")
.matches(&recorded[0].started),
"the entry's stamp is not this process's own start: {recorded:?}"
);
drop(claim);
assert!(
dispatches_of(&paths)
.expect("the registry reads")
.is_empty(),
"a dispatch that ended left the run claiming its process"
);
fs::remove_dir_all(&root).ok();
}
#[test]
fn two_dispatches_in_one_process_are_two_entries_and_each_ends_alone() {
let root = scratch("dispatches-shared-process");
let paths = RunPaths::under(&root, "demo");
paths.create().expect("the run directory");
let first = claim_dispatch(&paths, "first", sys::pid()).expect("the first is recorded");
let second = claim_dispatch(&paths, "second", sys::pid()).expect("the second is recorded");
let nodes = |paths: &RunPaths| {
let mut named: Vec<String> = dispatches_of(paths)
.expect("the registry reads")
.into_iter()
.map(|held| held.node)
.collect();
named.sort();
named
};
assert_eq!(
nodes(&paths),
vec!["first".to_string(), "second".to_string()],
"two dispatches in one process did not record two entries"
);
drop(first);
assert_eq!(
nodes(&paths),
vec!["second".to_string()],
"a dispatch that ended took a live one's registration with it"
);
drop(second);
assert!(dispatches_of(&paths)
.expect("the registry reads")
.is_empty());
fs::remove_dir_all(&root).ok();
}
#[test]
fn a_claim_that_ends_leaves_a_later_dispatchs_entry_alone() {
let root = scratch("dispatches-reused");
let paths = RunPaths::under(&root, "demo");
paths.create().expect("the run directory");
let first = claim_dispatch(&paths, "build", sys::pid()).expect("the dispatch is recorded");
write_json(
&paths.dispatch(sys::pid(), 0),
&DispatchRecord {
started: "the process that took it, which is not the first one".into(),
..dispatches_of(&paths).expect("the registry reads")[0].clone()
},
)
.expect("the entry is rewritten");
drop(first);
let recorded = dispatches_of(&paths).expect("the registry reads");
assert_eq!(
recorded.len(),
1,
"a dispatch that ended removed an entry it did not write: {recorded:?}"
);
fs::remove_dir_all(&root).ok();
}
#[test]
fn the_registry_reads_in_pid_order() {
let root = scratch("dispatches-order");
let paths = RunPaths::under(&root, "demo");
paths.create().expect("the run directory");
for (node, pid) in [("later", 900_u32), ("earlier", 90), ("middle", 300)] {
write_json(
&paths.dispatch(pid, 0),
&DispatchRecord {
node: node.to_string(),
pid,
host: sys::hostname(),
dispatched_at: sys::now_rfc3339(),
started: "a start this host once reported".into(),
},
)
.expect("an entry");
}
let read: Vec<u32> = dispatches_of(&paths)
.expect("the registry reads")
.iter()
.map(|held| held.pid)
.collect();
assert_eq!(read, vec![90, 300, 900]);
fs::remove_dir_all(&root).ok();
}
#[test]
fn a_registry_this_build_cannot_read_is_reported_and_never_read_as_an_empty_one() {
let root = scratch("dispatches-unreadable");
let paths = RunPaths::under(&root, "demo");
paths.create().expect("the run directory");
let usable = DispatchRecord {
node: "build".into(),
pid: 4_242,
host: sys::hostname(),
dispatched_at: sys::now_rfc3339(),
started: "a start this host once reported".into(),
};
fs::write(
paths.dispatch(usable.pid, 0),
serde_json::to_string(&serde_json::json!({
"node": usable.node,
"pid": usable.pid,
"host": usable.host,
"dispatched_at": usable.dispatched_at,
"started": usable.started,
"reaped_by": "a build that came later",
}))
.expect("an entry from a newer writer"),
)
.expect("an entry");
assert_eq!(
dispatches_of(&paths).expect("an entry from a newer writer reads"),
vec![usable.clone()],
"an entry carrying a field this build does not know took the whole registry with it"
);
for (what, entry) in [
(
"a record that is not JSON at all",
"not an entry".to_string(),
),
(
"a record missing the stamp entirely",
serde_json::to_string(&serde_json::json!({
"node": usable.node,
"pid": usable.pid,
"host": usable.host,
"dispatched_at": usable.dispatched_at,
}))
.expect("an entry from a writer that recorded no stamp"),
),
(
"a record whose stamp proves nothing",
serde_json::to_string(&DispatchRecord {
started: String::new(),
..usable.clone()
})
.expect("an unstamped entry"),
),
] {
fs::write(paths.dispatch(usable.pid, 0), entry).expect("an entry");
let refused =
dispatches_of(&paths).expect_err(&format!("{what} was read as a registry"));
assert!(
refused.to_string().contains(&usable.pid.to_string()),
"the refusal over {what} does not name what caused it: {refused}"
);
}
fs::remove_dir_all(paths.dispatches()).expect("the registry is taken away");
let refused = dispatches_of(&paths)
.expect_err("a registry that is not there was read as a run with nothing running");
assert!(
refused
.to_string()
.contains(&paths.dispatches().display().to_string()),
"the refusal does not name the registry it could not read: {refused}"
);
fs::remove_dir_all(&root).ok();
}
#[test]
fn a_dispatch_the_registry_cannot_record_is_refused() {
let root = scratch("dispatches-unwritable");
let paths = RunPaths::under(&root, "demo");
paths.create().expect("the run directory");
let reaped = sys::reaped_pid();
let refused = claim_dispatch(&paths, "build", reaped)
.expect_err("a dispatch nothing can stamp was recorded anyway");
assert!(
refused.to_string().contains(&reaped.to_string()),
"the refusal does not name the process it could not stamp: {refused}"
);
fs::remove_dir_all(paths.dispatches()).expect("the registry is taken away");
fs::write(paths.dispatches(), "not a directory").expect("something in the way");
let refused = claim_dispatch(&paths, "build", sys::pid())
.expect_err("a claim that could not be written was reported as held");
assert!(
refused
.to_string()
.contains(&paths.dispatches().display().to_string()),
"the refusal does not name what it could not write: {refused}"
);
fs::remove_dir_all(&root).ok();
}
#[test]
fn a_lock_without_a_start_token_reads_and_is_written_back_without_one() {
let record: LockRecord = serde_json::from_str(
r#"{"pid":1,"host":"h","acquired_at":"2026-01-01T00:00:00.000Z","verb":"drive"}"#,
)
.expect("a lock from a build that predates the stamp");
assert!(record.started.is_empty());
let written = serde_json::to_string(&record).expect("it serializes");
assert!(!written.contains("started"), "{written}");
}
const OLDER_RECORD: &str =
include_str!("../tests/recorded/launch/otg-closed-state-writes-status.json");
fn older_record() -> (LaunchRecord, serde_json::Value) {
let recorded: serde_json::Value =
serde_json::from_str(OLDER_RECORD).expect("the older record is JSON");
let codec = &recorded["bus_config"]["codecs"]["onejudge"];
assert!(codec.get("select").is_none() && codec.get("frames").is_none());
let record: LaunchRecord =
serde_json::from_str(OLDER_RECORD).expect("the older record is read");
(record, recorded)
}
#[test]
fn an_older_records_bus_config_reads_with_its_missing_codec_fields_empty() {
let (record, recorded) = older_record();
let bus = record
.bus_config
.as_ref()
.expect("the record names a bus config");
let codec = &bus.config().codecs[&"onejudge".parse().expect("a codec name")];
assert_eq!(codec.select, "");
assert!(codec.frames.is_empty());
assert_eq!(codec.reply_window_seconds.map(NonZeroU64::get), Some(3000));
assert_eq!(bus.config().validators.len(), 1);
assert_eq!(
serde_json::to_value(&record).expect("the record serializes")["bus_config"],
recorded["bus_config"],
"the bus configuration was not written back as recorded"
);
let launched = RecordedBusConfig::from(bus.config().clone());
assert_eq!(
serde_json::to_value(&launched).expect("it serializes"),
serde_json::to_value(bus.config()).expect("it serializes")
);
}
#[test]
fn a_bus_config_that_is_not_one_is_still_refused() {
let mut recorded: serde_json::Value =
serde_json::from_str(OLDER_RECORD).expect("the older record is JSON");
recorded["bus_config"]["codecs"]["onejudge"]["select"] = serde_json::json!(7);
let refused = serde_json::from_value::<LaunchRecord>(recorded)
.expect_err("a codec whose select is not a string");
assert!(refused.to_string().contains("invalid type"), "{refused}");
}
#[test]
fn a_serve_resolving_an_older_records_codec_is_refused_naming_the_field() {
let (record, _) = older_record();
let config = record.bus_config.expect("a bus config").config().clone();
let name: onemessagebus::CodecName = "onejudge".parse().expect("a codec name");
let codec = config.codecs[&name].clone();
let refused = onemessagebus::ConfiguredCodec::new(name.clone(), codec.clone())
.expect_err("a codec the record did not fully describe");
assert_eq!(
refused,
"codecs.onejudge.select is not a dot-separated object path"
);
let selected = onemessagebus::CodecConfig {
select: "op".to_owned(),
..codec.clone()
};
let refused = onemessagebus::ConfiguredCodec::new(name.clone(), selected.clone())
.expect_err("a codec naming no frames");
assert_eq!(refused, "codecs.onejudge.frames is empty");
let described = onemessagebus::CodecConfig {
frames: [(
"judge".to_owned(),
onemessagebus::FrameConfig {
schema: crate::channel::layout::SURFACE_SCHEMA,
bindings: vec![onemessagebus::Binding {
when: None,
action: onemessagebus::BindingAction::Refuse {
message: "not here".to_owned(),
},
}],
},
)]
.into(),
..selected
};
let mut served = onemessagebus::ConfiguredCodec::new(name, described)
.expect("the codec, fully described, resolves");
let dir = scratch("older-codec-serve");
let bus = onemessagebus::Config::local(&dir, Some(crate::channel::layout::PLANNER_CHANNEL))
.resolve(
&onemessagebus::Layouts::new()
.with(std::sync::Arc::new(crate::channel::layout::PlannerChannel)),
&onemessagebus::TransportKinds::builtin(),
)
.expect("the channel resolves");
let mut written = Vec::new();
bus.serve(
&crate::channel::layout::SURFACES
.parse()
.expect("a queue name"),
&mut served,
&onemessagebus::ServeOptions::default(),
Box::new(std::io::Cursor::new(Vec::new())),
&mut written,
)
.expect("the resolved codec is served");
let _ = fs::remove_dir_all(&dir);
}
}