kranz_engine/event_log.rs
1//! Append-only JSONL event log (plan §4.3) — the single source of truth.
2//!
3//! One engine process owns `events.jsonl` at a time, guarded by a lock file
4//! (`events.jsonl.lock`) rather than POSIX advisory locks so Windows stays
5//! first-class. Lifecycle events are flushed + fsynced per append; stream
6//! deltas (`worker.message`) are buffered in memory and drained by age
7//! (throttle) — checked on the next append, or on demand via
8//! [`EventLog::flush_if_due`] — by the next lifecycle append, by explicit
9//! [`EventLog::flush`], or on drop. Losing buffered deltas on a crash is
10//! recoverable; losing a lifecycle event is not, hence the asymmetry.
11//!
12//! # Line integrity (audit 2026-09-01 H6)
13//!
14//! The log is not only an audit record: three gate decisions read it back
15//! mid-run, so a well-formed forged append or a rollback by truncation is a
16//! live consent bypass, not a post-hoc bookkeeping problem. Every line the
17//! writer produces therefore carries three extra fields, kept to one character
18//! because they ride every event:
19//!
20//! - `v` — canonicalization version, currently 2. Absent means the legacy
21//! float parser. Version 2 preserves floating-point bits and prefixes the
22//! hash input with `kranz.event-log.v2\n`, so stripping `v` cannot downgrade it.
23//! - `h` — the CHAIN. Hex sha256 over the version prefix, the previous line's
24//! `h` (empty for the first chained line), and this line's
25//! canonical event bytes, which are the event re-serialized WITHOUT `v`, `h`,
26//! and `m`. Legacy seals have no prefix and retain their original numeric
27//! interpretation. Version 2 sorts all object keys, independently of Cargo's
28//! `serde_json/preserve_order` feature. Both sides start from an [`Event`].
29//! The chain makes an edit anywhere in the file loud instead of local.
30//! - `m` — the MAC. Hex HMAC-SHA256 of `h` under the repository authority key
31//! ([`crate::paths::authority_key_path`]).
32//!
33//! Be clear about which does what. The chain ALONE only catches accidental
34//! corruption: an attacker who rewrites a line can recompute every following
35//! `h` themselves. `m` is what defeats a same-uid forger, because the key
36//! lives outside the repository, in the sandbox's authority-read-deny set and
37//! behind the agent CLI's `Read(~/.kranz/**)` deny rule, so they cannot
38//! compute it.
39//!
40//! Compatibility, and why it is not a hole. A log written before this existed
41//! has no `h` on any line, and refusing it would strand every in-flight
42//! mission, so an unsealed line is not refused on its own. Two rules stop
43//! that from becoming a free downgrade:
44//!
45//! 1. No downgrade WITHIN a log. Once a line carries `h`, every later line
46//! must; once a line carries `m`, every later line must. An attacker
47//! cannot strip integrity off just the tail they want to rewrite.
48//! 2. A SEAL FLOOR outside the repository. The first time a writer with the
49//! key opens a mission's log it records, under `~/.kranz/seals/`, the seq
50//! it will start sealing from ([`crate::paths::record_seal_floor`]). Every
51//! line at or above that seq must carry a valid `h` and `m`. That is what
52//! stops the whole-log rewrite: stripping every line makes the file look
53//! legacy, but the floor is not in the file and the attacker cannot lower
54//! it. It is also why the `m` rule is not simply "the key exists, so every
55//! line needs `m`" — the key can be minted mid-mission, and the lines
56//! written before that legitimately have none.
57
58use crate::error::{EngineError, Result};
59use crate::events::{Event, EventKind};
60use crate::paths::MissionPaths;
61use crate::scrub::SecretFinding;
62use cap_std::fs::Dir;
63use chrono::Utc;
64#[cfg(unix)]
65use std::ffi::OsString;
66use std::fs::File;
67use std::io::{ErrorKind, Write};
68use std::path::{Path, PathBuf};
69use std::time::{Duration, Instant};
70use subtle::ConstantTimeEq as _;
71
72/// JSON key holding a line's chain hash.
73const CHAIN_FIELD: &str = "h";
74
75/// JSON key holding a line's MAC over the chain hash.
76const MAC_FIELD: &str = "m";
77const VERSION_FIELD: &str = "v";
78const EXACT_FLOAT_VERSION: u64 = 2;
79
80/// Canonical bytes for one event, excluding the `v`/`h`/`m` envelope.
81fn canonical_event_bytes(event: &Event, version: u64) -> Result<String> {
82 if version == EXACT_FLOAT_VERSION {
83 let mut value = serde_json::to_value(event)?;
84 value.sort_all_objects();
85 Ok(serde_json::to_string(&value)?)
86 } else {
87 Ok(serde_json::to_string(event)?)
88 }
89}
90
91/// Next link in the chain: sha256 over the previous link and this event's
92/// canonical bytes.
93fn chain_hash(prev: &str, body: &str, version: u64) -> String {
94 let prefix = if version == EXACT_FLOAT_VERSION {
95 "kranz.event-log.v2\n"
96 } else {
97 ""
98 };
99 let mut input = String::with_capacity(prefix.len() + prev.len() + body.len());
100 input.push_str(prefix);
101 input.push_str(prev);
102 input.push_str(body);
103 crate::standards_waiver::sha256_hex(input.as_bytes())
104}
105
106/// Serialize one event as a sealed log line (no trailing newline), returning
107/// the line and the chain hash the NEXT line must build on.
108fn seal_line(event: &Event, prev_hash: &str, key: Option<&[u8]>) -> Result<(String, String)> {
109 let body = canonical_event_bytes(event, EXACT_FLOAT_VERSION)?;
110 let hash = chain_hash(prev_hash, &body, EXACT_FLOAT_VERSION);
111 let mut value = serde_json::to_value(event)?;
112 let object = value.as_object_mut().ok_or_else(|| {
113 EngineError::InvalidState("event did not serialize as a JSON object".to_string())
114 })?;
115 object.insert(VERSION_FIELD.to_string(), EXACT_FLOAT_VERSION.into());
116 object.insert(
117 CHAIN_FIELD.to_string(),
118 serde_json::Value::String(hash.clone()),
119 );
120 if let Some(key) = key {
121 object.insert(
122 MAC_FIELD.to_string(),
123 serde_json::Value::String(crate::hooks::hmac_sha256_hex(key, hash.as_bytes())),
124 );
125 }
126 Ok((serde_json::to_string(&value)?, hash))
127}
128
129// The old writer parsed its serialized body once before writing, and readers
130// parsed it again. Some valid seals depend on that parser's rounded arithmetic.
131// Reproduce it only for legacy lines, using the canonical float token emitted
132// by serde_json (at most 17 significant digits). Integers and strings stay intact.
133fn restore_legacy_float_parsing(value: &mut serde_json::Value) -> Result<()> {
134 match value {
135 serde_json::Value::Number(number) if number.is_f64() => {
136 *number = legacy_float_number(number).ok_or_else(|| {
137 EngineError::LogCorruption("legacy event has an out-of-range float".into())
138 })?;
139 }
140 serde_json::Value::Array(values) => {
141 for value in values {
142 restore_legacy_float_parsing(value)?;
143 }
144 }
145 serde_json::Value::Object(values) => {
146 for value in values.values_mut() {
147 restore_legacy_float_parsing(value)?;
148 }
149 }
150 _ => {}
151 }
152 Ok(())
153}
154
155fn legacy_float_number(number: &serde_json::Number) -> Option<serde_json::Number> {
156 let decimal = number.to_string();
157 let negative = decimal.starts_with('-');
158 let unsigned = decimal.strip_prefix('-').unwrap_or(&decimal);
159 let (mantissa, exponent) = unsigned.split_once('e').unwrap_or((unsigned, "0"));
160 let fraction_digits = mantissa
161 .split_once('.')
162 .map_or(0, |(_, fraction)| fraction.len());
163 let mut exponent = exponent.parse::<i32>().ok()? - i32::try_from(fraction_digits).ok()?;
164 let coefficient = mantissa.replace('.', "").parse::<u64>().ok()?;
165 let mut parsed = coefficient as f64;
166 if exponent < -308 {
167 parsed /= 1e308;
168 exponent += 308;
169 }
170 if exponent.unsigned_abs() > 308 {
171 return None;
172 }
173 // Decimal parsing gives the same rounded powers as the old parser's
174 // literal table. powi() can round differently and cannot substitute here.
175 let power = format!("1e{}", exponent.unsigned_abs())
176 .parse::<f64>()
177 .ok()?;
178 parsed = if exponent < 0 {
179 parsed / power
180 } else {
181 parsed * power
182 };
183 serde_json::Number::from_f64(if negative { -parsed } else { parsed })
184}
185
186/// Seal a whole event sequence into `events.jsonl` bytes exactly as the
187/// writer would. Public so tests and tooling can build a log that satisfies
188/// [`EventLog::read_events`] without driving a live [`EventLog`].
189pub fn seal_events(events: &[Event], key: Option<&[u8]>) -> Result<String> {
190 let mut out = String::new();
191 let mut prev = String::new();
192 for event in events {
193 let (line, hash) = seal_line(event, &prev, key)?;
194 out.push_str(&line);
195 out.push('\n');
196 prev = hash;
197 }
198 Ok(out)
199}
200
201/// Refuse to carry on when the log is SHORTER than the last snapshot says it
202/// was (audit 2026-09-01 H6, rollback by truncation).
203///
204/// `state.json` is a derived cache, but it holds an independent copy of the
205/// high-water mark (`MissionState::last_seq`), and nothing compared the two.
206/// A log cut at a line boundary stays internally valid, so the only signal
207/// that events were erased is that the snapshot remembers more of them.
208///
209/// A snapshot that is BEHIND the log is normal: the log is the source of
210/// truth and the snapshot is rewritten after the fold, so a crash between the
211/// two leaves it stale. Only the shorter-log direction is refused, and the
212/// error names both numbers so an operator can see the size of the gap.
213///
214/// A missing or unreadable snapshot is not an error: a mission resumed on a
215/// machine that never wrote one has nothing to compare against.
216pub fn check_no_rollback(paths: &MissionPaths, events: &[Event]) -> Result<()> {
217 // Two witnesses, and the higher one wins. The snapshot lives in the
218 // repository beside the log, so it only catches a crash or a careless
219 // edit; the high-water mark lives outside the repository beside the
220 // authority key, so it also catches a writer who trimmed both.
221 let snapshot_seq = crate::reducer::read_snapshot(&paths.state_file())
222 .map(|snapshot| snapshot.last_seq)
223 .unwrap_or(0);
224 let mark_seq = crate::paths::read_high_water(&paths.repo_root, &paths.mission_id).unwrap_or(0);
225 let (witness_seq, witness) = if mark_seq >= snapshot_seq {
226 (mark_seq, "the out-of-repo high-water mark")
227 } else {
228 (snapshot_seq, "the last snapshot")
229 };
230 let log_last_seq = events.last().map(|e| e.seq).unwrap_or(0);
231 if witness_seq > log_last_seq {
232 return Err(EngineError::LogCorruption(format!(
233 "refusing to resume mission '{}': {} ends at seq {log_last_seq} but {witness} \
234 recorded seq {witness_seq}. The log has lost {} event(s) since it was written; \
235 resuming would overwrite the snapshot with the rolled-back state and erase the \
236 evidence. Restore the log from the mission branch or abandon the mission.",
237 paths.mission_id,
238 paths.events_file().display(),
239 witness_seq - log_last_seq
240 )));
241 }
242 Ok(())
243}
244
245/// The repository root and mission id a log sits under, recovered from its
246/// path (`<repo>/.kranz/missions/<id>/events.jsonl`). Used to locate the
247/// authority key and the seal floor from the static reader entry points,
248/// which take only a path.
249fn log_identity(path: &Path) -> Option<(&Path, &str)> {
250 let mission_dir = path.parent()?;
251 let mission_id = mission_dir.file_name()?.to_str()?;
252 let missions_dir = mission_dir.parent()?;
253 if missions_dir.file_name()? != "missions" {
254 return None;
255 }
256 let kranz_dir = missions_dir.parent()?;
257 if kranz_dir.file_name()? != ".kranz" {
258 return None;
259 }
260 Some((kranz_dir.parent()?, mission_id))
261}
262
263/// How aggressively [`EventLog::acquire`] may steal an existing lock.
264///
265/// A holder that is provably DEAD is always stolen (a stale lock from a
266/// crashed engine), regardless of tier. The tiers only govern holders that
267/// are alive or of indeterminate liveness. Automatic Dead detection is
268/// unix-only (`kill(pid, 0)` returning `ESRCH`, plus the own-pid
269/// token-reuse screen on any platform): on non-unix targets only the
270/// own-pid token-reuse screen can ever yield Dead, so recovering the lock
271/// from a foreign crashed holder there always requires an explicit force
272/// tier (`--force-lock` / `--dangerously-steal-live-lock`).
273///
274/// | holder liveness | `No` | `IfNotLive` | `EvenIfLive` |
275/// |-----------------|------------|-------------|--------------|
276/// | Dead | steal | steal | steal |
277/// | Unknown | `LockHeld` | steal | steal |
278/// | Alive | `LockHeld` | `LockHeld` | steal (loud) |
279#[derive(Debug, Clone, Copy, PartialEq, Eq)]
280pub enum LockForce {
281 /// Honor any lock whose holder is not provably dead.
282 No,
283 /// `--force-lock`: steal unless the holder is provably ALIVE. This is the
284 /// historical force behavior on platforms/lockfiles where liveness cannot
285 /// be probed (Unknown), but it refuses to rip the lock from a running
286 /// engine.
287 IfNotLive,
288 /// `--dangerously-steal-live-lock`: steal even from a live holder. Only
289 /// correct when the operator has verified the holder is a zombie or an
290 /// unrelated (pid-reused) process — stealing from a live kranz engine
291 /// means two engines write one log.
292 EvenIfLive,
293}
294
295/// A serialized delta line waiting in the write buffer.
296#[derive(Debug)]
297struct BufferedLine {
298 buffered_at: Instant,
299 line: String,
300}
301
302/// Result of validating a log file, including how many leading bytes hold
303/// successfully parsed lines (so `acquire` can truncate a torn tail).
304#[derive(Debug)]
305struct ParsedLog {
306 events: Vec<Event>,
307 /// Byte length of the valid prefix: the end of the last successfully
308 /// parsed line, including its trailing newline when present.
309 valid_len: u64,
310 /// False only when the last parsed line lacked a trailing newline (a torn
311 /// write that cut exactly at the terminator).
312 terminated: bool,
313 /// Chain hash of the last parsed line, `None` when the log is empty or
314 /// its tail is still unchained (a legacy log). The next append builds on
315 /// this, so a legacy log starts a fresh chain from the empty string.
316 last_hash: Option<String>,
317 /// True once any parsed line carried a MAC, so the writer keeps MACing
318 /// even if the key becomes unreadable rather than silently downgrading a
319 /// log every reader would then refuse.
320 saw_mac: bool,
321}
322
323/// Single-writer, append-only handle on a mission's `events.jsonl`.
324///
325/// Constructed via [`EventLog::acquire`]; the lock file is released on drop.
326#[derive(Debug)]
327pub struct EventLog {
328 mission_id: String,
329 events_path: PathBuf,
330 lock_path: PathBuf,
331 /// Generation written into the lock file at acquire. Re-checked on every
332 /// append so a stolen-from process fails closed instead of dual-writing.
333 lock_generation: u64,
334 /// Identity token written into the lock file at acquire (when the
335 /// platform can produce one). Drop deletes the lock file only when the
336 /// on-disk generation (and token, when present) still match — so a
337 /// stolen-from teardown cannot wipe the stealer's lock.
338 lock_token: Option<String>,
339 /// Pinned mission directory capability retained from acquisition through
340 /// every lock read/removal. Absolute paths below are display-only.
341 mission_dir: Dir,
342 file: File,
343 /// Seq to assign to the next appended event.
344 next_seq: u64,
345 throttle: Duration,
346 buffer: Vec<BufferedLine>,
347 /// Repository root, for the out-of-repo high-water mark recorded after
348 /// every durable append (see [`crate::paths::record_high_water`]).
349 repo_root: PathBuf,
350 /// Chain hash of the last line written (or loaded at acquire); the empty
351 /// string for a fresh or still-unchained log.
352 prev_hash: String,
353 /// Repository authority key, when one is readable. `None` writes the
354 /// chain without a MAC, which is what a reader with no key can verify
355 /// anyway.
356 authority_key: Option<Vec<u8>>,
357}
358
359/// Write buffered lines to `sink` from the front, removing each line from
360/// `buffer` only after it is successfully written. On the first write error
361/// the failing line and everything after it stay in `buffer`, in original
362/// order, so a later retry can pick up exactly where this call left off —
363/// `Vec::drain` cannot do this since its guard discards not-yet-yielded
364/// items if the iteration is cut short by `?`.
365fn drain_lines<W: std::io::Write>(
366 sink: &mut W,
367 buffer: &mut Vec<BufferedLine>,
368) -> std::io::Result<()> {
369 while !buffer.is_empty() {
370 sink.write_all(buffer[0].line.as_bytes())?;
371 buffer.remove(0);
372 }
373 Ok(())
374}
375
376fn ensure_absent_or_regular_at(dir: &Dir, name: &str, display: &Path) -> Result<()> {
377 match dir.symlink_metadata(name) {
378 Ok(metadata) if metadata.file_type().is_file() => Ok(()),
379 Ok(_) => Err(EngineError::InvalidState(format!(
380 "refusing non-regular mission runtime path {}",
381 display.display()
382 ))),
383 Err(error) if error.kind() == ErrorKind::NotFound => Ok(()),
384 Err(error) => Err(error.into()),
385 }
386}
387
388fn open_create_new_at(dir: &Dir, name: &str) -> std::io::Result<File> {
389 use cap_fs_ext::OpenOptionsFollowExt as _;
390 use cap_primitives::fs::FollowSymlinks;
391 let mut options = cap_std::fs::OpenOptions::new();
392 options
393 .write(true)
394 .create_new(true)
395 .follow(FollowSymlinks::No);
396 dir.open_with(name, &options).map(|file| file.into_std())
397}
398
399fn open_write_at(dir: &Dir, name: &str, create: bool) -> std::io::Result<File> {
400 use cap_fs_ext::OpenOptionsFollowExt as _;
401 use cap_primitives::fs::FollowSymlinks;
402 let mut options = cap_std::fs::OpenOptions::new();
403 options
404 .write(true)
405 .create(create)
406 .follow(FollowSymlinks::No);
407 dir.open_with(name, &options).map(|file| file.into_std())
408}
409
410fn open_append_at(dir: &Dir, name: &str, create: bool) -> std::io::Result<File> {
411 use cap_fs_ext::OpenOptionsFollowExt as _;
412 use cap_primitives::fs::FollowSymlinks;
413 let mut options = cap_std::fs::OpenOptions::new();
414 options
415 .append(true)
416 .create(create)
417 .follow(FollowSymlinks::No);
418 dir.open_with(name, &options).map(|file| file.into_std())
419}
420
421fn open_read_at(dir: &Dir, name: &str) -> std::io::Result<File> {
422 use cap_fs_ext::OpenOptionsFollowExt as _;
423 use cap_primitives::fs::FollowSymlinks;
424 let mut options = cap_std::fs::OpenOptions::new();
425 options.read(true).follow(FollowSymlinks::No);
426 #[cfg(unix)]
427 {
428 use cap_fs_ext::OpenOptionsExt as _;
429 options.custom_flags(libc::O_NONBLOCK);
430 }
431 dir.open_with(name, &options).map(|file| file.into_std())
432}
433
434impl EventLog {
435 /// Acquire the single-writer lock for a mission and open its event log.
436 ///
437 /// Creates the mission directory tree (mission dir, `runs/`, `control/`)
438 /// if missing — no-follow: a symlinked `.kranz`/`missions`/mission dir or
439 /// runtime file is refused (P1 mission-path-no-follow), never followed
440 /// into another repository's tree. If the lock file already exists, the
441 /// holder's liveness decides against the [`LockForce`] tier (see its
442 /// matrix): a provably
443 /// dead holder is always stolen; a live or indeterminate one fails with
444 /// [`EngineError::LockHeld`] naming the holder's pid unless the tier
445 /// permits the steal. Existing events are loaded to resume the seq
446 /// counter and to verify `mission_id` matches the log. Any torn final
447 /// line left by a crash is repaired (truncated, or newline-terminated if
448 /// the line itself is intact) before the append handle opens, so new
449 /// events never glue onto a partial line.
450 ///
451 /// The lock file records up to three lines — `<pid>`, the acquire time as
452 /// unix epoch seconds (diagnostics only), and the holder's own process
453 /// identity token — so a later acquire can detect pid reuse: a holder
454 /// whose CURRENT token differs from the recorded one is not the process
455 /// that wrote the lock. Tokens are compared for raw equality (see
456 /// [`process_identity_token`]) — never via clock arithmetic, which
457 /// wall-clock steps would poison. The legacy one- and two-line formats
458 /// are still accepted; they just forgo reuse detection.
459 pub fn acquire(
460 paths: &MissionPaths,
461 mission_id: &str,
462 throttle: Duration,
463 force: LockForce,
464 ) -> Result<EventLog> {
465 // Resolve the mission tree no-follow (P1 mission-path-no-follow): a
466 // symlinked component or runtime file is refused before any create or
467 // open — an append through a symlink would write another repo's tree.
468 let mission_dir = paths.open_mission_dir_nofollow(true)?;
469 crate::paths::create_real_subdir(&mission_dir, "runs", &paths.runs_dir())?;
470 crate::paths::create_real_subdir(&mission_dir, "control", &paths.control_dir())?;
471 ensure_absent_or_regular_at(&mission_dir, "events.jsonl.lock", &paths.lock_file())?;
472 ensure_absent_or_regular_at(&mission_dir, "events.jsonl", &paths.events_file())?;
473
474 let lock_path = paths.lock_file();
475 let (mut lock_file, lock_generation) =
476 match open_create_new_at(&mission_dir, "events.jsonl.lock") {
477 Ok(f) => (f, 0u64),
478 Err(e) if e.kind() == ErrorKind::AlreadyExists => {
479 let (f, prev_gen) =
480 steal_lock(&mission_dir, "events.jsonl.lock", &lock_path, force)?;
481 (f, prev_gen.saturating_add(1))
482 }
483 Err(e) => return Err(e.into()),
484 };
485
486 // From here on we hold the lock; release it if the rest of the
487 // acquisition fails so a failed open doesn't strand the mission.
488 let mut open = || -> Result<EventLog> {
489 // Line 1: pid. Line 2: acquire time (diagnostics only — reuse
490 // detection is the token's job). Line 3: our own identity token,
491 // where this platform can produce one; a probe that finds it
492 // missing degrades to plain pid liveness, never to Dead.
493 // Line 4: generation — increments on every steal so a stolen-from
494 // process fails closed on its next append.
495 lock_file.write_all(
496 current_lock_holder_record_with_generation(lock_generation).as_bytes(),
497 )?;
498 lock_file.flush()?;
499
500 let events_path = paths.events_file();
501 // Chain state carried forward from whatever is already on disk.
502 let mut prev_hash = String::new();
503 let mut tail_had_mac = false;
504 let last_seq = if mission_dir
505 .symlink_metadata("events.jsonl")
506 .is_ok_and(|metadata| metadata.file_type().is_file())
507 {
508 let parsed = Self::parse_log_file(
509 open_read_at(&mission_dir, "events.jsonl")?,
510 &events_path,
511 )?;
512 if let Some(first) = parsed.events.first() {
513 if first.mission_id != mission_id {
514 return Err(EngineError::InvalidState(format!(
515 "event log {} belongs to mission '{}', not '{}'",
516 events_path.display(),
517 first.mission_id,
518 mission_id
519 )));
520 }
521 }
522 // Repair torn writes from a crashed predecessor BEFORE opening
523 // the append handle. A torn final line is tolerated on read,
524 // but if left in place the next append glues onto it; once a
525 // further event lands the spliced garbage is no longer final
526 // and every read fails with LogCorruption forever.
527 let file_len = mission_dir.metadata("events.jsonl")?.len();
528 if parsed.valid_len < file_len {
529 // Unparseable garbage past the last good line: cut it off.
530 let repair = open_write_at(&mission_dir, "events.jsonl", false)?;
531 repair.set_len(parsed.valid_len)?;
532 repair.sync_data()?;
533 } else if !parsed.terminated {
534 // The final line parsed but the tear ate its trailing
535 // newline; terminate it so the next append starts fresh.
536 let mut repair = open_append_at(&mission_dir, "events.jsonl", false)?;
537 repair.write_all(b"\n")?;
538 repair.sync_data()?;
539 }
540 prev_hash = parsed.last_hash.clone().unwrap_or_default();
541 tail_had_mac = parsed.saw_mac;
542 parsed.events.last().map(|e| e.seq).unwrap_or(0)
543 } else {
544 0
545 };
546
547 let file = open_append_at(&mission_dir, "events.jsonl", true)?;
548 // Acquiring the writer IS the operator action that starts or
549 // resumes a mission, so it mints the repository key on first
550 // use: a mission that never saw a control command must still
551 // be sealed, or the MAC and the high-water mark protect nothing
552 // until the first `kranz msg`. Minting failure (no resolvable
553 // home, an unwritable one) degrades to an unsealed log with a
554 // warning rather than refusing every mission on such a host;
555 // the readers treat an unsealed log exactly as before.
556 // Refuse, never degrade: a writer that carried on unsealed
557 // because the key file was unreadable would hand a same-uid
558 // attacker exactly the downgrade the seal exists to prevent
559 // (zero the key, delete the floor, rewrite the log keyless;
560 // follow-up review F-2). No key, no mission.
561 let authority_key = Some(
562 crate::paths::load_or_create_authority_key(&paths.repo_root).map_err(|error| {
563 EngineError::InvalidState(format!(
564 "cannot mint or read the repository authority key for mission '{mission_id}' \
565 ({}): {error}. The event log is not written unsealed; restore the key \
566 directory before running this mission",
567 crate::paths::authority_key_path(&paths.repo_root)
568 .map(|p| p.display().to_string())
569 .unwrap_or_else(|| "<global kranz dir>/keys/<repo>.key".to_string())
570 ))
571 })?,
572 );
573 if authority_key.is_none() && tail_had_mac {
574 // The tail is MACed and we cannot MAC any more: appending
575 // would write a downgrade every reader then refuses. Refuse
576 // now, naming the key, instead of corrupting the log.
577 return Err(EngineError::InvalidState(format!(
578 "event log {} is MAC-protected but the repository authority key is unreadable; \
579 restore {} before running this mission",
580 events_path.display(),
581 crate::paths::authority_key_path(&paths.repo_root)
582 .map(|p| p.display().to_string())
583 .unwrap_or_else(|| "~/.kranz/keys/<repo>.key".to_string())
584 )));
585 }
586 if authority_key.is_some() {
587 // Record, outside the repo and exactly once, the first seq
588 // this writer will seal. From here on an unsealed line at or
589 // above that seq is a forgery, not a legacy line, and the
590 // lines below it stay grandfathered.
591 crate::paths::record_seal_floor(
592 &paths.repo_root,
593 mission_id,
594 last_seq.saturating_add(1),
595 )?;
596 }
597 Ok(EventLog {
598 mission_id: mission_id.to_string(),
599 events_path,
600 lock_path: lock_path.clone(),
601 lock_generation,
602 lock_token: process_identity_token(std::process::id() as i32),
603 mission_dir: mission_dir.try_clone()?,
604 file,
605 next_seq: last_seq + 1,
606 throttle,
607 buffer: Vec::new(),
608 repo_root: paths.repo_root.clone(),
609 prev_hash,
610 authority_key,
611 })
612 };
613
614 match open() {
615 Ok(log) => Ok(log),
616 Err(e) => {
617 let _ = mission_dir.remove_file("events.jsonl.lock");
618 Err(e)
619 }
620 }
621 }
622
623 /// Mission id this log was acquired for.
624 pub fn mission_id(&self) -> &str {
625 &self.mission_id
626 }
627
628 /// Seq of the last appended (or loaded) event; 0 for a fresh log.
629 pub fn last_seq(&self) -> u64 {
630 self.next_seq - 1
631 }
632
633 /// Path of the underlying `events.jsonl`.
634 pub fn events_path(&self) -> &Path {
635 &self.events_path
636 }
637
638 /// Append one event: assigns the next seq and the current timestamp,
639 /// serializes to a single JSON line, and returns a clone of the stored
640 /// event so the caller can broadcast it. If this boundary redacts any
641 /// string payload, `secret.redacted` audit events are appended immediately
642 /// after the sanitized event.
643 ///
644 /// Durability: lifecycle events drain any buffered deltas first (file
645 /// order == append order), then write + flush + fsync. Stream deltas are
646 /// buffered and drained once the oldest buffered delta exceeds the
647 /// throttle age — checked here on each append, or on demand (without
648 /// waiting for another append) via [`EventLog::flush_if_due`].
649 pub fn append(&mut self, kind: EventKind) -> Result<Event> {
650 Ok(self.append_with_redaction_audits(kind)?.0)
651 }
652
653 /// Append one event and any required `secret.redacted` audit events.
654 /// Returns the sanitized primary event plus the audit events that followed
655 /// it, so callers that maintain snapshots can fold the same sequence.
656 pub fn append_with_redaction_audits(&mut self, kind: EventKind) -> Result<(Event, Vec<Event>)> {
657 let (event, redactions) = self.append_redacting(kind)?;
658 let mut audits = Vec::new();
659 for finding in redactions {
660 let (audit, _) = self.append_redacting(EventKind::SecretRedacted {
661 rule_id: finding.rule_id,
662 fingerprint: finding.fingerprint,
663 location: finding.location,
664 })?;
665 audits.push(audit);
666 }
667 Ok((event, audits))
668 }
669
670 /// Append one event after scanning/redacting string payloads. Returns the
671 /// sanitized event plus secret findings (fingerprints only, never values).
672 pub fn append_redacting(&mut self, kind: EventKind) -> Result<(Event, Vec<SecretFinding>)> {
673 // Fail closed if another process stole the lock out from under us —
674 // otherwise two engines dual-write one log (seq gaps / corruption).
675 let current_gen = read_lock_info_at(&self.mission_dir, "events.jsonl.lock")
676 .generation
677 .unwrap_or(0);
678 if current_gen != self.lock_generation {
679 return Err(EngineError::LockHeld(format!(
680 "event log lock for '{}' was stolen (generation {} → {}); refusing append",
681 self.mission_id, self.lock_generation, current_gen
682 )));
683 }
684
685 let event = Event {
686 seq: self.next_seq,
687 ts: Utc::now(),
688 mission_id: self.mission_id.clone(),
689 kind,
690 };
691 let mut value = serde_json::to_value(&event)?;
692 let findings = crate::scrub::scrub_json_value(&mut value, "event");
693 let event: Event = serde_json::from_value(value)?;
694 // Seal AFTER scrubbing, so the chain covers the bytes that actually
695 // land on disk rather than the pre-redaction event.
696 let (mut line, hash) = seal_line(&event, &self.prev_hash, self.authority_key.as_deref())?;
697 self.prev_hash = hash;
698 line.push('\n');
699
700 if event.kind.is_stream_delta() {
701 self.buffer.push(BufferedLine {
702 buffered_at: Instant::now(),
703 line,
704 });
705 let oldest = self.buffer.first().expect("just pushed").buffered_at;
706 if oldest.elapsed() >= self.throttle {
707 self.drain_buffer()?;
708 }
709 } else {
710 self.drain_buffer()?;
711 self.file.write_all(line.as_bytes())?;
712 self.file.flush()?;
713 self.file.sync_data()?;
714 // The line is durable; move the out-of-repo witness up to it.
715 // Only sealed missions carry one, the same missions whose lines
716 // a forger cannot rewrite, so the mark and the MAC cover the
717 // same set.
718 if self.authority_key.is_some() {
719 crate::paths::record_high_water(&self.repo_root, &self.mission_id, event.seq)?;
720 }
721 }
722
723 self.next_seq += 1;
724 Ok((event, findings))
725 }
726
727 /// Write any buffered deltas out to the file (no fsync — deltas are
728 /// recoverable).
729 pub fn flush(&mut self) -> Result<()> {
730 self.drain_buffer()?;
731 self.file.flush()?;
732 Ok(())
733 }
734
735 /// Elapsed time since the OLDEST buffered delta, or `None` when the
736 /// buffer is empty.
737 pub fn buffer_age(&self) -> Option<Duration> {
738 self.buffer.first().map(|b| b.buffered_at.elapsed())
739 }
740
741 /// Drain the buffer to the file, WITHOUT waiting for another [`append`]
742 /// call, if it is non-empty and has aged past `throttle`. Gives idle
743 /// missions (waiting on an approval gate, worker stopped) a wall-clock-
744 /// driven flush instead of leaving deltas buffered indefinitely.
745 ///
746 /// [`append`]: EventLog::append
747 pub fn flush_if_due(&mut self) -> Result<bool> {
748 match self.buffer_age() {
749 Some(age) if age >= self.throttle => {
750 self.drain_buffer()?;
751 self.file.flush()?;
752 Ok(true)
753 }
754 _ => Ok(false),
755 }
756 }
757
758 fn drain_buffer(&mut self) -> Result<()> {
759 drain_lines(&mut self.file, &mut self.buffer)?;
760 Ok(())
761 }
762
763 // -- readers (no lock required; used by the server/CLI to tail) ---------
764
765 /// Read and validate the full event log at `path`.
766 ///
767 /// Seq must start at 1 and increase by exactly 1; any gap or duplicate is
768 /// [`EngineError::LogCorruption`]. An unparseable FINAL line is a torn
769 /// write from a crash and is dropped with a warning; an unparseable line
770 /// anywhere else is corruption.
771 pub fn read_events(path: &Path) -> Result<Vec<Event>> {
772 Ok(Self::parse_log(path)?.events)
773 }
774
775 /// Read the log at `path` ONCE and return the validated events together
776 /// with the exact byte prefix they were parsed from (12th-pass review):
777 /// the evidence bundle must ship `events.jsonl` bytes that reproduce the
778 /// chain/cost/escalations it derived, so parsing one snapshot and then
779 /// rereading the file for the raw copy is not allowed — a concurrent
780 /// append between the two opens would ship bytes the folds never saw.
781 ///
782 /// Torn-tail rule (the honest one): an unparseable FINAL line is
783 /// dropped from the events AND excluded from the returned bytes — the
784 /// shipped prefix is exactly what parsed, so the bundle's log always
785 /// re-folds to the bundle's derived files. bytes-shipped == bytes-parsed.
786 pub fn read_events_and_log_bytes(path: &Path) -> Result<(Vec<Event>, Vec<u8>)> {
787 use std::io::Read;
788 // Same no-follow refusal as `parse_log`: never read through a
789 // symlink, `O_NOFOLLOW` on unix so there is no check-then-open window.
790 let mut bytes = Vec::new();
791 crate::paths::open_read_nofollow(path)?.read_to_end(&mut bytes)?;
792 let parsed = Self::parse_log_bytes(&bytes, path)?;
793 bytes.truncate(parsed.valid_len as usize);
794 Ok((parsed.events, bytes))
795 }
796
797 /// Parse and validate the log at `path`, tracking how many leading bytes
798 /// form the valid prefix so [`EventLog::acquire`] can truncate torn tails.
799 ///
800 /// Works on raw bytes (decoding each line lossily) because a torn write
801 /// can split a multi-byte UTF-8 character, which must not render the
802 /// whole log unreadable.
803 fn parse_log(path: &Path) -> Result<ParsedLog> {
804 // A symlinked log file is refused (never read through into another
805 // tree); an absent one errors NotFound from the read below, as
806 // before. `O_NOFOLLOW` on unix — no check-then-open window.
807 Self::parse_log_file(crate::paths::open_read_nofollow(path)?, path)
808 }
809
810 /// Parse from an already-open file. Acquisition uses this form so log
811 /// recovery reads through the same retained mission capability later
812 /// used for truncation, append, and lock removal.
813 fn parse_log_file(mut file: File, path: &Path) -> Result<ParsedLog> {
814 use std::io::Read;
815 let mut bytes = Vec::new();
816 file.read_to_end(&mut bytes)?;
817 Self::parse_log_bytes(&bytes, path)
818 }
819
820 /// Parse one in-memory buffer — the single entry point every reader
821 /// funnels into, so the validation rules (seq contiguity, one mission
822 /// id, torn-final-line drop, and the `h`/`m` integrity checks described
823 /// in the module docs) can never drift between the file-reading forms and
824 /// the single-snapshot form.
825 ///
826 /// Check order is load-bearing: seq and mission id are verified BEFORE
827 /// the chain, so a plain seq gap still reports as a seq discontinuity
828 /// rather than as the broken chain it also is.
829 fn parse_log_bytes(bytes: &[u8], path: &Path) -> Result<ParsedLog> {
830 let identity = log_identity(path);
831 if identity.is_none() {
832 // A log read from outside the `<repo>/.kranz/missions/<id>/`
833 // layout (a bundle, an archive, a copied file) gets the chain
834 // check only: no key, no floor, no mark. Say so, because
835 // chain-only is no defence against a forger (follow-up review
836 // F-6).
837 tracing::warn!(
838 path = %path.display(),
839 "event log read from a non-mission path: integrity verified by chain only"
840 );
841 }
842 let key = identity.and_then(|(root, _)| crate::paths::load_authority_key(root));
843 // The floor lives outside the repository, so an attacker who rewrites
844 // every line cannot lower it back to "this log was never sealed".
845 let seal_floor = identity
846 .and_then(|(root, mission)| crate::paths::read_seal_floor(root, mission))
847 .unwrap_or(u64::MAX);
848 let mut events = Vec::new();
849 let mut valid_len: usize = 0;
850 let mut terminated = true;
851 let mut offset: usize = 0;
852 let mut line_no: usize = 0;
853 let mut prev_hash = String::new();
854 let mut last_hash: Option<String> = None;
855 let mut saw_mac = false;
856 while offset < bytes.len() {
857 line_no += 1;
858 let rest = &bytes[offset..];
859 let (line_end, step) = match rest.iter().position(|&b| b == b'\n') {
860 Some(nl) => (nl, nl + 1),
861 None => (rest.len(), rest.len()),
862 };
863 let is_final = offset + step == bytes.len();
864 let line = String::from_utf8_lossy(&rest[..line_end]);
865 let mut value: serde_json::Value = match serde_json::from_str(&line) {
866 Ok(v) => v,
867 Err(err) => {
868 if is_final {
869 tracing::warn!(
870 path = %path.display(),
871 line = line_no,
872 error = %err,
873 "dropping unparseable final event line (torn write)"
874 );
875 break;
876 }
877 return Err(EngineError::LogCorruption(format!(
878 "unparseable event at {}:{}: {err}",
879 path.display(),
880 line_no
881 )));
882 }
883 };
884 // Lift the envelope OUT before deserializing: the canonical bytes the
885 // chain covers are the event without them, and removing the keys
886 // here means the `Event` type never has to tolerate extras.
887 let (presented_hash, presented_mac, version) = match value.as_object_mut() {
888 Some(object) => (
889 object
890 .remove(CHAIN_FIELD)
891 .and_then(|v| v.as_str().map(str::to_string)),
892 object
893 .remove(MAC_FIELD)
894 .and_then(|v| v.as_str().map(str::to_string)),
895 object.remove(VERSION_FIELD),
896 ),
897 None => (None, None, None),
898 };
899 let version = match version {
900 None => 1,
901 Some(value) if value.as_u64() == Some(EXACT_FLOAT_VERSION) => EXACT_FLOAT_VERSION,
902 Some(_) => {
903 return Err(EngineError::LogCorruption(format!(
904 "unsupported integrity version at {}:{}",
905 path.display(),
906 line_no
907 )));
908 }
909 };
910 if version == 1 {
911 restore_legacy_float_parsing(&mut value)?;
912 }
913 let event: Event = match serde_json::from_value(value) {
914 Ok(e) => e,
915 Err(err) => {
916 if is_final {
917 tracing::warn!(
918 path = %path.display(),
919 line = line_no,
920 error = %err,
921 "dropping unparseable final event line (torn write)"
922 );
923 break;
924 }
925 return Err(EngineError::LogCorruption(format!(
926 "unparseable event at {}:{}: {err}",
927 path.display(),
928 line_no
929 )));
930 }
931 };
932 let expected = events.len() as u64 + 1;
933 if event.seq != expected {
934 return Err(EngineError::LogCorruption(format!(
935 "seq discontinuity at {}:{}: expected {expected}, found {}",
936 path.display(),
937 line_no,
938 event.seq
939 )));
940 }
941 if let Some(first_mission_id) = events.first().map(|e: &Event| &e.mission_id) {
942 if event.mission_id != *first_mission_id {
943 return Err(EngineError::LogCorruption(format!(
944 "mission_id mismatch at {}:{}: expected '{}' (from first event), found '{}'",
945 path.display(),
946 line_no,
947 first_mission_id,
948 event.mission_id
949 )));
950 }
951 }
952 match &presented_hash {
953 Some(hash) => {
954 let body = canonical_event_bytes(&event, version)?;
955 let expected_hash = chain_hash(&prev_hash, &body, version);
956 if expected_hash != *hash {
957 return Err(EngineError::LogCorruption(format!(
958 "integrity chain broken at {}:{}: the line does not hash to its recorded `h`",
959 path.display(),
960 line_no
961 )));
962 }
963 match (&key, &presented_mac) {
964 (Some(key), Some(mac)) => {
965 let expected_mac = crate::hooks::hmac_sha256_hex(key, hash.as_bytes());
966 if !bool::from(expected_mac.as_bytes().ct_eq(mac.as_bytes())) {
967 return Err(EngineError::LogCorruption(format!(
968 "integrity mac does not verify at {}:{}",
969 path.display(),
970 line_no
971 )));
972 }
973 }
974 // A MAC we cannot check is not a MAC we reject: a
975 // reader with no key still gets the chain, which is
976 // the whole point of chaining separately.
977 (None, Some(_)) => {}
978 (_, None) if saw_mac || event.seq >= seal_floor => {
979 return Err(EngineError::LogCorruption(format!(
980 "integrity mac missing at {}:{}: this mission is sealed from seq {}, and earlier lines carry `m`",
981 path.display(),
982 line_no,
983 seal_floor
984 )));
985 }
986 (_, None) => {}
987 }
988 saw_mac |= presented_mac.is_some();
989 prev_hash = hash.clone();
990 last_hash = Some(hash.clone());
991 }
992 None if last_hash.is_some() => {
993 return Err(EngineError::LogCorruption(format!(
994 "integrity chain dropped at {}:{}: earlier lines carry `h`, so a line without one is a downgrade",
995 path.display(),
996 line_no
997 )));
998 }
999 None if event.seq >= seal_floor => {
1000 return Err(EngineError::LogCorruption(format!(
1001 "integrity chain missing at {}:{}: this mission is sealed from seq {seal_floor} on",
1002 path.display(),
1003 line_no
1004 )));
1005 }
1006 None if version == EXACT_FLOAT_VERSION => {
1007 return Err(EngineError::LogCorruption(format!(
1008 "integrity chain missing at {}:{}: versioned lines must be sealed",
1009 path.display(),
1010 line_no
1011 )));
1012 }
1013 // Unchained legacy prefix: tolerated, and the chain starts
1014 // fresh from the empty string at the first line that has one.
1015 None => {}
1016 }
1017 events.push(event);
1018 offset += step;
1019 valid_len = offset;
1020 terminated = step > line_end;
1021 }
1022 Ok(ParsedLog {
1023 events,
1024 valid_len: valid_len as u64,
1025 terminated,
1026 last_hash,
1027 saw_mac,
1028 })
1029 }
1030
1031 /// Read events with `seq > after_seq` (WS reconnect / tailing). The whole
1032 /// log is still validated — a corrupt prefix must not go unnoticed.
1033 pub fn read_events_after(path: &Path, after_seq: u64) -> Result<Vec<Event>> {
1034 let mut events = Self::read_events(path)?;
1035 events.retain(|e| e.seq > after_seq);
1036 Ok(events)
1037 }
1038
1039 /// Read the events on the last `max_bytes` of the log WITHOUT reading or
1040 /// validating the full file — O(tail) I/O for hot callers that only need
1041 /// trailing facts (e.g. "has a terminal lifecycle event been appended?").
1042 ///
1043 /// The window is aligned to the first complete line inside it, and
1044 /// unparseable lines (a torn final write) are skipped rather than treated
1045 /// as corruption — callers that need validation use
1046 /// [`EventLog::read_events`]. Returns the whole log when the file fits
1047 /// inside the window.
1048 pub fn read_tail_events(path: &Path, max_bytes: u64) -> Result<Vec<Event>> {
1049 use std::io::{Read, Seek, SeekFrom};
1050 // Same no-follow refusal as `parse_log`: never tail through a
1051 // symlink, `O_NOFOLLOW` on unix so there is no check-then-open window.
1052 let mut file = crate::paths::open_read_nofollow(path)?;
1053 let len = file.metadata()?.len();
1054 let window_start = len.saturating_sub(max_bytes);
1055 // Read from one byte BEFORE the window: when the window happens to
1056 // start exactly on a line boundary, that extra byte is the previous
1057 // line's '\n', so the drop-through-first-'\n' below discards zero
1058 // content bytes instead of eating one complete in-window line.
1059 let start = window_start.saturating_sub(1);
1060 file.seek(SeekFrom::Start(start))?;
1061 let mut bytes = Vec::with_capacity((len - start) as usize);
1062 file.read_to_end(&mut bytes)?;
1063 let mut slice = bytes.as_slice();
1064 if window_start > 0 {
1065 // Drop the line the window cut into; its head is outside.
1066 match slice.iter().position(|&b| b == b'\n') {
1067 Some(nl) => slice = &slice[nl + 1..],
1068 None => return Ok(Vec::new()),
1069 }
1070 }
1071 Ok(slice
1072 .split(|&b| b == b'\n')
1073 .filter(|line| !line.is_empty())
1074 .filter_map(|line| {
1075 let mut value: serde_json::Value =
1076 serde_json::from_str(&String::from_utf8_lossy(line)).ok()?;
1077 let object = value.as_object_mut()?;
1078 let version = object.remove(VERSION_FIELD);
1079 object.remove(CHAIN_FIELD);
1080 object.remove(MAC_FIELD);
1081 match version {
1082 None => restore_legacy_float_parsing(&mut value).ok()?,
1083 Some(version) if version.as_u64() == Some(EXACT_FLOAT_VERSION) => {}
1084 Some(_) => return None,
1085 }
1086 serde_json::from_value(value).ok()
1087 })
1088 .collect())
1089 }
1090}
1091
1092impl Drop for EventLog {
1093 fn drop(&mut self) {
1094 if let Err(e) = self.flush() {
1095 tracing::warn!(
1096 error = %e,
1097 retained = self.buffer.len(),
1098 "failed to flush event buffer on drop; buffered deltas retained for a future drain"
1099 );
1100 }
1101 // Only remove the lock if we still own it. After a --force-lock steal
1102 // the stolen-from process's teardown would otherwise delete the
1103 // stealer's lock file; the stealer would then see generation 0 and
1104 // fail closed on its next append (MutationLock in queue.rs uses the
1105 // same still-ours check).
1106 let info = read_lock_info_at(&self.mission_dir, "events.jsonl.lock");
1107 let generation_matches = info.generation.unwrap_or(0) == self.lock_generation;
1108 let token_matches = match (&self.lock_token, &info.token) {
1109 (Some(ours), Some(theirs)) => ours == theirs,
1110 // Platforms/legacy files without a token: generation alone is
1111 // the ownership fence.
1112 _ => true,
1113 };
1114 if generation_matches && token_matches {
1115 if let Err(e) = self.mission_dir.remove_file("events.jsonl.lock") {
1116 if e.kind() != ErrorKind::NotFound {
1117 tracing::warn!(
1118 path = %self.lock_path.display(),
1119 error = %e,
1120 "failed to remove lock file on drop"
1121 );
1122 }
1123 }
1124 }
1125 }
1126}
1127
1128/// Contended-path acquire: decide whether the existing lock at `lock_path`
1129/// may be stolen under `force` and, if so, atomically replace it with a fresh
1130/// lock file owned by this process.
1131///
1132/// unix: the whole probe→remove→create sequence is serialized under an
1133/// exclusive `flock` on a sibling guard file (`<lock>.steal`). Unserialized,
1134/// two concurrent acquires can both judge the same stale holder Dead; the
1135/// slower one's `remove_file` then deletes the FASTER one's freshly created
1136/// lock and both end up holding. After winning the flock the lock is
1137/// re-attempted and re-probed from scratch — it may have been released, or
1138/// stolen and rewritten, while we waited.
1139///
1140/// non-unix: no guard, and that is acceptable — liveness is never probed
1141/// there (every verdict is Unknown, see [`probe_liveness`]), so the Dead
1142/// auto-steal path cannot trigger; steals happen only under explicit operator
1143/// force flags, which are deliberate one-off actions rather than the
1144/// concurrent-by-accident crash-recovery restarts the guard defends against.
1145fn steal_lock(
1146 mission_dir: &Dir,
1147 lock_name: &str,
1148 lock_path: &Path,
1149 force: LockForce,
1150) -> Result<(File, u64)> {
1151 #[cfg(unix)]
1152 let _guard = StealGuard::acquire(mission_dir, lock_name)?;
1153
1154 loop {
1155 // The lock may have been RELEASED while we waited for the guard:
1156 // retry the clean create before probing anything.
1157 match open_create_new_at(mission_dir, lock_name) {
1158 Ok(f) => return Ok((f, 0)),
1159 Err(e) if e.kind() == ErrorKind::AlreadyExists => {}
1160 Err(e) => return Err(e.into()),
1161 }
1162
1163 let info = read_lock_info_at(mission_dir, lock_name);
1164 authorize_steal(lock_path, &info, force)?;
1165 let prev_gen = info.generation.unwrap_or(0);
1166
1167 // Guarded steals never interleave here, but a rival acquire's FIRST
1168 // (unguarded) create attempt can still slip into the remove→create
1169 // window and win the freed slot. If it does, loop back and judge
1170 // THAT holder like any other — never surface the raw io collision.
1171 match mission_dir.remove_file(lock_name) {
1172 Ok(()) => {}
1173 Err(e) if e.kind() == ErrorKind::NotFound => continue,
1174 Err(e) => return Err(e.into()),
1175 }
1176 match open_create_new_at(mission_dir, lock_name) {
1177 Ok(f) => return Ok((f, prev_gen)),
1178 Err(e) if e.kind() == ErrorKind::AlreadyExists => continue,
1179 Err(e) => return Err(e.into()),
1180 }
1181 }
1182}
1183
1184/// Probe the CURRENT holder recorded at `lock_path` and decide, against the
1185/// [`LockForce`] matrix, whether stealing is permitted: `Ok(())` authorizes
1186/// the steal, `Err(LockHeld)` refuses with operator guidance.
1187fn authorize_steal(lock_path: &Path, info: &LockInfo, force: LockForce) -> Result<()> {
1188 match (probe_liveness(info), force) {
1189 // A provably-dead holder is stale (e.g. the engine was Ctrl-C'd —
1190 // SIGINT skips destructors — or its pid was provably reused): steal
1191 // it at every tier without demanding --force-lock.
1192 (LockLiveness::Dead, _) => {
1193 tracing::warn!(
1194 lock = %lock_path.display(),
1195 holder = %info.holder,
1196 "stale engine lock (holder is dead); taking over"
1197 );
1198 Ok(())
1199 }
1200 // Not provably dead and no force: refuse. Same message whether the
1201 // holder is alive or indeterminate — without force the distinction
1202 // changes nothing for the operator.
1203 (LockLiveness::Unknown | LockLiveness::Alive, LockForce::No) => {
1204 Err(EngineError::LockHeld(format!(
1205 "lock file {} exists (held by pid {}); if that process \
1206 is truly gone, re-run with --force-lock",
1207 lock_path.display(),
1208 info.holder
1209 )))
1210 }
1211 // Indeterminate liveness (unparseable pid, non-unix platform):
1212 // --force-lock keeps its historical meaning and steals.
1213 (LockLiveness::Unknown, LockForce::IfNotLive | LockForce::EvenIfLive) => {
1214 tracing::warn!(
1215 lock = %lock_path.display(),
1216 holder = %info.holder,
1217 "forced takeover of a lock whose holder's liveness \
1218 cannot be determined"
1219 );
1220 Ok(())
1221 }
1222 // A provably ALIVE holder survives --force-lock: this is exactly how
1223 // an operator who believes a long run is "stuck" would otherwise
1224 // corrupt it.
1225 (LockLiveness::Alive, LockForce::IfNotLive) => Err(EngineError::LockHeld(format!(
1226 "lock file {} is held by pid {}, and that process is \
1227 ALIVE — refusing --force-lock. Identify it with \
1228 `ps -p {}`; pass --dangerously-steal-live-lock ONLY \
1229 if you are certain it is a zombie or foreign process \
1230 and not a running kranz engine (two engines on one \
1231 mission corrupt its event log)",
1232 lock_path.display(),
1233 info.holder,
1234 info.holder
1235 ))),
1236 (LockLiveness::Alive, LockForce::EvenIfLive) => {
1237 tracing::warn!(
1238 lock = %lock_path.display(),
1239 holder = %info.holder,
1240 "DANGEROUS: stealing the mission lock from a LIVE \
1241 process at operator request \
1242 (--dangerously-steal-live-lock); if that process is \
1243 a kranz engine, two engines now write one event log"
1244 );
1245 Ok(())
1246 }
1247 }
1248}
1249
1250/// RAII serialization of the lock-steal sequence (unix): an exclusive
1251/// `flock` on a sibling guard file (`<lock>.steal`).
1252///
1253/// The guard file is best-effort removed on drop, so acquisition must defend
1254/// against the unlink race: a waiter can win the flock on an inode that was
1255/// unlinked (and possibly recreated) while it slept, which would serialize
1256/// nothing. After each flock win the held fd's identity is compared to
1257/// whatever the path names NOW; a mismatch retries on the fresh file.
1258///
1259/// `flock` contends across separate fds within one process too, so the guard
1260/// serializes racing threads exactly like racing processes.
1261#[cfg(unix)]
1262struct StealGuard {
1263 /// Held only for the flock; dropping (closing) it releases the lock.
1264 _file: File,
1265 dir: Dir,
1266 name: OsString,
1267}
1268
1269#[cfg(unix)]
1270impl StealGuard {
1271 fn acquire(dir: &Dir, lock_name: &str) -> Result<StealGuard> {
1272 use std::os::unix::io::AsRawFd;
1273 let mut name = OsString::from(lock_name);
1274 name.push(".steal");
1275 loop {
1276 // Contents are irrelevant (the file exists only to be flock'd),
1277 // but be explicit that nothing is truncated.
1278 use cap_fs_ext::OpenOptionsExt as _;
1279 use cap_fs_ext::OpenOptionsFollowExt as _;
1280 use cap_primitives::fs::FollowSymlinks;
1281 let mut options = cap_std::fs::OpenOptions::new();
1282 options
1283 .write(true)
1284 .truncate(false)
1285 .follow(FollowSymlinks::No);
1286 options.custom_flags(libc::O_NONBLOCK);
1287 let file = match dir.open_with(&name, &options) {
1288 Ok(file) => file.into_std(),
1289 Err(error) if error.kind() == ErrorKind::NotFound => {
1290 let mut create = cap_std::fs::OpenOptions::new();
1291 create
1292 .write(true)
1293 .create_new(true)
1294 .follow(FollowSymlinks::No);
1295 create.custom_flags(libc::O_NONBLOCK);
1296 match dir.open_with(&name, &create) {
1297 Ok(file) => file.into_std(),
1298 Err(error) if error.kind() == ErrorKind::AlreadyExists => continue,
1299 Err(error) => return Err(error.into()),
1300 }
1301 }
1302 Err(error) => return Err(error.into()),
1303 };
1304 if !file.metadata()?.is_file() {
1305 return Err(EngineError::InvalidState(format!(
1306 "event-log steal guard {} is not a regular file",
1307 name.to_string_lossy()
1308 )));
1309 }
1310 loop {
1311 if unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX) } == 0 {
1312 break;
1313 }
1314 let err = std::io::Error::last_os_error();
1315 if err.raw_os_error() != Some(libc::EINTR) {
1316 return Err(err.into());
1317 }
1318 }
1319 let held = file.metadata()?;
1320 match dir.symlink_metadata(&name) {
1321 Ok(m)
1322 if cap_fs_ext::MetadataExt::dev(&m)
1323 == std::os::unix::fs::MetadataExt::dev(&held)
1324 && cap_fs_ext::MetadataExt::ino(&m)
1325 == std::os::unix::fs::MetadataExt::ino(&held) =>
1326 {
1327 return Ok(StealGuard {
1328 _file: file,
1329 dir: dir.try_clone()?,
1330 name,
1331 });
1332 }
1333 // The guard file was unlinked (and possibly recreated) while
1334 // we waited: this flock guards a dead inode and serializes
1335 // nothing. Retry on whatever the path names now.
1336 _ => continue,
1337 }
1338 }
1339 }
1340}
1341
1342#[cfg(unix)]
1343impl Drop for StealGuard {
1344 fn drop(&mut self) {
1345 // Best-effort cleanup; the identity re-check in acquire() keeps this
1346 // safe against waiters still blocked on the removed inode.
1347 let _ = self.dir.remove_file(&self.name);
1348 }
1349}
1350
1351/// Whether the holder recorded in the lock file at `lock_path` is still
1352/// alive — the ONE liveness query shared by every subsystem that asks "is
1353/// this mission's engine running?" (the event-log acquire path itself, queue
1354/// busy checks, hygiene sweeps), so no caller can diverge from the canonical
1355/// lock-file format.
1356///
1357/// Missing lock file → `false` (nothing holds it). A provably-dead holder
1358/// (ESRCH, or a token-proven pid reuse) → `false`. Everything else — alive
1359/// holder, unparseable/garbage lock file, unprobeable platform — → `true`:
1360/// conservative, because a false "alive" merely delays a queued mission or
1361/// spares a directory from cleaning, while a false "dead" runs two engines
1362/// on one working tree.
1363pub fn lock_holder_is_alive(lock_path: &Path) -> bool {
1364 if !std::fs::symlink_metadata(lock_path).is_ok_and(|m| m.file_type().is_file()) {
1365 return false;
1366 }
1367 let info = read_lock_info(lock_path);
1368 match probe_liveness(&info) {
1369 LockLiveness::Dead => false,
1370 LockLiveness::Alive | LockLiveness::Unknown => true,
1371 }
1372}
1373
1374/// Lock-file contents for a lock held by the current process, in the same
1375/// format parsed by [`lock_holder_is_alive`].
1376///
1377/// Format (four lines):
1378/// `<pid>\n<acquired_unix_secs>\n<identity_token>\n<generation>\n`
1379///
1380/// `generation` increments on every steal so a stolen-from process can detect
1381/// that its append handle is no longer authoritative.
1382pub fn current_lock_holder_record() -> String {
1383 current_lock_holder_record_with_generation(0)
1384}
1385
1386fn current_lock_holder_record_with_generation(generation: u64) -> String {
1387 let acquired_secs = std::time::SystemTime::now()
1388 .duration_since(std::time::UNIX_EPOCH)
1389 .map(|d| d.as_secs())
1390 .unwrap_or(0);
1391 let mut contents = format!("{}\n{}\n", std::process::id(), acquired_secs);
1392 if let Some(token) = process_identity_token(std::process::id() as i32) {
1393 contents.push_str(&token);
1394 contents.push('\n');
1395 } else {
1396 contents.push('\n');
1397 }
1398 contents.push_str(&format!("{generation}\n"));
1399 contents
1400}
1401
1402/// Liveness verdict for the process recorded in a lock file.
1403///
1404/// INVARIANT: anything uncertain must NEVER report `Dead` — a false `Dead`
1405/// lets two engines write one log. `Dead` requires positive proof: ESRCH
1406/// from `kill(pid, 0)`, or a detected pid reuse (the holder process started
1407/// after the lock was acquired).
1408#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1409enum LockLiveness {
1410 /// The holder is provably gone (or the pid provably belongs to a
1411 /// different, younger process): the lock is stale.
1412 Dead,
1413 /// The holder pid maps to a running process.
1414 Alive,
1415 /// Cannot tell (unparseable pid, non-unix platform, unexpected probe
1416 /// errno): honor the lock unless forced.
1417 Unknown,
1418}
1419
1420/// What a lock file records about its holder.
1421#[derive(Debug)]
1422struct LockInfo {
1423 /// First line of the lock file, for error messages (`"unknown"` when the
1424 /// file is empty or unreadable).
1425 holder: String,
1426 /// The holder pid, when the first line parses as a positive i32.
1427 pid: Option<i32>,
1428 /// Unix epoch seconds at acquire time (second line): diagnostics only.
1429 /// `None` for the legacy one-line pid-only format.
1430 acquired_secs: Option<u64>,
1431 /// Process identity token the holder recorded for ITSELF at acquire
1432 /// (third line, see [`process_identity_token`]). `None` for the older
1433 /// one-/two-line formats and on platforms that cannot produce one —
1434 /// reuse detection is then impossible and an alive pid is simply Alive.
1435 token: Option<String>,
1436 /// Steal generation (fourth line). `None` for legacy lock files — treated
1437 /// as generation 0 by append ownership checks.
1438 generation: Option<u64>,
1439}
1440
1441/// Best-effort parse of a lock file:
1442/// `<pid>\n<acquired_unix_epoch_secs>\n<identity_token>\n<generation>`,
1443/// tolerating the legacy one-/two-/three-line formats and arbitrary garbage.
1444fn read_lock_info(lock_path: &Path) -> LockInfo {
1445 use std::io::Read;
1446 let mut contents = String::new();
1447 if let Ok(file) = crate::paths::open_read_nofollow(lock_path) {
1448 let _ = file.take(8 * 1024).read_to_string(&mut contents);
1449 }
1450 parse_lock_info(&contents)
1451}
1452
1453/// Capability-relative lock read used after [`EventLog::acquire`] pins the
1454/// mission directory. The small bound prevents a hostile stale lock from
1455/// turning liveness checks into unbounded allocation.
1456fn read_lock_info_at(dir: &Dir, name: &str) -> LockInfo {
1457 use std::io::Read;
1458 let mut contents = String::new();
1459 if let Ok(file) = open_read_at(dir, name) {
1460 let _ = file.take(8 * 1024).read_to_string(&mut contents);
1461 }
1462 parse_lock_info(&contents)
1463}
1464
1465fn parse_lock_info(contents: &str) -> LockInfo {
1466 let mut lines = contents.lines();
1467 let first = lines.next().unwrap_or("").trim();
1468 let holder = if first.is_empty() {
1469 "unknown".to_string()
1470 } else {
1471 first.to_string()
1472 };
1473 let pid = first.parse::<i32>().ok().filter(|p| *p > 0);
1474 let acquired_secs = lines.next().and_then(|l| l.trim().parse::<u64>().ok());
1475 let token = lines
1476 .next()
1477 .map(str::trim)
1478 .filter(|t| !t.is_empty())
1479 .map(String::from);
1480 let generation = lines.next().and_then(|l| l.trim().parse::<u64>().ok());
1481 LockInfo {
1482 holder,
1483 pid,
1484 acquired_secs,
1485 token,
1486 generation,
1487 }
1488}
1489
1490/// Probe the liveness of a lock file's recorded holder.
1491///
1492/// unix: `kill(pid, 0)` == 0 → Alive; EPERM → Alive (exists, not ours);
1493/// ESRCH → Dead; any other errno → Unknown. Unparseable or non-positive pid →
1494/// Unknown. Our OWN pid → Alive (we hold it). Non-unix → Unknown. An Alive
1495/// verdict is then screened for pid reuse (see [`alive_or_reused`]).
1496fn probe_liveness(info: &LockInfo) -> LockLiveness {
1497 let Some(pid) = info.pid else {
1498 return LockLiveness::Unknown;
1499 };
1500 if pid as u32 == std::process::id() {
1501 // We recorded this pid ourselves (double acquire) — or a dead engine
1502 // did and the OS recycled its pid onto us, which the reuse screen
1503 // can prove from the timestamps.
1504 return alive_or_reused(pid, info);
1505 }
1506 #[cfg(unix)]
1507 {
1508 // kill(pid, 0): 0 = alive; EPERM = alive but not ours; ESRCH = dead.
1509 if unsafe { libc::kill(pid, 0) } == 0 {
1510 return alive_or_reused(pid, info);
1511 }
1512 match std::io::Error::last_os_error().raw_os_error() {
1513 Some(libc::EPERM) => alive_or_reused(pid, info),
1514 Some(libc::ESRCH) => LockLiveness::Dead,
1515 _ => LockLiveness::Unknown,
1516 }
1517 }
1518 #[cfg(not(unix))]
1519 {
1520 // No non-unix equivalent of `kill(pid, 0)` is wired up here, and
1521 // `process_identity_token`'s non-unix stub always returns `None`, so
1522 // `alive_or_reused` can never reach `Dead` for a foreign pid on this
1523 // platform. Liveness is therefore unprovable for a foreign holder:
1524 // report Unknown rather than guessing, and never Dead.
1525 tracing::debug!(
1526 pid,
1527 "liveness cannot be proven for a foreign pid on this platform; \
1528 reporting Unknown (Dead is unreachable here)"
1529 );
1530 non_unix_liveness_fallback()
1531 }
1532}
1533
1534/// The verdict `probe_liveness` reports for a foreign pid on non-unix
1535/// targets, where liveness cannot be proven. Factored out (and compiled on
1536/// every platform) so a cross-platform test can pin that it is `Unknown` and
1537/// never `Dead` — uncertainty must never demote to Dead.
1538#[cfg_attr(unix, allow(dead_code))]
1539fn non_unix_liveness_fallback() -> LockLiveness {
1540 LockLiveness::Unknown
1541}
1542
1543/// Screen an Alive pid for reuse by comparing process identity tokens: the
1544/// token the holder recorded for ITSELF at acquire (lock line 3) against the
1545/// token of whatever occupies that pid NOW. Tokens are compared for raw
1546/// equality — no clocks, no slack — so wall-clock steps (NTP corrections,
1547/// manual resets) can never misclassify a live holder as Dead the way the
1548/// old start-time-vs-acquire-time arithmetic could. Different tokens prove
1549/// the pid was recycled within this boot, or the machine rebooted; the
1550/// process that wrote the lock is dead either way, so the verdict is `Dead`.
1551/// Any inability to obtain a token (legacy one-/two-line lock file,
1552/// unsupported platform, probe error) keeps the verdict `Alive` — uncertainty
1553/// must never demote Alive to Dead.
1554fn alive_or_reused(pid: i32, info: &LockInfo) -> LockLiveness {
1555 let Some(recorded) = info.token.as_deref() else {
1556 return LockLiveness::Alive;
1557 };
1558 let Some(current) = process_identity_token(pid) else {
1559 return LockLiveness::Alive;
1560 };
1561 if current == recorded {
1562 LockLiveness::Alive
1563 } else {
1564 tracing::warn!(
1565 pid,
1566 recorded_token = recorded,
1567 current_token = %current,
1568 lock_acquired_epoch_secs = ?info.acquired_secs,
1569 "lock holder pid was REUSED: the process now at this pid is not \
1570 the one that recorded the lock, so the engine that wrote the \
1571 lock is dead"
1572 );
1573 LockLiveness::Dead
1574 }
1575}
1576
1577/// Platform-opaque identity token for process `pid`, or `None` when this
1578/// platform cannot produce one. Two readings of a LIVE process's token are
1579/// byte-identical because both derive from the same immutable kernel state
1580/// (boot + spawn identity), never from wall-clock arithmetic — so tokens are
1581/// compared for raw equality only.
1582///
1583/// linux: `"<boot_id>:<starttime_ticks>"`. `/proc/sys/kernel/random/boot_id`
1584/// is fixed per boot; `/proc/<pid>/stat` field 22 (starttime) is a
1585/// CLOCK_MONOTONIC-based tick count fixed for the life of the process. comm
1586/// (field 2) is parenthesized and may itself contain spaces or ')', so
1587/// fields 3.. are indexed from after the LAST ')'.
1588#[cfg(target_os = "linux")]
1589pub(crate) fn process_identity_token(pid: i32) -> Option<String> {
1590 let boot_id = std::fs::read_to_string("/proc/sys/kernel/random/boot_id").ok()?;
1591 let boot_id = boot_id.trim();
1592 if boot_id.is_empty() {
1593 return None;
1594 }
1595 let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
1596 let rest = stat.get(stat.rfind(')')? + 1..)?;
1597 let start_ticks = rest.split_whitespace().nth(19)?;
1598 start_ticks.parse::<u64>().ok()?; // reject garbage rather than record it
1599 Some(format!("{boot_id}:{start_ticks}"))
1600}
1601
1602/// macOS: `ps -p <pid> -o lstart=` prints an absolute spawn timestamp
1603/// rendered from the kernel's stored `p_starttime`. Locale and timezone are
1604/// pinned so the acquire-time and probe-time renderings of the SAME stored
1605/// value are byte-identical; the strings are compared for equality, never
1606/// parsed back into clock arithmetic.
1607///
1608/// This is the actual `ps` spawn — the seam [`process_identity_token`] caches
1609/// in front of. Kept as a separate function (rather than inlining the
1610/// `Command` call) so a test can observe how many times it actually ran, via
1611/// [`PS_SPAWN_COUNT`].
1612#[cfg(target_os = "macos")]
1613fn ps_identity_token(pid: i32) -> Option<String> {
1614 #[cfg(test)]
1615 {
1616 *PS_SPAWN_COUNTS
1617 .get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()))
1618 .lock()
1619 .unwrap()
1620 .entry(pid)
1621 .or_insert(0) += 1;
1622 }
1623 let out = std::process::Command::new("ps")
1624 .env("LC_ALL", "C")
1625 .env("TZ", "UTC")
1626 .args(["-p", &pid.to_string(), "-o", "lstart="])
1627 .output()
1628 .ok()?;
1629 if !out.status.success() {
1630 return None;
1631 }
1632 let token = String::from_utf8_lossy(&out.stdout).trim().to_string();
1633 if token.is_empty() {
1634 None
1635 } else {
1636 Some(token)
1637 }
1638}
1639
1640/// Counts real `ps` spawns from [`ps_identity_token`], keyed by pid, so a
1641/// test can prove the cache in [`process_identity_token`] collapses repeated
1642/// checks for one pid into a single spawn — without being confused by other
1643/// tests in this file concurrently spawning `ps` for a DIFFERENT pid.
1644/// Test-only: it exists purely to observe the seam.
1645#[cfg(all(test, target_os = "macos"))]
1646static PS_SPAWN_COUNTS: std::sync::OnceLock<
1647 std::sync::Mutex<std::collections::HashMap<i32, usize>>,
1648> = std::sync::OnceLock::new();
1649
1650/// How long a cached macOS identity token may be served before a fresh `ps`
1651/// spawn is required. This window only needs to be long enough to collapse
1652/// the handful of `alive_or_reused` calls a single steal decision or hygiene
1653/// sweep makes for the SAME pid (microseconds to low milliseconds apart in
1654/// practice); it must stay far shorter than any realistic pid-reuse
1655/// turnaround (the OS has to fully tear down the old process and allocate a
1656/// new one, which takes at least tens of milliseconds, typically much more).
1657/// A cached token therefore can never span an actual reuse: by the time a
1658/// pid is recycled, the cache entry for it has long since expired and the
1659/// next probe spawns fresh `ps`.
1660#[cfg(target_os = "macos")]
1661const IDENTITY_TOKEN_CACHE_TTL: Duration = Duration::from_millis(50);
1662
1663/// Per-pid cache of [`ps_identity_token`] results, so a burst of liveness
1664/// checks against the same pid (queue-busy checks, hygiene sweeps, a single
1665/// steal decision) spawns at most one `ps`. Keyed by pid so a lookup for one
1666/// pid can never return another pid's token. Guarded by a `Mutex` for safe
1667/// concurrent access.
1668#[cfg(target_os = "macos")]
1669type IdentityTokenCache =
1670 std::sync::Mutex<std::collections::HashMap<i32, (Option<String>, Instant)>>;
1671
1672#[cfg(target_os = "macos")]
1673static IDENTITY_TOKEN_CACHE: std::sync::OnceLock<IdentityTokenCache> = std::sync::OnceLock::new();
1674
1675/// macOS identity token WITHOUT the `ps` spawn: `proc_pidinfo(
1676/// PROC_PIDTBSDINFO)` reads the kernel's stored `p_starttime` directly — the
1677/// same immutable value `ps -o lstart=` renders — and this renders it
1678/// byte-identically (ctime shape, UTC: probed 2026-08-05 against
1679/// `LC_ALL=C TZ=UTC ps -p <pid> -o lstart=`, e.g. `Wed Aug 5 00:34:16 2026`
1680/// from both paths for the same process). Byte-identity is load-bearing:
1681/// tokens are compared for raw equality against lock-file recordings that
1682/// may predate this path (recorded via `ps`), so the rendering must not
1683/// drift.
1684///
1685/// Why this path exists (ticket gate-sandbox-supervision-dogfood): `/bin/ps`
1686/// is setuid root, and setuid exec is kernel-denied inside ANY Seatbelt
1687/// sandbox — probed: EPERM even under `(allow default)`, not expressible in
1688/// SBPL, and a copied binary is AMFI-killed on exec. A process inside the
1689/// gate sandbox wrap (a wrapped `cargo test --workspace` dogfooding this
1690/// repo, or any wrapped contract command that probes a kranz lock) could
1691/// therefore NEVER obtain a token via `ps`. `proc_pidinfo` is not
1692/// sandbox-gated for same-uid targets (probed under the session-profile
1693/// posture: self, children, and unrelated same-uid host processes all
1694/// answer) and needs no spawn at all.
1695///
1696/// The limit: OTHER-UID pids. Unprivileged `proc_pidinfo` on pid 1 is EPERM
1697/// (probed, unsandboxed included) — which is exactly why `/bin/ps` carries
1698/// the setuid bit. Those pids fall back to the [`ps_identity_token`] spawn
1699/// seam, which keeps answering them wherever setuid exec is permitted.
1700#[cfg(target_os = "macos")]
1701fn proc_pidinfo_identity_token(pid: i32) -> Option<String> {
1702 let mut info: libc::proc_bsdinfo = unsafe { std::mem::zeroed() };
1703 let rc = unsafe {
1704 libc::proc_pidinfo(
1705 pid,
1706 libc::PROC_PIDTBSDINFO,
1707 0,
1708 &mut info as *mut libc::proc_bsdinfo as *mut libc::c_void,
1709 std::mem::size_of::<libc::proc_bsdinfo>() as i32,
1710 )
1711 };
1712 if rc <= 0 {
1713 return None;
1714 }
1715 let secs = i64::try_from(info.pbi_start_tvsec).ok()?;
1716 let rendered = chrono::DateTime::from_timestamp(secs, 0)?
1717 .format("%a %b %e %H:%M:%S %Y")
1718 .to_string();
1719 if rendered.is_empty() {
1720 None
1721 } else {
1722 Some(rendered)
1723 }
1724}
1725
1726/// The uncached token lookup [`process_identity_token`] memoizes:
1727/// [`proc_pidinfo_identity_token`] first (no spawn, works inside the gate
1728/// sandbox wrap), the `ps` spawn seam only for the pids the unprivileged
1729/// syscall cannot read (other-uid — see its doc). [`PS_SPAWN_COUNTS`] still
1730/// counts REAL spawns only, so the cache test's pid-1 probe stays the sole
1731/// contributor to its own count.
1732#[cfg(target_os = "macos")]
1733fn uncached_identity_token(pid: i32) -> Option<String> {
1734 if let Some(token) = proc_pidinfo_identity_token(pid) {
1735 return Some(token);
1736 }
1737 ps_identity_token(pid)
1738}
1739
1740/// Cache layer in front of [`uncached_identity_token`]: the raw token lookup
1741/// (proc_pidinfo first, the real `ps` spawn seam behind it).
1742/// The VERDICT (Alive/Dead) is never cached or short-circuited here — only
1743/// the raw token lookup is memoized; [`alive_or_reused`] still compares
1744/// `recorded == current` on every call, using whatever token this returns.
1745#[cfg(target_os = "macos")]
1746pub(crate) fn process_identity_token(pid: i32) -> Option<String> {
1747 let cache = IDENTITY_TOKEN_CACHE
1748 .get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()));
1749 let now = Instant::now();
1750 if let Some((token, captured)) = cache.lock().unwrap().get(&pid) {
1751 if now.duration_since(*captured) < IDENTITY_TOKEN_CACHE_TTL {
1752 return token.clone();
1753 }
1754 }
1755 let identity = uncached_identity_token(pid);
1756 cache.lock().unwrap().insert(pid, (identity.clone(), now));
1757 identity
1758}
1759
1760/// Everywhere else (windows, exotic unix): no identity token, so pid reuse
1761/// cannot be proven and an alive holder stays Alive.
1762#[cfg(not(any(target_os = "linux", target_os = "macos")))]
1763pub(crate) fn process_identity_token(_pid: i32) -> Option<String> {
1764 None
1765}
1766
1767#[cfg(test)]
1768mod tests {
1769 use super::*;
1770
1771 #[test]
1772 fn legacy_float_parser_matches_independent_reference() {
1773 let reference: serde_json::Value =
1774 serde_json::from_str(include_str!("../tests/fixtures/legacy-json-floats.json"))
1775 .unwrap();
1776 let cases = reference["cases"].as_array().unwrap();
1777 assert_eq!(cases.len(), 96);
1778 for case in cases {
1779 let number: serde_json::Number =
1780 serde_json::from_str(case["json"].as_str().unwrap()).unwrap();
1781 let actual = legacy_float_number(&number)
1782 .and_then(|n| n.as_f64())
1783 .map(f64::to_bits);
1784 let expected = case["expected_bits"]
1785 .as_str()
1786 .map(|bits| bits.parse::<u64>().unwrap());
1787 assert_eq!(actual, expected, "reference: {case}");
1788 }
1789 }
1790
1791 #[test]
1792 fn versioned_canonical_bytes_sort_nested_objects() {
1793 let event = Event {
1794 seq: 1,
1795 ts: "2026-09-05T00:00:00Z".parse().unwrap(),
1796 mission_id: "m-canonical".into(),
1797 kind: EventKind::ConfigChanged {
1798 patch: serde_json::json!({"z": 1, "a": [{"d": 2, "b": 3}]}),
1799 },
1800 };
1801 assert_eq!(
1802 canonical_event_bytes(&event, EXACT_FLOAT_VERSION).unwrap(),
1803 r#"{"missionId":"m-canonical","payload":{"patch":{"a":[{"b":3,"d":2}],"z":1}},"seq":1,"ts":"2026-09-05T00:00:00Z","type":"config.changed"}"#
1804 );
1805 }
1806
1807 // `probe_liveness`'s non-unix arm (the actual code path this pins) only
1808 // compiles under `#[cfg(not(unix))]`, and our CI runs macOS/Linux, so it
1809 // cannot be exercised directly here. `non_unix_liveness_fallback` is
1810 // factored out and compiled on ALL platforms so this cross-platform test
1811 // can still pin its invariant: uncertainty must never demote to Dead.
1812 #[test]
1813 fn non_unix_liveness_fallback_is_never_dead() {
1814 assert_eq!(non_unix_liveness_fallback(), LockLiveness::Unknown);
1815 assert_ne!(non_unix_liveness_fallback(), LockLiveness::Dead);
1816 }
1817
1818 /// A valid one-event log body for a not-yet-acquired mission.
1819 // Only the unix symlink tests use this; silence dead_code off-unix
1820 // without masking it on unix (windows-latest clippy gates -D warnings).
1821 #[cfg_attr(not(unix), allow(dead_code))]
1822 fn one_event_line() -> String {
1823 let event = Event {
1824 seq: 1,
1825 ts: Utc::now(),
1826 mission_id: "m-1".to_string(),
1827 kind: EventKind::MissionCreated {
1828 goal: "goal".into(),
1829 base_branch: "main".into(),
1830 mission_branch: "kranz/mission-m-1".into(),
1831 config: crate::types::MissionConfig::default(),
1832 },
1833 };
1834 let mut line = serde_json::to_string(&event).unwrap();
1835 line.push('\n');
1836 line
1837 }
1838
1839 // Symlink-creating tests are unix-only, exactly like the lessons guard's
1840 // tests; Windows needs privileges to create symlinks.
1841
1842 #[cfg(unix)]
1843 #[test]
1844 fn read_events_refuses_a_symlinked_log() {
1845 use std::os::unix::fs::symlink;
1846 let dir = tempfile::tempdir().unwrap();
1847 // A valid log at the symlink TARGET: the read must refuse, not
1848 // return the target's events.
1849 let target = dir.path().join("target.jsonl");
1850 std::fs::write(&target, one_event_line()).unwrap();
1851 let link = dir.path().join("events.jsonl");
1852 symlink(&target, &link).unwrap();
1853 let err = EventLog::read_events(&link).unwrap_err();
1854 assert!(err.to_string().contains("refusing"), "{err}");
1855 }
1856
1857 #[cfg(unix)]
1858 #[test]
1859 fn acquire_refuses_symlinked_runtime_files_without_writing_through() {
1860 use std::os::unix::fs::symlink;
1861 let dir = tempfile::tempdir().unwrap();
1862 let paths = MissionPaths::new(dir.path(), "m-1");
1863 std::fs::create_dir_all(paths.mission_dir()).unwrap();
1864 let elsewhere = tempfile::tempdir().unwrap();
1865 let target = elsewhere.path().join("elsewhere.jsonl");
1866 std::fs::write(&target, b"").unwrap();
1867
1868 // A symlinked events.jsonl is refused before the lock is taken.
1869 symlink(&target, paths.events_file()).unwrap();
1870 let err = EventLog::acquire(&paths, "m-1", Duration::ZERO, LockForce::No).unwrap_err();
1871 assert!(err.to_string().contains("refusing"), "{err}");
1872 assert_eq!(std::fs::read(&target).unwrap(), b"");
1873 assert!(!paths.lock_file().exists(), "no lock taken on refusal");
1874
1875 // A symlinked lock file is refused before any steal logic.
1876 std::fs::remove_file(paths.events_file()).unwrap();
1877 symlink(&target, paths.lock_file()).unwrap();
1878 let err = EventLog::acquire(&paths, "m-1", Duration::ZERO, LockForce::No).unwrap_err();
1879 assert!(err.to_string().contains("refusing"), "{err}");
1880 assert_eq!(std::fs::read(&target).unwrap(), b"");
1881 }
1882
1883 #[cfg(unix)]
1884 #[test]
1885 fn acquired_log_retains_mission_capability_across_parent_swap() {
1886 use std::os::unix::fs::symlink;
1887 let repo = tempfile::tempdir().unwrap();
1888 let paths = MissionPaths::new(repo.path(), "m-1");
1889 let mut log = EventLog::acquire(&paths, "m-1", Duration::ZERO, LockForce::No).unwrap();
1890 let original = paths.missions_dir().join("m-original");
1891 std::fs::rename(paths.mission_dir(), &original).unwrap();
1892
1893 let outside = tempfile::tempdir().unwrap();
1894 std::fs::write(outside.path().join("events.jsonl.lock"), "outside-lock").unwrap();
1895 std::fs::write(outside.path().join("events.jsonl"), "outside-events").unwrap();
1896 symlink(outside.path(), paths.mission_dir()).unwrap();
1897
1898 log.append(EventKind::MissionPaused {}).unwrap();
1899 drop(log);
1900
1901 assert!(
1902 std::fs::read_to_string(original.join("events.jsonl"))
1903 .unwrap()
1904 .contains("mission.paused"),
1905 "the retained append handle must stay on the originally pinned mission"
1906 );
1907 assert!(
1908 !original.join("events.jsonl.lock").exists(),
1909 "drop must remove the lock relative to the retained capability"
1910 );
1911 assert_eq!(
1912 std::fs::read_to_string(outside.path().join("events.jsonl")).unwrap(),
1913 "outside-events"
1914 );
1915 assert_eq!(
1916 std::fs::read_to_string(outside.path().join("events.jsonl.lock")).unwrap(),
1917 "outside-lock"
1918 );
1919 }
1920
1921 #[test]
1922 fn lock_info_parses_all_formats() {
1923 let dir = tempfile::tempdir().unwrap();
1924 let lock = dir.path().join("l");
1925
1926 // Current three-line format: pid, acquire time, identity token.
1927 std::fs::write(&lock, "1234\n1700000000\nabcd-boot-id:5678\n").unwrap();
1928 let info = read_lock_info(&lock);
1929 assert_eq!(info.holder, "1234");
1930 assert_eq!(info.pid, Some(1234));
1931 assert_eq!(info.acquired_secs, Some(1_700_000_000));
1932 assert_eq!(info.token.as_deref(), Some("abcd-boot-id:5678"));
1933
1934 // Two-line format: token unknown, reuse screen impossible.
1935 std::fs::write(&lock, "1234\n1700000000\n").unwrap();
1936 let info = read_lock_info(&lock);
1937 assert_eq!(info.pid, Some(1234));
1938 assert_eq!(info.acquired_secs, Some(1_700_000_000));
1939 assert_eq!(info.token, None);
1940
1941 // An EMPTY third line (a platform with no token) is the same as none.
1942 std::fs::write(&lock, "1234\n1700000000\n\n").unwrap();
1943 assert_eq!(read_lock_info(&lock).token, None);
1944
1945 // Legacy one-line format: pid known, everything else unknown.
1946 std::fs::write(&lock, "1234").unwrap();
1947 let info = read_lock_info(&lock);
1948 assert_eq!(info.pid, Some(1234));
1949 assert_eq!(info.acquired_secs, None);
1950 assert_eq!(info.token, None);
1951
1952 // Garbage: nothing parseable, holder preserved for the message.
1953 std::fs::write(&lock, "not-a-pid\nnot-a-time").unwrap();
1954 let info = read_lock_info(&lock);
1955 assert_eq!(info.holder, "not-a-pid");
1956 assert_eq!(info.pid, None);
1957 assert_eq!(info.acquired_secs, None);
1958 assert_eq!(info.token, None);
1959
1960 // Non-positive pids are never probeable.
1961 std::fs::write(&lock, "-4\n1700000000").unwrap();
1962 assert_eq!(read_lock_info(&lock).pid, None);
1963 }
1964
1965 /// Two readings of a live process's identity token must be byte-identical
1966 /// — the entire reuse screen rests on this. (The old design reconstructed
1967 /// a wall-clock start time on every probe, which clock steps shifted.)
1968 #[cfg(any(target_os = "linux", target_os = "macos"))]
1969 #[test]
1970 fn identity_token_is_stable_for_a_live_process() {
1971 let pid = std::process::id() as i32;
1972 let a = process_identity_token(pid).expect("own token must be obtainable");
1973 let b = process_identity_token(pid).expect("own token must be obtainable");
1974 assert_eq!(a, b, "token readings of the same live process must match");
1975 assert!(
1976 !a.is_empty() && !a.contains('\n'),
1977 "token must be a single non-empty line: {a:?}"
1978 );
1979 }
1980
1981 /// A pid that provably has no process yields no token (dead pids have no
1982 /// /proc entry on linux and no ps row on macOS) — never a fabricated one.
1983 #[cfg(any(target_os = "linux", target_os = "macos"))]
1984 #[test]
1985 fn identity_token_of_a_dead_pid_is_none() {
1986 assert_eq!(process_identity_token(i32::MAX), None);
1987 }
1988
1989 /// Token mismatch on a lock that records OUR pid: the recorder was a
1990 /// different process whose pid the OS recycled onto us — provably Dead.
1991 /// Matching token: an ordinary double acquire — Alive.
1992 #[cfg(any(target_os = "linux", target_os = "macos"))]
1993 #[test]
1994 fn own_pid_reuse_is_decided_by_token_equality() {
1995 let pid = std::process::id() as i32;
1996 let own = process_identity_token(pid).expect("own token must be obtainable");
1997
1998 let info = LockInfo {
1999 holder: pid.to_string(),
2000 pid: Some(pid),
2001 acquired_secs: Some(0),
2002 token: Some(own.clone()),
2003 generation: None,
2004 };
2005 assert_eq!(probe_liveness(&info), LockLiveness::Alive);
2006
2007 let info = LockInfo {
2008 token: Some(format!("{own}-not")),
2009 ..info
2010 };
2011 assert_eq!(probe_liveness(&info), LockLiveness::Dead);
2012 }
2013
2014 /// Two `process_identity_token` calls for the SAME pid in quick
2015 /// succession must spawn `ps` only once — the cache should serve the
2016 /// second call from memory, and both returned tokens must still match.
2017 ///
2018 /// Uses pid 1 (launchd — always alive on macOS) rather than our own pid,
2019 /// so this test's spawn-count delta is not polluted by other tests in
2020 /// this file that concurrently probe `process_identity_token` for the
2021 /// test process's own pid.
2022 ///
2023 /// Premise-gated (ticket gate-sandbox-supervision-dogfood): reading
2024 /// launchd's token needs the setuid `/bin/ps` (unprivileged
2025 /// `proc_pidinfo` on pid 1 is EPERM — see
2026 /// [`proc_pidinfo_identity_token`]), and setuid exec is kernel-denied
2027 /// inside the gate sandbox wrap. Under a wrapped `cargo test` the raw
2028 /// `ps` seam cannot answer for pid 1, so the test skips with a
2029 /// detectable marker rather than failing on the sandbox's presence.
2030 #[cfg(target_os = "macos")]
2031 #[test]
2032 fn macos_identity_token_caches_one_ps_per_pid() {
2033 let pid = 1;
2034 if ps_identity_token(pid).is_none() {
2035 eprintln!(
2036 "SKIP-UNDER-WRAP (gate-sandbox-supervision-dogfood): \
2037 macos_identity_token_caches_one_ps_per_pid — the setuid /bin/ps cannot \
2038 execute inside the gate sandbox wrap, so pid 1's token is unreadable here; \
2039 skipping"
2040 );
2041 return;
2042 }
2043 let count_for_pid = |p: i32| {
2044 *PS_SPAWN_COUNTS
2045 .get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()))
2046 .lock()
2047 .unwrap()
2048 .get(&p)
2049 .unwrap_or(&0)
2050 };
2051 let before = count_for_pid(pid);
2052
2053 let a = process_identity_token(pid).expect("own token must be obtainable");
2054 let b = process_identity_token(pid).expect("own token must be obtainable");
2055
2056 let after = count_for_pid(pid);
2057 assert_eq!(
2058 after - before,
2059 1,
2060 "second call within the cache window must not spawn ps again"
2061 );
2062 assert_eq!(a, b, "cached token must match the freshly spawned one");
2063 }
2064
2065 /// The cache must never mask pid reuse: it only ever memoizes the
2066 /// CURRENT token lookup, never the recorded-vs-current comparison. Even
2067 /// though `current` is served from cache here, a differing `recorded`
2068 /// token must still yield Dead.
2069 #[cfg(target_os = "macos")]
2070 #[test]
2071 fn macos_cache_never_masks_pid_reuse() {
2072 let pid = std::process::id() as i32;
2073 // Prime the cache for this pid.
2074 let own = process_identity_token(pid).expect("own token must be obtainable");
2075
2076 let info = LockInfo {
2077 holder: pid.to_string(),
2078 pid: Some(pid),
2079 acquired_secs: Some(0),
2080 token: Some(format!("{own}-not")),
2081 generation: None,
2082 };
2083 // `current` comes from the cache primed above, but the differing
2084 // `recorded` token must still be judged Dead — the comparison is
2085 // never skipped just because `current` was cached.
2086 assert_eq!(probe_liveness(&info), LockLiveness::Dead);
2087 }
2088
2089 /// A writer that succeeds for the first `fail_at` writes and then always
2090 /// errors, recording every line it actually wrote.
2091 struct FlakyWriter {
2092 fail_at: usize,
2093 writes: Vec<String>,
2094 }
2095
2096 impl std::io::Write for FlakyWriter {
2097 fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
2098 if self.writes.len() >= self.fail_at {
2099 return Err(std::io::Error::other("simulated write failure"));
2100 }
2101 self.writes.push(String::from_utf8_lossy(buf).into_owned());
2102 Ok(buf.len())
2103 }
2104
2105 fn flush(&mut self) -> std::io::Result<()> {
2106 Ok(())
2107 }
2108 }
2109
2110 fn buffered(n: usize) -> Vec<BufferedLine> {
2111 (0..n)
2112 .map(|i| BufferedLine {
2113 buffered_at: Instant::now(),
2114 line: format!("line-{i}\n"),
2115 })
2116 .collect()
2117 }
2118
2119 #[test]
2120 fn drain_retains_unwritten_deltas_on_write_failure() {
2121 let k = 3;
2122 let n = 7;
2123 let mut writer = FlakyWriter {
2124 fail_at: k,
2125 writes: Vec::new(),
2126 };
2127 let mut buffer = buffered(n);
2128
2129 let result = drain_lines(&mut writer, &mut buffer);
2130
2131 assert!(result.is_err(), "drain must surface the write error");
2132 assert_eq!(
2133 writer.writes,
2134 (0..k).map(|i| format!("line-{i}\n")).collect::<Vec<_>>(),
2135 "exactly the first k lines must have been written, in order"
2136 );
2137 assert_eq!(
2138 buffer.iter().map(|b| b.line.clone()).collect::<Vec<_>>(),
2139 (k..n).map(|i| format!("line-{i}\n")).collect::<Vec<_>>(),
2140 "the remaining lines, including the one that failed, must stay buffered in order"
2141 );
2142
2143 // A subsequent drain with a working writer must recover the retained
2144 // lines successfully — nothing is permanently lost.
2145 let mut retry_writer = FlakyWriter {
2146 fail_at: usize::MAX,
2147 writes: Vec::new(),
2148 };
2149 let retry_result = drain_lines(&mut retry_writer, &mut buffer);
2150 assert!(retry_result.is_ok());
2151 assert!(buffer.is_empty());
2152 assert_eq!(
2153 retry_writer.writes,
2154 (k..n).map(|i| format!("line-{i}\n")).collect::<Vec<_>>()
2155 );
2156 }
2157
2158 /// Build a raw JSONL line for a `mission.paused` event with the given
2159 /// `seq`/`mission_id` — enough to exercise seq continuity and mission-id
2160 /// consistency without pulling in the full Event field set.
2161 fn event_line(seq: u64, mission_id: &str) -> String {
2162 let event = Event {
2163 seq,
2164 ts: Utc::now(),
2165 mission_id: mission_id.to_string(),
2166 kind: EventKind::MissionPaused {},
2167 };
2168 let mut line = serde_json::to_string(&event).unwrap();
2169 line.push('\n');
2170 line
2171 }
2172
2173 #[test]
2174 fn acquire_rejects_foreign_mission_id_in_later_event() {
2175 let dir = tempfile::tempdir().unwrap();
2176 let paths = MissionPaths::new(dir.path(), "m-a");
2177 std::fs::create_dir_all(paths.mission_dir()).unwrap();
2178 let events_path = paths.events_file();
2179
2180 // Only defect: seq 2's mission_id differs from seq 1's.
2181 std::fs::write(
2182 &events_path,
2183 format!("{}{}", event_line(1, "m-a"), event_line(2, "m-b")),
2184 )
2185 .unwrap();
2186
2187 let err = EventLog::acquire(&paths, "m-a", Duration::from_secs(1), LockForce::No)
2188 .expect_err("mixed mission_id log must be rejected");
2189 assert!(
2190 matches!(err, EngineError::LogCorruption(_)),
2191 "expected LogCorruption, got {err:?}"
2192 );
2193
2194 // Control: identical seqs, single consistent mission_id, acquires cleanly.
2195 let dir2 = tempfile::tempdir().unwrap();
2196 let paths2 = MissionPaths::new(dir2.path(), "m-a");
2197 std::fs::create_dir_all(paths2.mission_dir()).unwrap();
2198 std::fs::write(
2199 paths2.events_file(),
2200 format!("{}{}", event_line(1, "m-a"), event_line(2, "m-a")),
2201 )
2202 .unwrap();
2203 let log = EventLog::acquire(&paths2, "m-a", Duration::from_secs(1), LockForce::No)
2204 .expect("consistent-mission log must acquire cleanly");
2205 assert_eq!(log.last_seq(), 2);
2206 }
2207
2208 #[test]
2209 fn parse_log_rejects_mixed_mission_ids() {
2210 let dir = tempfile::tempdir().unwrap();
2211 let path = dir.path().join("events.jsonl");
2212 std::fs::write(
2213 &path,
2214 format!("{}{}", event_line(1, "m-a"), event_line(2, "m-b")),
2215 )
2216 .unwrap();
2217
2218 let err = EventLog::read_events(&path).expect_err("mixed mission_id must be rejected");
2219 assert!(
2220 matches!(err, EngineError::LogCorruption(_)),
2221 "expected LogCorruption, got {err:?}"
2222 );
2223 }
2224
2225 #[test]
2226 fn acquire_adopts_empty_preexisting_log_as_fresh() {
2227 let dir = tempfile::tempdir().unwrap();
2228 let paths = MissionPaths::new(dir.path(), "m-a");
2229 std::fs::create_dir_all(paths.mission_dir()).unwrap();
2230 std::fs::write(paths.events_file(), "").unwrap();
2231
2232 let log = EventLog::acquire(&paths, "m-a", Duration::from_secs(1), LockForce::No)
2233 .expect("empty pre-existing log must be adopted as fresh");
2234 assert_eq!(log.last_seq(), 0);
2235 let appended = {
2236 let mut log = log;
2237 log.append(EventKind::MissionPaused {}).unwrap()
2238 };
2239 assert_eq!(appended.seq, 1);
2240 }
2241}