Skip to main content

car_server_core/
run_store.rs

1//! Disk-backed run-trace store (agent run tracing, U3).
2//!
3//! Persists the per-run trace stream — the `RunStarted` line, each
4//! `RunTurn`, then the terminal `RunEnded`/`Incomplete` — as JSONL so a
5//! run survives daemon restarts and stays grouped by run independent of
6//! the WS connection that produced it (R4). One file per run:
7//!
8//! ```text
9//! ~/.car/runs/{agent_id}/{run_id}.jsonl
10//! ```
11//!
12//! ## Source-of-truth split
13//!
14//! This disk store is the source of truth for **replay** (U5):
15//! [`RunStore::get_run_trace_page_for`] / [`RunStore::list_runs_page`] read it
16//! and work after a restart when memory is empty. The in-memory
17//! `RunMeta.turns` buffer (U2) stays the source for the **live** stream
18//! (U4) — U3 does not remove it; it mirrors what was recorded onto disk.
19//!
20//! ## Layout, perms, backup exclusion (R14)
21//!
22//! Prompts and CLI output may carry secrets, so the `runs/` tree is
23//! created `0700` and every file `0600` (Unix). The `runs/` dir is
24//! marked backup-excluded — a `.nobackup` marker file plus, on macOS,
25//! the `com.apple.metadata:com_apple_backup_excludeItem` xattr — so Time
26//! Machine / iCloud don't silently copy plaintext traces off the box.
27//!
28//! ## Index
29//!
30//! Each agent has a fixed-record newest-first summary index plus one small
31//! summary sidecar per run. `runs.list` seeks directly to its requested page;
32//! it never scans every run file. The JSONL trace remains authoritative and
33//! startup GC rebuilds the derived index/sidecars so interrupted updates are
34//! repaired before the daemon listens. `run_id -> agent_id` resolves by
35//! scanning `runs/*/` for the matching `{run_id}.jsonl` (U5's
36//! `runs.get_trace` takes only a `run_id`).
37//!
38//! ## Retention (R6)
39//!
40//! GC runs on daemon boot ([`RunStore::gc`]): per agent it keeps the **50
41//! most recent completed runs** and drops anything older than **30 days**,
42//! whichever is more restrictive. A still-in-progress run (no terminal
43//! record) is **never** evicted. Both limits are configurable via
44//! `~/.car/config.toml` (`[runs] max_per_agent` / `max_age_days`) with
45//! that restrictive default. **`0` disables a cap** — see
46//! [`RetentionConfig`].
47//!
48//! Records are appended at turn granularity (not per token) — the same
49//! coarse boundary the in-memory buffer uses. A corrupt/partial trailing
50//! JSONL line loads the prior valid records rather than failing the whole
51//! run (the error-path test).
52
53use car_ir::{ActionProposal, ProposalLineageStatus, ProposalResult};
54use car_proto::{RunRecord, RunTermination};
55use chrono::{DateTime, Utc};
56use serde::{Deserialize, Serialize};
57use serde_json::Value;
58use sha2::{Digest, Sha256};
59use std::collections::{HashMap, VecDeque};
60use std::fs::File;
61use std::io::{BufRead, Read, Seek, SeekFrom, Write};
62use std::path::{Path, PathBuf};
63use std::sync::{Arc, Condvar, Mutex, Weak};
64
65#[cfg(not(target_os = "windows"))]
66fn sync_directory(path: impl AsRef<Path>) -> std::io::Result<()> {
67    File::open(path)?.sync_all()
68}
69
70#[cfg(target_os = "windows")]
71fn sync_directory(_path: impl AsRef<Path>) -> std::io::Result<()> {
72    // Windows FlushFileBuffers rejects directory handles with
73    // ERROR_ACCESS_DENIED. File contents are synchronized before each call;
74    // there is no supported directory-fsync equivalent to add here.
75    Ok(())
76}
77
78const COMPLETED_PROPOSAL_OWNER_INDEX_MIGRATION_VERSION: u32 = 2;
79const RUN_SUMMARY_KEY_BYTES: usize = 32;
80const RUN_SUMMARY_INDEX_RECORD_BYTES: usize = 8 + RUN_SUMMARY_KEY_BYTES;
81const RUN_SUMMARY_INDEX_FILE: &str = ".run-summary-index-v2";
82const RUN_SUMMARY_SIDECAR_DIR: &str = ".run-summaries-v2";
83const RUN_TRACE_CORRUPTION_SIDECAR_DIR: &str = ".run-trace-corruption-v1";
84#[cfg(not(test))]
85const MAX_RESUMED_PROPOSAL_RECEIPTS_PER_RUN: usize = 1024;
86#[cfg(test)]
87const MAX_RESUMED_PROPOSAL_RECEIPTS_PER_RUN: usize = 4;
88#[cfg(not(test))]
89const MAX_RESUMED_PROPOSAL_RECEIPT_BYTES_PER_RUN: u64 = 16 * 1024 * 1024;
90#[cfg(test)]
91const MAX_RESUMED_PROPOSAL_RECEIPT_BYTES_PER_RUN: u64 = 64 * 1024;
92
93#[derive(Debug, Serialize, Deserialize)]
94struct CompletedProposalOwnerIndexMigration {
95    version: u32,
96}
97
98/// Default per-agent cap: keep the 50 most recent completed runs.
99pub const DEFAULT_MAX_RUNS_PER_AGENT: usize = 50;
100/// Default age cap: drop completed runs older than 30 days.
101pub const DEFAULT_MAX_AGE_DAYS: i64 = 30;
102
103/// Terminal/in-progress status of a run, derived from its records.
104///
105/// Distinct from the run-level `OutcomeStatus` carried inside a terminal
106/// `Outcome` — this is the coarse "what state is this run in?" the run
107/// list renders. `Incomplete` is the orphan case (no terminal record + no
108/// live harness, R5); `InProgress` is a run still being written.
109#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
110#[serde(rename_all = "snake_case")]
111pub enum RunStatus {
112    /// No terminal record yet — still being written by a live harness.
113    InProgress,
114    /// `runs.complete` reported a terminal `AgentOutcome`. The concrete
115    /// `OutcomeStatus` lives on the `RunEnded` record itself.
116    Completed,
117    /// CAR confirmed controlled work stopped and committed cancellation.
118    Cancelled,
119    /// The harness disconnected without reporting an outcome (R5).
120    Incomplete,
121    /// A durable cancellation request exists without a terminal boundary.
122    CancellationPending,
123}
124
125#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
126#[serde(rename_all = "snake_case")]
127pub enum RunTraceCorruptionKind {
128    MalformedRecord,
129}
130
131#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
132pub struct RunTraceCorruption {
133    pub kind: RunTraceCorruptionKind,
134    /// One-based JSONL line number.
135    pub line: usize,
136}
137
138#[derive(Debug, Clone, Serialize, Deserialize)]
139struct RunTraceCorruptionMarker {
140    run_id: String,
141    agent_id: String,
142    corruption: RunTraceCorruption,
143}
144
145/// One row of [`RunStore::list_runs`] — the summary U5's `runs.list`
146/// returns. Persisted in a sidecar so bounded listing never loads every turn.
147#[derive(Debug, Clone, Serialize, Deserialize)]
148pub struct RunSummary {
149    pub run_id: String,
150    pub agent_id: String,
151    pub intent: String,
152    pub started_at: DateTime<Utc>,
153    /// When the terminal record was written, if any.
154    #[serde(default, skip_serializing_if = "Option::is_none")]
155    pub ended_at: Option<DateTime<Utc>>,
156    pub status: RunStatus,
157    /// Number of `RunTurn` records persisted for this run.
158    pub turn_count: usize,
159    /// Immutable per-agent keyset used by bounded newest-first pagination.
160    /// Zero is reserved for legacy sidecars that startup has not migrated yet.
161    #[serde(default)]
162    pub sequence: u64,
163    /// Present only when a durable newline-terminated record is malformed.
164    #[serde(default, skip_serializing_if = "Option::is_none")]
165    pub trace_corruption: Option<RunTraceCorruption>,
166}
167
168/// Durable preimage for a proposal whose actions already finished but whose
169/// `proposal_completed` journal boundary has not yet been acknowledged.
170///
171/// The payload is deliberately server-owned instead of a `car_proto` run row:
172/// it is an outbox transaction, not part of the public run trace. Keeping the
173/// exact normal-serde result and its JCS digest lets a retry or daemon restart
174/// finish the terminal without dispatching an action twice.
175#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
176pub struct PendingProposalFinalization {
177    pub run_id: String,
178    pub client_id: String,
179    /// Exact caller-selected policy context. This is kept separately from
180    /// `policy_session_id`: an unknown session is deliberately not trusted or
181    /// journaled, but an exact retry must still present the same context.
182    #[serde(default, skip_serializing_if = "Option::is_none")]
183    pub requested_policy_session_id: Option<String>,
184    #[serde(default, skip_serializing_if = "Option::is_none")]
185    pub policy_session_id: Option<String>,
186    pub original_proposal_id: String,
187    pub final_proposal_id: String,
188    /// Exact caller-supplied JSON before serde fills defaults. This recognizes
189    /// an exact retry even when an omitted timestamp is regenerated on decode.
190    pub original_submission: Value,
191    /// Exact normal-serde proposal CAR accepted and journaled.
192    pub original_proposal: ActionProposal,
193    /// Exact normal-serde proposal selected by the runtime. This duplicates
194    /// the active-v3 `ProposalResult.final_proposal` intentionally: validator
195    /// equality makes an outbox edit or partial producer upgrade fail closed.
196    pub final_proposal: ActionProposal,
197    /// Generation-keyed normal-serde preimages for every accepted lineage
198    /// entry, including generation zero when it was admitted.
199    #[serde(default)]
200    pub accepted_proposal_preimages: Vec<AcceptedProposalPreimage>,
201    #[serde(deserialize_with = "deserialize_active_proposal_result")]
202    pub proposal_result: ProposalResult,
203    pub result_digest: String,
204}
205
206/// Durable exact response retained after `proposal_completed` is fsynced.
207///
208/// Unlike the finalization outbox, this receipt intentionally survives guard
209/// cleanup. A same-bound-session retry can therefore recover the typed result
210/// without dispatch even if the original JSON-RPC response was lost, guard
211/// unlink/fsync failed, or run-trace retention later removed `RunStarted`.
212#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
213pub struct CompletedProposalResponse {
214    pub finalization: PendingProposalFinalization,
215}
216
217/// Content-addressed ownership claim for the retry tuple used before a live
218/// run is selected. The claim is deliberately independent of the run trace:
219/// completed responses outlive trace retention, and an ownership lookup must
220/// read at most this one exact file rather than enumerate every receipt.
221#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
222struct CompletedProposalOwnership {
223    run_id: String,
224    client_id: String,
225    #[serde(default, skip_serializing_if = "Option::is_none")]
226    requested_policy_session_id: Option<String>,
227    original_submission: Value,
228}
229
230impl CompletedProposalOwnership {
231    fn new(
232        run_id: &str,
233        client_id: &str,
234        requested_policy_session_id: Option<&str>,
235        original_submission: &Value,
236    ) -> Self {
237        Self {
238            run_id: run_id.to_string(),
239            client_id: client_id.to_string(),
240            requested_policy_session_id: requested_policy_session_id.map(str::to_string),
241            original_submission: original_submission.clone(),
242        }
243    }
244}
245
246/// Result of atomically reserving one proposal retry tuple before dispatch.
247#[derive(Debug, Clone, PartialEq, Eq)]
248pub enum ProposalRetryReservation {
249    Acquired,
250    Existing { run_id: String, client_id: String },
251}
252
253/// Durable rollback authority written before a retry owner is reserved.
254///
255/// If the daemon exits before the execution marker is durable, startup uses
256/// this exact preimage to release the owner. Once a marker exists, the marker
257/// remains the conservative outcome-unknown guard and startup only removes the
258/// stale rollback intent.
259#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
260pub struct ProposalRetryRollback {
261    pub run_id: String,
262    pub client_id: String,
263    #[serde(default, skip_serializing_if = "Option::is_none")]
264    pub requested_policy_session_id: Option<String>,
265    pub original_submission: Value,
266}
267
268impl ProposalRetryRollback {
269    fn validate(&self) -> std::io::Result<()> {
270        if self.run_id.is_empty()
271            || self.client_id.is_empty()
272            || self.requested_policy_session_id.as_deref() == Some("")
273        {
274            return Err(std::io::Error::new(
275                std::io::ErrorKind::InvalidInput,
276                "proposal retry rollback requires non-empty run/client/policy identities",
277            ));
278        }
279        Ok(())
280    }
281}
282
283#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
284struct ProposalIdClaim {
285    run_id: String,
286    client_id: String,
287    proposal_id: String,
288    original_submission: Value,
289}
290
291#[derive(Debug, Clone, Copy, PartialEq, Eq)]
292pub enum ProposalIdClaimOutcome {
293    Acquired,
294    ExistingExact,
295}
296
297impl CompletedProposalResponse {
298    pub fn proposal_result(&self) -> &ProposalResult {
299        &self.finalization.proposal_result
300    }
301
302    fn validate(&self) -> std::io::Result<()> {
303        self.finalization.validate()
304    }
305
306    fn execution_marker(&self) -> std::io::Result<ProposalExecutionMarker> {
307        let pending = &self.finalization;
308        let original_proposal = serde_json::to_value(&pending.original_proposal)
309            .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
310        let marker = ProposalExecutionMarker {
311            run_id: pending.run_id.clone(),
312            client_id: pending.client_id.clone(),
313            requested_policy_session_id: pending.requested_policy_session_id.clone(),
314            policy_session_id: pending.policy_session_id.clone(),
315            original_proposal_id: pending.original_proposal_id.clone(),
316            original_submission: pending.original_submission.clone(),
317            original_proposal,
318            proposal_digest: proposal_digest(&pending.original_proposal)
319                .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?,
320        };
321        marker.validate()?;
322        Ok(marker)
323    }
324}
325
326#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
327pub struct AcceptedProposalPreimage {
328    pub generation: u32,
329    pub proposal: ActionProposal,
330}
331
332fn deserialize_active_proposal_result<'de, D>(deserializer: D) -> Result<ProposalResult, D::Error>
333where
334    D: serde::Deserializer<'de>,
335{
336    let value = Value::deserialize(deserializer)?;
337    let object = value
338        .as_object()
339        .ok_or_else(|| serde::de::Error::custom("proposal_result must be an object"))?;
340    for field in [
341        "proposal_id",
342        "original_proposal_id",
343        "final_proposal",
344        "replan_lineage",
345        "results",
346        "cost",
347    ] {
348        if !object.contains_key(field) {
349            return Err(serde::de::Error::custom(format!(
350                "active v3 proposal_result is missing `{field}`"
351            )));
352        }
353    }
354    serde_json::from_value(value).map_err(serde::de::Error::custom)
355}
356
357fn is_lowercase_sha256(value: &str) -> bool {
358    value.len() == 64
359        && value
360            .bytes()
361            .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
362}
363
364fn proposal_digest(proposal: &ActionProposal) -> Result<String, String> {
365    let value = serde_json::to_value(proposal).map_err(|error| error.to_string())?;
366    let canonical = car_inference::catalog_identity::canonical_json(&value)?;
367    Ok(format!("{:x}", Sha256::digest(canonical.as_bytes())))
368}
369
370fn validate_original_submission(
371    submission: &Value,
372    proposal: &ActionProposal,
373) -> std::io::Result<()> {
374    let invalid = |message: String| std::io::Error::new(std::io::ErrorKind::InvalidData, message);
375    let object = submission.as_object().ok_or_else(|| {
376        invalid("active proposal original submission must be an object".to_string())
377    })?;
378    if object.get("id").and_then(Value::as_str) != Some(proposal.id.as_str()) {
379        return Err(invalid(
380            "active proposal raw submission requires an exact string id".to_string(),
381        ));
382    }
383    let raw_actions = object.get("actions").ok_or_else(|| {
384        invalid("active proposal raw submission is missing its action array".to_string())
385    })?;
386    let actions: Vec<car_ir::Action> = serde_json::from_value(raw_actions.clone())
387        .map_err(|error| invalid(format!("raw submission actions are invalid: {error}")))?;
388    if actions != proposal.actions {
389        return Err(invalid(
390            "raw submission actions do not match the accepted normal-serde proposal".to_string(),
391        ));
392    }
393    if object
394        .get("source")
395        .is_some_and(|source| source.as_str() != Some(proposal.source.as_str()))
396    {
397        return Err(invalid(
398            "raw submission source does not match the accepted proposal".to_string(),
399        ));
400    }
401    if let Some(context) = object.get("context") {
402        let context: HashMap<String, Value> = serde_json::from_value(context.clone())
403            .map_err(|error| invalid(format!("raw submission context is invalid: {error}")))?;
404        if context != proposal.context {
405            return Err(invalid(
406                "raw submission context does not match the accepted proposal".to_string(),
407            ));
408        }
409    }
410    if let Some(timestamp) = object.get("timestamp") {
411        let timestamp: DateTime<Utc> = serde_json::from_value(timestamp.clone())
412            .map_err(|error| invalid(format!("raw submission timestamp is invalid: {error}")))?;
413        if timestamp != proposal.timestamp {
414            return Err(invalid(
415                "raw submission timestamp does not match the accepted proposal".to_string(),
416            ));
417        }
418    }
419    Ok(())
420}
421
422/// Fsynced before runtime dispatch. If CAR restarts with this marker and no
423/// exact finalization payload, the action outcome is unknown: the server must
424/// quarantine the run instead of guessing that redispatch is safe.
425#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
426pub struct ProposalExecutionMarker {
427    pub run_id: String,
428    pub client_id: String,
429    #[serde(default, skip_serializing_if = "Option::is_none")]
430    pub requested_policy_session_id: Option<String>,
431    #[serde(default, skip_serializing_if = "Option::is_none")]
432    pub policy_session_id: Option<String>,
433    pub original_proposal_id: String,
434    pub original_submission: Value,
435    pub original_proposal: Value,
436    pub proposal_digest: String,
437}
438
439impl ProposalExecutionMarker {
440    pub fn validate(&self) -> std::io::Result<()> {
441        let invalid =
442            |message: String| std::io::Error::new(std::io::ErrorKind::InvalidData, message);
443        if self.run_id.is_empty()
444            || self.client_id.is_empty()
445            || self.original_proposal_id.is_empty()
446        {
447            return Err(invalid(
448                "proposal execution marker requires non-empty run/client/proposal identities"
449                    .to_string(),
450            ));
451        }
452        if self.policy_session_id.is_some()
453            && self.policy_session_id != self.requested_policy_session_id
454        {
455            return Err(invalid(
456                "authenticated policy session is not the caller-requested policy session"
457                    .to_string(),
458            ));
459        }
460        let proposal: ActionProposal = serde_json::from_value(self.original_proposal.clone())
461            .map_err(|error| invalid(format!("execution marker proposal is invalid: {error}")))?;
462        if proposal.id != self.original_proposal_id {
463            return Err(invalid(
464                "execution marker proposal id does not match original proposal identity"
465                    .to_string(),
466            ));
467        }
468        let digest = proposal_digest(&proposal)
469            .map_err(|error| invalid(format!("execution marker proposal JCS failed: {error}")))?;
470        if self.proposal_digest != digest || !is_lowercase_sha256(&self.proposal_digest) {
471            return Err(invalid(
472                "execution marker digest does not bind its exact proposal preimage".to_string(),
473            ));
474        }
475        validate_original_submission(&self.original_submission, &proposal)
476    }
477}
478
479impl PendingProposalFinalization {
480    pub fn validate(&self) -> std::io::Result<()> {
481        let invalid =
482            |message: String| std::io::Error::new(std::io::ErrorKind::InvalidData, message);
483        if self.run_id.is_empty()
484            || self.client_id.is_empty()
485            || self.original_proposal_id.is_empty()
486            || self.final_proposal_id.is_empty()
487        {
488            return Err(invalid(
489                "proposal finalization requires non-empty run/client/proposal identities"
490                    .to_string(),
491            ));
492        }
493        if self.policy_session_id.is_some()
494            && self.policy_session_id != self.requested_policy_session_id
495        {
496            return Err(invalid(
497                "authenticated pending policy session is not the requested policy session"
498                    .to_string(),
499            ));
500        }
501        if self.original_proposal.id != self.original_proposal_id
502            || self.proposal_result.original_proposal_id != self.original_proposal_id
503        {
504            return Err(invalid(
505                "proposal finalization original proposal identities do not match".to_string(),
506            ));
507        }
508        if self.final_proposal.id != self.final_proposal_id
509            || self.proposal_result.proposal_id != self.final_proposal_id
510            || self.proposal_result.final_proposal.as_ref() != Some(&self.final_proposal)
511        {
512            return Err(invalid(
513                "active v3 proposal finalization final proposal preimage/id do not match"
514                    .to_string(),
515            ));
516        }
517        validate_original_submission(&self.original_submission, &self.original_proposal)?;
518
519        let result_value = serde_json::to_value(&self.proposal_result)
520            .map_err(|error| invalid(format!("proposal result serialization failed: {error}")))?;
521        let canonical = car_inference::catalog_identity::canonical_json(&result_value)
522            .map_err(|error| invalid(format!("proposal result JCS failed: {error}")))?;
523        let result_digest = format!("{:x}", Sha256::digest(canonical.as_bytes()));
524        if self.result_digest != result_digest || !is_lowercase_sha256(&self.result_digest) {
525            return Err(invalid(
526                "proposal finalization result digest does not match typed result".to_string(),
527            ));
528        }
529
530        let lineage = &self.proposal_result.replan_lineage;
531        if lineage.is_empty() {
532            return Err(invalid(
533                "active v3 proposal result is missing generation-zero lineage".to_string(),
534            ));
535        }
536        let original_digest = proposal_digest(&self.original_proposal)
537            .map_err(|error| invalid(format!("original proposal JCS failed: {error}")))?;
538        if lineage[0].generation != 0
539            || lineage[0].proposal_id != self.original_proposal_id
540            || lineage[0].proposal_digest.as_deref() != Some(original_digest.as_str())
541        {
542            return Err(invalid(
543                "generation-zero lineage does not bind the exact original proposal preimage"
544                    .to_string(),
545            ));
546        }
547        if lineage[0].status == ProposalLineageStatus::Rejected && lineage.len() != 1 {
548            return Err(invalid(
549                "a generation-zero rejection cannot be followed by another generation".to_string(),
550            ));
551        }
552
553        let mut accepted_by_generation = HashMap::new();
554        for accepted in &self.accepted_proposal_preimages {
555            if accepted_by_generation
556                .insert(accepted.generation, &accepted.proposal)
557                .is_some()
558            {
559                return Err(invalid(format!(
560                    "duplicate accepted proposal preimage for generation {}",
561                    accepted.generation
562                )));
563            }
564        }
565        for (index, entry) in lineage.iter().enumerate() {
566            if entry.generation != u32::try_from(index).unwrap_or(u32::MAX) {
567                return Err(invalid(
568                    "proposal lineage generations are not contiguous from zero".to_string(),
569                ));
570            }
571            match entry.status {
572                ProposalLineageStatus::Accepted => {
573                    if entry.rejection_reason.is_some() {
574                        return Err(invalid(format!(
575                            "accepted lineage generation {} cannot carry a rejection reason",
576                            entry.generation
577                        )));
578                    }
579                    let proposal =
580                        accepted_by_generation
581                            .get(&entry.generation)
582                            .ok_or_else(|| {
583                                invalid(format!(
584                            "accepted lineage generation {} is missing its exact proposal preimage",
585                            entry.generation
586                        ))
587                            })?;
588                    let digest = proposal_digest(proposal).map_err(|error| {
589                        invalid(format!(
590                            "accepted proposal generation {} JCS failed: {error}",
591                            entry.generation
592                        ))
593                    })?;
594                    if proposal.id != entry.proposal_id
595                        || entry.proposal_digest.as_deref() != Some(digest.as_str())
596                        || !is_lowercase_sha256(&digest)
597                    {
598                        return Err(invalid(format!(
599                            "accepted lineage generation {} does not bind its exact lowercase-JCS preimage",
600                            entry.generation
601                        )));
602                    }
603                }
604                ProposalLineageStatus::Rejected => {
605                    if entry
606                        .rejection_reason
607                        .as_deref()
608                        .is_none_or(|reason| reason.trim().is_empty())
609                    {
610                        return Err(invalid(format!(
611                            "rejected lineage generation {} is missing its exact rejection reason",
612                            entry.generation
613                        )));
614                    }
615                    if entry
616                        .proposal_digest
617                        .as_deref()
618                        .is_some_and(|digest| !is_lowercase_sha256(digest))
619                    {
620                        return Err(invalid(format!(
621                            "rejected lineage generation {} has an invalid proposal digest",
622                            entry.generation
623                        )));
624                    }
625                    if accepted_by_generation.contains_key(&entry.generation) {
626                        return Err(invalid(format!(
627                            "rejected lineage generation {} cannot carry an accepted preimage",
628                            entry.generation
629                        )));
630                    }
631                }
632            }
633        }
634        if accepted_by_generation.len()
635            != lineage
636                .iter()
637                .filter(|entry| entry.status == ProposalLineageStatus::Accepted)
638                .count()
639        {
640            return Err(invalid(
641                "proposal finalization contains an unbound accepted proposal preimage".to_string(),
642            ));
643        }
644
645        if let Some(last_accepted) = lineage
646            .iter()
647            .rev()
648            .find(|entry| entry.status == ProposalLineageStatus::Accepted)
649        {
650            let final_preimage = accepted_by_generation
651                .get(&last_accepted.generation)
652                .expect("accepted lineage was validated above");
653            if *final_preimage != &self.final_proposal {
654                return Err(invalid(
655                    "most recent accepted proposal is not the active v3 final proposal".to_string(),
656                ));
657            }
658        } else if lineage.len() != 1
659            || lineage[0].status != ProposalLineageStatus::Rejected
660            || self.final_proposal != self.original_proposal
661        {
662            return Err(invalid(
663                "a result without an accepted generation must be a generation-zero rejection"
664                    .to_string(),
665            ));
666        }
667
668        let final_action_ids: std::collections::HashSet<_> = self
669            .final_proposal
670            .actions
671            .iter()
672            .map(|action| action.id.as_str())
673            .collect();
674        let result_action_ids: std::collections::HashSet<_> = self
675            .proposal_result
676            .results
677            .iter()
678            .map(|result| result.action_id.as_str())
679            .collect();
680        if final_action_ids.len() != self.final_proposal.actions.len()
681            || result_action_ids.len() != self.proposal_result.results.len()
682            || final_action_ids != result_action_ids
683        {
684            return Err(invalid(
685                "typed proposal results do not match the final proposal action identities"
686                    .to_string(),
687            ));
688        }
689        Ok(())
690    }
691
692    fn validate_provenance(
693        &self,
694        started: &car_proto::RunStarted,
695        marker: &ProposalExecutionMarker,
696    ) -> std::io::Result<()> {
697        let invalid =
698            |message: String| std::io::Error::new(std::io::ErrorKind::InvalidData, message);
699        let started_client = started.client_id.as_deref().ok_or_else(|| {
700            invalid("durable RunStarted is missing its client identity".to_string())
701        })?;
702        if started.run_id != self.run_id || started_client != self.client_id {
703            return Err(invalid(
704                "pending run/client identity does not match durable RunStarted".to_string(),
705            ));
706        }
707        let original_proposal = serde_json::to_value(&self.original_proposal).map_err(|error| {
708            invalid(format!(
709                "pending original proposal serialization failed: {error}"
710            ))
711        })?;
712        let original_digest = proposal_digest(&self.original_proposal)
713            .map_err(|error| invalid(format!("pending original proposal JCS failed: {error}")))?;
714        if marker.run_id != self.run_id
715            || marker.client_id != self.client_id
716            || marker.requested_policy_session_id != self.requested_policy_session_id
717            || marker.policy_session_id != self.policy_session_id
718            || marker.original_proposal_id != self.original_proposal_id
719            || marker.original_submission != self.original_submission
720            || marker.original_proposal != original_proposal
721            || marker.proposal_digest != original_digest
722        {
723            return Err(invalid(
724                "pending finalization identity does not match durable execution marker".to_string(),
725            ));
726        }
727        Ok(())
728    }
729
730    pub fn event_data(&self) -> HashMap<String, Value> {
731        HashMap::from([
732            (
733                "original_submission_id".to_string(),
734                Value::from(self.original_proposal_id.clone()),
735            ),
736            (
737                "final_proposal_id".to_string(),
738                Value::from(self.final_proposal_id.clone()),
739            ),
740            (
741                "original_proposal".to_string(),
742                serde_json::to_value(&self.original_proposal).unwrap_or(Value::Null),
743            ),
744            (
745                "final_proposal".to_string(),
746                serde_json::to_value(&self.final_proposal).unwrap_or(Value::Null),
747            ),
748            (
749                "original_submission".to_string(),
750                self.original_submission.clone(),
751            ),
752            (
753                "replan_lineage".to_string(),
754                serde_json::to_value(&self.proposal_result.replan_lineage).unwrap_or(Value::Null),
755            ),
756            (
757                "all_succeeded".to_string(),
758                Value::from(self.proposal_result.all_succeeded()),
759            ),
760            (
761                "action_count".to_string(),
762                Value::from(self.proposal_result.results.len()),
763            ),
764            (
765                "proposal_result".to_string(),
766                serde_json::to_value(&self.proposal_result).unwrap_or(Value::Null),
767            ),
768            (
769                "result_digest".to_string(),
770                Value::from(self.result_digest.clone()),
771            ),
772        ])
773    }
774}
775
776/// Retention policy for the run store (R6). Defaults are the restrictive
777/// 50-per-agent / 30-day caps; `~/.car/config.toml [runs]` overrides.
778///
779/// **A non-positive cap is DISABLED, not zero-tolerance** (car#1338). This used
780/// to be read literally, and the literal reading is destructive in the one
781/// direction nobody intends: `max_per_agent = 0` made `completed_rank >= 0`
782/// always true and `max_age_days = 0` put the cutoff at now, so either value
783/// deleted every collectable run trace on the next boot, unrecoverably.
784///
785/// An operator writing `0` means "no cap" — that is what it means for
786/// `max_session_wall_secs`, and for the coder's own `max_sessions` /
787/// `max_session_age_days` in the neighbouring `~/.car/coder.toml`. Two
788/// similarly-named retention knobs in one daemon whose zero values inverted was
789/// a hazard whose failure mode was silent data loss, so the two now agree.
790///
791/// This is a choice, not an unqualified convention: `CoderConfig`'s
792/// `max_check_timeout_secs` and `approval_patch_bytes` read `0` as UNSET and
793/// fall back to their defaults, because zero would floor a check at one second
794/// or blind the approval surface. That treatment is the other candidate fix for
795/// this bug and it was rejected on the merits — "unset" restores 50/30, which
796/// still deletes, so it does not close the data loss. `0` is a legal value with
797/// an ambiguous meaning and this defines it; a value that could not have been
798/// intended (a negative or unrepresentable age) is a different case and is
799/// warned about at load.
800/// The fields are PRIVATE and the caps are read as `Option`s, so "disabled" is
801/// a state this type carries rather than a sentinel every call site has to
802/// remember. That is not decoration: the bug was a bare `rank >= cap`
803/// comparison, and leaving the raw values public would let the next one be
804/// written with no compiler complaint.
805#[derive(Debug, Clone, Copy)]
806pub struct RetentionConfig {
807    max_per_agent: usize,
808    max_age_days: i64,
809}
810
811impl RetentionConfig {
812    /// Build a policy. `0` (or a negative age) disables that cap — see the
813    /// type's note.
814    pub fn new(max_per_agent: usize, max_age_days: i64) -> Self {
815        Self {
816            max_per_agent,
817            max_age_days,
818        }
819    }
820
821    /// The count cap, or `None` when disabled.
822    pub fn count_cap(&self) -> Option<usize> {
823        (self.max_per_agent > 0).then_some(self.max_per_agent)
824    }
825
826    /// The age cap in days, or `None` when disabled — including when the
827    /// configured value cannot be turned into a cutoff at all.
828    pub fn age_cap_days(&self) -> Option<i64> {
829        self.age_cutoff(Utc::now()).map(|_| self.max_age_days)
830    }
831
832    /// The age cutoff at `now`, or `None` when disabled.
833    ///
834    /// `checked_sub_signed` rather than plain subtraction: `max_age_days` is an
835    /// operator-supplied `i64`, and `chrono` panics rather than saturating on a
836    /// value that overflows. A config typo must not take down the daemon at
837    /// boot, so an unrepresentable cutoff reads as no cap — the safe direction,
838    /// since the alternative is deleting on a value nobody could have meant.
839    fn age_cutoff(&self, now: DateTime<Utc>) -> Option<DateTime<Utc>> {
840        if self.max_age_days <= 0 {
841            return None;
842        }
843        chrono::Duration::try_days(self.max_age_days).and_then(|d| now.checked_sub_signed(d))
844    }
845}
846
847impl Default for RetentionConfig {
848    fn default() -> Self {
849        Self::new(DEFAULT_MAX_RUNS_PER_AGENT, DEFAULT_MAX_AGE_DAYS)
850    }
851}
852
853/// `[runs]` section of `~/.car/config.toml`. Both keys optional; an
854/// absent key keeps the restrictive default.
855#[derive(Debug, Clone, Default, Deserialize)]
856struct RunsConfigFile {
857    #[serde(default)]
858    runs: RunsSection,
859}
860
861#[derive(Debug, Clone, Default, Deserialize)]
862struct RunsSection {
863    #[serde(default)]
864    max_per_agent: Option<usize>,
865    #[serde(default)]
866    max_age_days: Option<i64>,
867}
868
869impl RetentionConfig {
870    /// Load the retention policy from `<car_dir>/config.toml`'s `[runs]`
871    /// section, falling back to the restrictive default for any missing
872    /// key or an unreadable/malformed file (config errors must never make
873    /// the daemon refuse to start — same posture as the rest of `.car`).
874    ///
875    /// A present key is taken as written, `0` included — see the type's note on
876    /// what `0` means. Filtering it here would silently restore the default
877    /// instead of honoring "keep everything".
878    pub fn from_car_dir(car_dir: &Path) -> Self {
879        let mut cfg = Self::default();
880        let path = car_dir.join("config.toml");
881        let Ok(text) = std::fs::read_to_string(&path) else {
882            return cfg;
883        };
884        let Ok(parsed) = toml::from_str::<RunsConfigFile>(&text) else {
885            return cfg;
886        };
887        if let Some(n) = parsed.runs.max_per_agent {
888            cfg.max_per_agent = n;
889        }
890        if let Some(d) = parsed.runs.max_age_days {
891            cfg.max_age_days = d;
892        }
893        // `0` is a choice and is documented; a NEGATIVE or unrepresentable age
894        // is a typo, and absorbing one without saying so is the only genuinely
895        // silent behaviour here. It reads as no cap — the safe direction, since
896        // the alternative is deleting on a number nobody meant — but an operator
897        // who wrote it has to be able to find out.
898        if cfg.max_age_days < 0 || (cfg.max_age_days > 0 && cfg.age_cutoff(Utc::now()).is_none()) {
899            tracing::warn!(
900                max_age_days = cfg.max_age_days,
901                path = %path.display(),
902                "[runs] max_age_days is not a usable number of days; treating the age cap as disabled"
903            );
904        }
905        cfg
906    }
907}
908
909/// JSONL run-trace store rooted at `<car_dir>/runs/`.
910///
911/// Stateless across calls — each append opens, writes, and closes the
912/// run's file. Flush points are sparse (turn granularity), so there is no
913/// long-lived file handle to manage, and a concurrently-restarting daemon
914/// always sees a consistent on-disk tail.
915#[derive(Debug, Clone)]
916pub struct RunStore {
917    /// `<car_dir>/runs` — the tree root, created `0700`.
918    root: PathBuf,
919    retention: RetentionConfig,
920    failures: RunStoreFailureInjector,
921    summary_read_gate: Option<RunStoreSummaryReadGate>,
922    summary_write_gate: Option<RunStoreSummaryWriteGate>,
923    append_gate: Option<RunStoreAppendGate>,
924    lookup_gate: Option<RunStoreLookupGate>,
925    private_path_failures: Option<car_secrets::PrivatePathDurabilityFailureInjector>,
926    append_locks: Arc<Mutex<HashMap<PathBuf, Weak<Mutex<()>>>>>,
927    trace_corruptions: Arc<Mutex<HashMap<(String, String), RunTraceCorruption>>>,
928}
929
930#[derive(Debug, Default)]
931struct SummaryReadGateState {
932    armed: bool,
933    entered: bool,
934}
935
936/// Deterministic test/embedder seam for proving summary reads do not hold
937/// async runtime locks. It is inert unless explicitly armed.
938#[derive(Debug, Clone, Default)]
939pub struct RunStoreSummaryReadGate {
940    state: Arc<(Mutex<SummaryReadGateState>, Condvar)>,
941}
942
943pub type RunStoreAppendGate = RunStoreSummaryReadGate;
944pub type RunStoreLookupGate = RunStoreSummaryReadGate;
945#[doc(hidden)]
946pub type RunStoreSummaryWriteGate = RunStoreSummaryReadGate;
947
948impl RunStoreSummaryReadGate {
949    pub fn block_next(&self) {
950        let (lock, _) = &*self.state;
951        let mut state = lock.lock().expect("run-store gate mutex poisoned");
952        state.armed = true;
953        state.entered = false;
954    }
955
956    pub fn wait_until_entered(&self, timeout: std::time::Duration) -> bool {
957        let (lock, ready) = &*self.state;
958        let state = lock.lock().expect("run-store gate mutex poisoned");
959        let (state, _) = ready
960            .wait_timeout_while(state, timeout, |state| !state.entered)
961            .expect("run-store gate mutex poisoned while waiting");
962        state.entered
963    }
964
965    pub fn release(&self) {
966        let (lock, ready) = &*self.state;
967        let mut state = lock.lock().expect("run-store gate mutex poisoned");
968        state.armed = false;
969        ready.notify_all();
970    }
971
972    fn wait_if_armed(&self) {
973        let (lock, ready) = &*self.state;
974        let mut state = lock.lock().expect("run-store gate mutex poisoned");
975        if !state.armed || state.entered {
976            return;
977        }
978        state.entered = true;
979        ready.notify_all();
980        while state.armed {
981            state = ready
982                .wait(state)
983                .expect("run-store gate mutex poisoned while blocked");
984        }
985    }
986}
987
988#[derive(Debug, Clone)]
989pub struct ProposalTraceEnsure {
990    pub records: Vec<RunRecord>,
991    pub appended: bool,
992}
993
994/// Exact durable append boundary used by deterministic fault-injection tests.
995#[derive(Debug, Clone, Copy, PartialEq, Eq)]
996pub enum RunStoreFailurePoint {
997    MarkerWrite,
998    Write,
999    Flush,
1000    Fsync,
1001    PendingUnlink,
1002    MarkerUnlink,
1003    DirectoryFsync,
1004    ResponseSerialization,
1005    SummaryWrite,
1006    CorruptionMarkerWrite,
1007    CorruptionSummaryInvalidate,
1008}
1009
1010#[derive(Debug, Clone, Default)]
1011pub struct RunStoreFailureInjector {
1012    failures: Arc<Mutex<VecDeque<RunStoreFailurePoint>>>,
1013}
1014
1015fn injected_storage_full() -> std::io::Error {
1016    std::io::Error::new(
1017        std::io::ErrorKind::StorageFull,
1018        "injected run-store storage full",
1019    )
1020}
1021
1022impl RunStoreFailureInjector {
1023    pub fn fail_next(&self, point: RunStoreFailurePoint) {
1024        self.failures
1025            .lock()
1026            .expect("run-store failure injector mutex poisoned")
1027            .push_back(point);
1028    }
1029
1030    fn take(&self, point: RunStoreFailurePoint) -> bool {
1031        let mut failures = self
1032            .failures
1033            .lock()
1034            .expect("run-store failure injector mutex poisoned");
1035        if failures.front() == Some(&point) {
1036            failures.pop_front();
1037            true
1038        } else {
1039            false
1040        }
1041    }
1042}
1043
1044impl RunStore {
1045    /// Construct a store rooted at `runs_root` (the `runs/` dir itself).
1046    /// Use [`RunStore::from_journal_dir`] from the daemon, which derives
1047    /// the root from the configured journal dir; this constructor is the
1048    /// test/embedder seam.
1049    /// The retention policy in force, for a caller reporting what GC did.
1050    pub fn retention(&self) -> RetentionConfig {
1051        self.retention
1052    }
1053
1054    pub fn new(runs_root: PathBuf, retention: RetentionConfig) -> Self {
1055        Self {
1056            root: runs_root,
1057            retention,
1058            failures: RunStoreFailureInjector::default(),
1059            summary_read_gate: None,
1060            summary_write_gate: None,
1061            append_gate: None,
1062            lookup_gate: None,
1063            private_path_failures: None,
1064            append_locks: Arc::new(Mutex::new(HashMap::new())),
1065            trace_corruptions: Arc::new(Mutex::new(HashMap::new())),
1066        }
1067    }
1068
1069    pub fn with_failure_injector(mut self, failures: RunStoreFailureInjector) -> Self {
1070        self.failures = failures;
1071        self
1072    }
1073
1074    pub fn with_summary_read_gate(mut self, gate: RunStoreSummaryReadGate) -> Self {
1075        self.summary_read_gate = Some(gate);
1076        self
1077    }
1078
1079    #[doc(hidden)]
1080    pub fn with_summary_write_gate(mut self, gate: RunStoreSummaryWriteGate) -> Self {
1081        self.summary_write_gate = Some(gate);
1082        self
1083    }
1084
1085    pub fn with_append_gate(mut self, gate: RunStoreAppendGate) -> Self {
1086        self.append_gate = Some(gate);
1087        self
1088    }
1089
1090    pub fn with_lookup_gate(mut self, gate: RunStoreLookupGate) -> Self {
1091        self.lookup_gate = Some(gate);
1092        self
1093    }
1094
1095    /// Test/embedder seam for deterministic first-use directory-entry faults.
1096    pub fn with_private_path_failure_injector(
1097        mut self,
1098        failures: car_secrets::PrivatePathDurabilityFailureInjector,
1099    ) -> Self {
1100        self.private_path_failures = Some(failures);
1101        self
1102    }
1103
1104    fn ensure_private_dir(&self, path: &Path) -> std::io::Result<()> {
1105        match self.private_path_failures.as_ref() {
1106            Some(failures) => car_secrets::ensure_private_dir_with_failure_injector(path, failures),
1107            None => car_secrets::ensure_private_dir(path),
1108        }
1109    }
1110
1111    fn create_private_file(&self, path: &Path) -> std::io::Result<File> {
1112        match self.private_path_failures.as_ref() {
1113            Some(failures) => {
1114                car_secrets::create_private_file_with_failure_injector(path, failures)
1115            }
1116            None => car_secrets::create_private_file(path),
1117        }
1118    }
1119
1120    fn open_private_append(&self, path: &Path) -> std::io::Result<File> {
1121        match self.private_path_failures.as_ref() {
1122            Some(failures) => {
1123                car_secrets::open_private_append_with_failure_injector(path, failures)
1124            }
1125            None => car_secrets::open_private_append(path),
1126        }
1127    }
1128
1129    /// Derive the store from the daemon's journal dir. The journal lives
1130    /// at `~/.car/journals`, so the run store is its sibling
1131    /// `~/.car/runs`; retention is read from `~/.car/config.toml`. When
1132    /// the journal dir has no parent (a bare relative path), the store
1133    /// falls back to `journal_dir/../runs` resolved lexically.
1134    pub fn from_journal_dir(journal_dir: &Path) -> Self {
1135        let car_dir = journal_dir
1136            .parent()
1137            .map(Path::to_path_buf)
1138            .unwrap_or_else(|| PathBuf::from("."));
1139        let root = car_dir.join("runs");
1140        let retention = RetentionConfig::from_car_dir(&car_dir);
1141        Self::new(root, retention)
1142    }
1143
1144    /// The `runs/` tree root.
1145    pub fn root(&self) -> &Path {
1146        &self.root
1147    }
1148
1149    fn proposal_outbox_root(&self) -> PathBuf {
1150        self.root
1151            .parent()
1152            .unwrap_or_else(|| Path::new("."))
1153            .join("proposal-finalization")
1154    }
1155
1156    fn proposal_outbox_path(&self, run_id: &str) -> PathBuf {
1157        let key = format!("{:x}", Sha256::digest(run_id.as_bytes()));
1158        self.proposal_outbox_root().join(format!("{key}.json"))
1159    }
1160
1161    fn proposal_execution_root(&self) -> PathBuf {
1162        self.root
1163            .parent()
1164            .unwrap_or_else(|| Path::new("."))
1165            .join("proposal-execution")
1166    }
1167
1168    fn proposal_execution_path(&self, run_id: &str) -> PathBuf {
1169        let key = format!("{:x}", Sha256::digest(run_id.as_bytes()));
1170        self.proposal_execution_root().join(format!("{key}.json"))
1171    }
1172
1173    fn proposal_retry_rollback_root(&self) -> PathBuf {
1174        self.root
1175            .parent()
1176            .unwrap_or_else(|| Path::new("."))
1177            .join("proposal-retry-rollbacks")
1178    }
1179
1180    fn proposal_retry_rollback_path(&self, run_id: &str) -> PathBuf {
1181        let key = format!("{:x}", Sha256::digest(run_id.as_bytes()));
1182        self.proposal_retry_rollback_root()
1183            .join(format!("{key}.json"))
1184    }
1185
1186    fn proposal_id_claim_root(&self, run_id: &str) -> PathBuf {
1187        let run_key = format!("{:x}", Sha256::digest(run_id.as_bytes()));
1188        self.root
1189            .parent()
1190            .unwrap_or_else(|| Path::new("."))
1191            .join("proposal-id-claims")
1192            .join(run_key)
1193    }
1194
1195    fn proposal_id_claim_path(&self, run_id: &str, proposal_id: &str) -> PathBuf {
1196        let proposal_key = format!("{:x}", Sha256::digest(proposal_id.as_bytes()));
1197        self.proposal_id_claim_root(run_id)
1198            .join(format!("{proposal_key}.json"))
1199    }
1200
1201    fn completed_response_root(&self) -> PathBuf {
1202        self.root
1203            .parent()
1204            .unwrap_or_else(|| Path::new("."))
1205            .join("proposal-completed")
1206    }
1207
1208    fn completed_response_run_root(&self, run_id: &str) -> PathBuf {
1209        let run_key = format!("{:x}", Sha256::digest(run_id.as_bytes()));
1210        self.completed_response_root().join(run_key)
1211    }
1212
1213    fn completed_response_index_root(&self) -> PathBuf {
1214        self.root
1215            .parent()
1216            .unwrap_or_else(|| Path::new("."))
1217            .join("proposal-completed-index")
1218    }
1219
1220    fn completed_response_index_migration_path(&self) -> PathBuf {
1221        self.completed_response_index_root()
1222            .join("owner-index-migration.json")
1223    }
1224
1225    fn completed_response_owner_key(
1226        requested_policy_session_id: Option<&str>,
1227        original_submission: &Value,
1228    ) -> std::io::Result<String> {
1229        let key_preimage = serde_json::to_vec(&(requested_policy_session_id, original_submission))
1230            .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
1231        Ok(format!("{:x}", Sha256::digest(key_preimage)))
1232    }
1233
1234    fn completed_response_owner_path(
1235        &self,
1236        requested_policy_session_id: Option<&str>,
1237        original_submission: &Value,
1238    ) -> std::io::Result<PathBuf> {
1239        Ok(self.completed_response_index_root().join(format!(
1240            "{}.json",
1241            Self::completed_response_owner_key(requested_policy_session_id, original_submission,)?
1242        )))
1243    }
1244
1245    fn completed_response_key(
1246        client_id: &str,
1247        requested_policy_session_id: Option<&str>,
1248        original_submission: &Value,
1249    ) -> std::io::Result<String> {
1250        let key_preimage =
1251            serde_json::to_vec(&(client_id, requested_policy_session_id, original_submission))
1252                .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
1253        Ok(format!("{:x}", Sha256::digest(key_preimage)))
1254    }
1255
1256    fn completed_response_path(
1257        &self,
1258        run_id: &str,
1259        client_id: &str,
1260        requested_policy_session_id: Option<&str>,
1261        original_submission: &Value,
1262    ) -> std::io::Result<PathBuf> {
1263        Ok(self.completed_response_run_root(run_id).join(format!(
1264            "{}.json",
1265            Self::completed_response_key(
1266                client_id,
1267                requested_policy_session_id,
1268                original_submission,
1269            )?
1270        )))
1271    }
1272
1273    /// Read the exact durable `RunStarted` boundary for `run_id` without
1274    /// collapsing absence into corruption. A zero-byte file left by an open
1275    /// that failed before its first write is still absent; any non-empty file
1276    /// without one valid, unambiguous `RunStarted` is invalid durable state.
1277    pub fn run_started(&self, run_id: &str) -> std::io::Result<Option<car_proto::RunStarted>> {
1278        if let Some(gate) = &self.lookup_gate {
1279            gate.wait_if_armed();
1280        }
1281        match self.open_root_for_read() {
1282            Ok(()) => {}
1283            Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
1284            Err(error) => return Err(error),
1285        }
1286        let file_name = format!("{}.jsonl", sanitize(run_id));
1287        let mut found = None;
1288        for agent in std::fs::read_dir(&self.root)? {
1289            let agent = agent?;
1290            let file_type = agent.file_type()?;
1291            if !file_type.is_dir() || file_type.is_symlink() {
1292                continue;
1293            }
1294            car_secrets::ensure_private_dir(&agent.path())?;
1295            let path = agent.path().join(&file_name);
1296            match car_secrets::open_private_read(&path) {
1297                Ok(file) if found.is_none() => found = Some((path, file)),
1298                Ok(_) => {
1299                    return Err(std::io::Error::new(
1300                        std::io::ErrorKind::InvalidData,
1301                        format!("durable run `{run_id}` exists under multiple agents"),
1302                    ))
1303                }
1304                Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
1305                Err(error) => return Err(error),
1306            }
1307        }
1308        let Some((path, file)) = found else {
1309            return Ok(None);
1310        };
1311        let file_len = file.metadata()?.len();
1312        let records = load_private_records(&path, &file)?;
1313        if records.is_empty() && file_len == 0 {
1314            return Ok(None);
1315        }
1316        let mut started_rows = records.iter().filter_map(|record| match record {
1317            RunRecord::Started(started) => Some(started),
1318            _ => None,
1319        });
1320        let started = started_rows.next().ok_or_else(|| {
1321            std::io::Error::new(
1322                std::io::ErrorKind::InvalidData,
1323                format!("durable run record for `{run_id}` has no RunStarted"),
1324            )
1325        })?;
1326        if started_rows.next().is_some()
1327            || started.run_id != run_id
1328            || started.client_id.as_deref().is_none_or(str::is_empty)
1329        {
1330            return Err(std::io::Error::new(
1331                std::io::ErrorKind::InvalidData,
1332                format!("durable RunStarted identity for `{run_id}` is ambiguous or invalid"),
1333            ));
1334        }
1335        Ok(Some(started.clone()))
1336    }
1337
1338    fn durable_run_started(&self, run_id: &str) -> std::io::Result<car_proto::RunStarted> {
1339        self.run_started(run_id)?.ok_or_else(|| {
1340            std::io::Error::new(
1341                std::io::ErrorKind::NotFound,
1342                format!("durable RunStarted for run `{run_id}` is absent"),
1343            )
1344        })
1345    }
1346
1347    /// Remove only the zero-byte trace created when `runs.start` opened its
1348    /// destination but failed before writing `RunStarted`. The full expected
1349    /// identity is accepted so callers cannot use this as a broad delete
1350    /// primitive. Any bytes or parsed rows make rollback fail closed.
1351    pub fn rollback_empty_run_start(&self, started: &car_proto::RunStarted) -> std::io::Result<()> {
1352        let dir = match self.open_agent_dir_for_read(&started.agent_id) {
1353            Ok(dir) => dir,
1354            Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()),
1355            Err(error) => return Err(error),
1356        };
1357        let path = self.run_path(&started.agent_id, &started.run_id);
1358        let append_lock = self.append_lock(&path);
1359        let _guard = append_lock
1360            .lock()
1361            .map_err(|_| std::io::Error::other("run append lock poisoned"))?;
1362        let file = match car_secrets::open_private_read(&path) {
1363            Ok(file) => file,
1364            Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()),
1365            Err(error) => return Err(error),
1366        };
1367        car_secrets::revalidate_private_path(&path, &file)?;
1368        if file.metadata()?.len() != 0 {
1369            return Err(std::io::Error::new(
1370                std::io::ErrorKind::AlreadyExists,
1371                format!(
1372                    "run `{}` acquired durable bytes while empty-start rollback was pending",
1373                    started.run_id
1374                ),
1375            ));
1376        }
1377        drop(file);
1378        std::fs::remove_file(&path)?;
1379        sync_directory(dir)
1380    }
1381
1382    pub fn pending_provenance(
1383        &self,
1384        pending: &PendingProposalFinalization,
1385    ) -> std::io::Result<(car_proto::RunStarted, ProposalExecutionMarker)> {
1386        pending.validate()?;
1387        let started = self.durable_run_started(&pending.run_id)?;
1388        let marker = self.execution_marker(&pending.run_id)?.ok_or_else(|| {
1389            std::io::Error::new(
1390                std::io::ErrorKind::InvalidData,
1391                "pending finalization is missing its durable execution marker",
1392            )
1393        })?;
1394        pending.validate_provenance(&started, &marker)?;
1395        Ok((started, marker))
1396    }
1397
1398    /// Write the durable `execution_in_progress` marker before tool dispatch.
1399    pub fn write_execution_marker(&self, marker: &ProposalExecutionMarker) -> std::io::Result<()> {
1400        marker.validate()?;
1401        if self.failures.take(RunStoreFailurePoint::MarkerWrite) {
1402            return Err(std::io::Error::other(
1403                "injected proposal execution marker write failure",
1404            ));
1405        }
1406        let root = self.proposal_execution_root();
1407        self.ensure_private_dir(&root)?;
1408        ensure_backup_excluded(&root)?;
1409        let path = self.proposal_execution_path(&marker.run_id);
1410        match car_secrets::open_private_read(&path) {
1411            Ok(file) => {
1412                let existing: ProposalExecutionMarker = serde_json::from_reader(file)
1413                    .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
1414                if existing == *marker {
1415                    return Ok(());
1416                }
1417                return Err(std::io::Error::new(
1418                    std::io::ErrorKind::AlreadyExists,
1419                    "run already has a different execution-in-progress marker",
1420                ));
1421            }
1422            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
1423            Err(error) => return Err(error),
1424        }
1425        let temp = root.join(format!(
1426            ".{}.{}.tmp",
1427            path.file_stem()
1428                .and_then(|value| value.to_str())
1429                .unwrap_or("proposal"),
1430            uuid::Uuid::new_v4().simple()
1431        ));
1432        let write_result = (|| {
1433            let mut file = self.create_private_file(&temp)?;
1434            serde_json::to_writer(&mut file, marker)
1435                .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
1436            file.flush()?;
1437            file.sync_all()?;
1438            car_secrets::revalidate_private_path(&temp, &file)?;
1439            drop(file);
1440            car_secrets::atomic_replace_private_file(&temp, &path)?;
1441            sync_directory(&root)
1442        })();
1443        if write_result.is_err() {
1444            let _ = std::fs::remove_file(&temp);
1445        }
1446        write_result
1447    }
1448
1449    pub fn execution_marker(
1450        &self,
1451        run_id: &str,
1452    ) -> std::io::Result<Option<ProposalExecutionMarker>> {
1453        let path = self.proposal_execution_path(run_id);
1454        let file = match car_secrets::open_private_read(&path) {
1455            Ok(file) => file,
1456            Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
1457            Err(error) => return Err(error),
1458        };
1459        let marker: ProposalExecutionMarker = serde_json::from_reader(file)
1460            .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
1461        if marker.run_id != run_id {
1462            return Err(std::io::Error::new(
1463                std::io::ErrorKind::InvalidData,
1464                format!(
1465                    "proposal execution marker run_id `{}` does not match requested run `{run_id}`",
1466                    marker.run_id
1467                ),
1468            ));
1469        }
1470        marker.validate()?;
1471        Ok(Some(marker))
1472    }
1473
1474    /// Persist the pre-execution rollback authority before acquiring the
1475    /// content-addressed retry owner. Exact rewrites are idempotent; a
1476    /// conflicting preimage for the same run fails closed.
1477    pub fn write_proposal_retry_rollback(
1478        &self,
1479        rollback: &ProposalRetryRollback,
1480    ) -> std::io::Result<()> {
1481        rollback.validate()?;
1482        let root = self.proposal_retry_rollback_root();
1483        self.ensure_private_dir(&root)?;
1484        ensure_backup_excluded(&root)?;
1485        let path = self.proposal_retry_rollback_path(&rollback.run_id);
1486        match car_secrets::open_private_read(&path) {
1487            Ok(file) => {
1488                let existing: ProposalRetryRollback = serde_json::from_reader(file)
1489                    .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
1490                existing.validate()?;
1491                if existing == *rollback {
1492                    return Ok(());
1493                }
1494                return Err(std::io::Error::new(
1495                    std::io::ErrorKind::AlreadyExists,
1496                    "run already has a different proposal retry rollback intent",
1497                ));
1498            }
1499            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
1500            Err(error) => return Err(error),
1501        }
1502        self.write_private_json_atomic(&root, &path, rollback)
1503    }
1504
1505    pub fn proposal_retry_rollback(
1506        &self,
1507        run_id: &str,
1508    ) -> std::io::Result<Option<ProposalRetryRollback>> {
1509        let path = self.proposal_retry_rollback_path(run_id);
1510        let file = match car_secrets::open_private_read(&path) {
1511            Ok(file) => file,
1512            Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
1513            Err(error) => return Err(error),
1514        };
1515        let rollback: ProposalRetryRollback = serde_json::from_reader(file)
1516            .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
1517        rollback.validate()?;
1518        if rollback.run_id != run_id || path != self.proposal_retry_rollback_path(&rollback.run_id)
1519        {
1520            return Err(std::io::Error::new(
1521                std::io::ErrorKind::InvalidData,
1522                "proposal retry rollback path does not match its run identity",
1523            ));
1524        }
1525        Ok(Some(rollback))
1526    }
1527
1528    pub fn clear_proposal_retry_rollback(
1529        &self,
1530        expected: &ProposalRetryRollback,
1531    ) -> std::io::Result<()> {
1532        let Some(existing) = self.proposal_retry_rollback(&expected.run_id)? else {
1533            return Ok(());
1534        };
1535        if existing != *expected {
1536            return Err(std::io::Error::new(
1537                std::io::ErrorKind::InvalidInput,
1538                "proposal retry rollback does not match the exact authority",
1539            ));
1540        }
1541        let root = self.proposal_retry_rollback_root();
1542        std::fs::remove_file(self.proposal_retry_rollback_path(&expected.run_id))?;
1543        sync_directory(root)
1544    }
1545
1546    /// Resolve every crash-left rollback intent. The intent is removed last:
1547    /// a second crash at any earlier boundary leaves enough authority for the
1548    /// next startup to repeat the exact cleanup.
1549    pub fn reconcile_proposal_retry_rollbacks(&self) -> std::io::Result<usize> {
1550        let root = self.proposal_retry_rollback_root();
1551        let entries = match std::fs::read_dir(&root) {
1552            Ok(entries) => entries,
1553            Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(0),
1554            Err(error) => return Err(error),
1555        };
1556        let mut reconciled = 0usize;
1557        for entry in entries {
1558            let entry = entry?;
1559            if entry.path().extension().and_then(|value| value.to_str()) != Some("json") {
1560                continue;
1561            }
1562            let file = car_secrets::open_private_read(&entry.path())?;
1563            let rollback: ProposalRetryRollback = serde_json::from_reader(file)
1564                .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
1565            rollback.validate()?;
1566            if entry.path() != self.proposal_retry_rollback_path(&rollback.run_id) {
1567                return Err(std::io::Error::new(
1568                    std::io::ErrorKind::InvalidData,
1569                    "proposal retry rollback filename does not match its run identity",
1570                ));
1571            }
1572            if self.execution_marker(&rollback.run_id)?.is_none()
1573                && self.completed_proposal_retry_owner(
1574                    rollback.requested_policy_session_id.as_deref(),
1575                    &rollback.original_submission,
1576                )? == Some((rollback.run_id.clone(), rollback.client_id.clone()))
1577            {
1578                self.release_proposal_retry_owner(
1579                    &rollback.run_id,
1580                    &rollback.client_id,
1581                    rollback.requested_policy_session_id.as_deref(),
1582                    &rollback.original_submission,
1583                )?;
1584            }
1585            self.clear_proposal_retry_rollback(&rollback)?;
1586            reconciled = reconciled.saturating_add(1);
1587        }
1588        Ok(reconciled)
1589    }
1590
1591    /// Claim a proposal identifier inside one authenticated run before any
1592    /// dispatch. Exact-submission retries are idempotent; reusing the same id
1593    /// for changed bytes is rejected durably, including after restart.
1594    pub fn claim_proposal_id(
1595        &self,
1596        run_id: &str,
1597        client_id: &str,
1598        proposal_id: &str,
1599        original_submission: &Value,
1600    ) -> std::io::Result<ProposalIdClaimOutcome> {
1601        if run_id.is_empty() || client_id.is_empty() || proposal_id.is_empty() {
1602            return Err(std::io::Error::new(
1603                std::io::ErrorKind::InvalidInput,
1604                "proposal id claim requires non-empty run/client/proposal identities",
1605            ));
1606        }
1607        let expected = ProposalIdClaim {
1608            run_id: run_id.to_string(),
1609            client_id: client_id.to_string(),
1610            proposal_id: proposal_id.to_string(),
1611            original_submission: original_submission.clone(),
1612        };
1613        let root = self.proposal_id_claim_root(run_id);
1614        self.ensure_private_dir(&root)?;
1615        ensure_backup_excluded(&root)?;
1616        let path = self.proposal_id_claim_path(run_id, proposal_id);
1617        let lock = self.append_lock(&path);
1618        let _guard = lock
1619            .lock()
1620            .map_err(|_| std::io::Error::other("proposal id claim lock poisoned"))?;
1621        match car_secrets::open_private_read(&path) {
1622            Ok(file) => {
1623                let existing: ProposalIdClaim = serde_json::from_reader(file)
1624                    .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
1625                if existing == expected {
1626                    return Ok(ProposalIdClaimOutcome::ExistingExact);
1627                }
1628                return Err(std::io::Error::new(
1629                    std::io::ErrorKind::AlreadyExists,
1630                    "proposal id is already bound to a different submission in this run",
1631                ));
1632            }
1633            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
1634            Err(error) => return Err(error),
1635        }
1636        self.write_private_json_atomic(&root, &path, &expected)?;
1637        Ok(ProposalIdClaimOutcome::Acquired)
1638    }
1639
1640    fn write_private_json_atomic<T: Serialize>(
1641        &self,
1642        root: &Path,
1643        path: &Path,
1644        value: &T,
1645    ) -> std::io::Result<()> {
1646        let temp = root.join(format!(
1647            ".{}.{}.tmp",
1648            path.file_stem()
1649                .and_then(|value| value.to_str())
1650                .unwrap_or("record"),
1651            uuid::Uuid::new_v4().simple()
1652        ));
1653        let write_result = (|| {
1654            let mut file = self.create_private_file(&temp)?;
1655            serde_json::to_writer(&mut file, value)
1656                .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
1657            file.flush()?;
1658            file.sync_all()?;
1659            car_secrets::revalidate_private_path(&temp, &file)?;
1660            drop(file);
1661            car_secrets::atomic_replace_private_file(&temp, path)?;
1662            sync_directory(root)
1663        })();
1664        if write_result.is_err() {
1665            let _ = std::fs::remove_file(&temp);
1666        }
1667        write_result
1668    }
1669
1670    pub fn clear_execution_marker(
1671        &self,
1672        expected: &ProposalExecutionMarker,
1673    ) -> std::io::Result<()> {
1674        let Some(existing) = self.execution_marker(&expected.run_id)? else {
1675            return Ok(());
1676        };
1677        if existing != *expected {
1678            return Err(std::io::Error::new(
1679                std::io::ErrorKind::InvalidInput,
1680                "proposal execution marker does not match the full expected identity tuple",
1681            ));
1682        }
1683        let root = self.proposal_execution_root();
1684        std::fs::remove_file(self.proposal_execution_path(&expected.run_id))?;
1685        sync_directory(root)
1686    }
1687
1688    /// Persist one exact proposal-finalization transaction with private
1689    /// permissions and an fsynced atomic replacement. Exact retries are
1690    /// idempotent; a conflicting preimage for the same run is rejected.
1691    pub fn write_pending_proposal(
1692        &self,
1693        pending: &PendingProposalFinalization,
1694    ) -> std::io::Result<()> {
1695        self.pending_provenance(pending)?;
1696        if pending.run_id.is_empty()
1697            || pending.client_id.is_empty()
1698            || pending.original_proposal_id.is_empty()
1699            || pending.final_proposal_id.is_empty()
1700            || pending.result_digest.len() != 64
1701        {
1702            return Err(std::io::Error::new(
1703                std::io::ErrorKind::InvalidInput,
1704                "proposal finalization requires run/client/proposal identities and a SHA-256 digest",
1705            ));
1706        }
1707        let root = self.proposal_outbox_root();
1708        self.ensure_private_dir(&root)?;
1709        ensure_backup_excluded(&root)?;
1710        let path = self.proposal_outbox_path(&pending.run_id);
1711        match car_secrets::open_private_read(&path) {
1712            Ok(file) => {
1713                let existing: PendingProposalFinalization = serde_json::from_reader(file)
1714                    .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
1715                if existing == *pending {
1716                    return Ok(());
1717                }
1718                return Err(std::io::Error::new(
1719                    std::io::ErrorKind::AlreadyExists,
1720                    "run already has a different pending proposal finalization",
1721                ));
1722            }
1723            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
1724            Err(error) => return Err(error),
1725        }
1726
1727        let temp = root.join(format!(
1728            ".{}.{}.tmp",
1729            path.file_stem()
1730                .and_then(|value| value.to_str())
1731                .unwrap_or("proposal"),
1732            uuid::Uuid::new_v4().simple()
1733        ));
1734        let write_result = (|| {
1735            let mut file = self.create_private_file(&temp)?;
1736            serde_json::to_writer(&mut file, pending)
1737                .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
1738            file.flush()?;
1739            file.sync_all()?;
1740            car_secrets::revalidate_private_path(&temp, &file)?;
1741            drop(file);
1742            car_secrets::atomic_replace_private_file(&temp, &path)?;
1743            sync_directory(&root)
1744        })();
1745        if write_result.is_err() {
1746            let _ = std::fs::remove_file(&temp);
1747        }
1748        write_result
1749    }
1750
1751    pub fn pending_proposal(
1752        &self,
1753        run_id: &str,
1754    ) -> std::io::Result<Option<PendingProposalFinalization>> {
1755        let path = self.proposal_outbox_path(run_id);
1756        let file = match car_secrets::open_private_read(&path) {
1757            Ok(file) => file,
1758            Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
1759            Err(error) => return Err(error),
1760        };
1761        let pending: PendingProposalFinalization = serde_json::from_reader(file)
1762            .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
1763        if pending.run_id != run_id {
1764            return Err(std::io::Error::new(
1765                std::io::ErrorKind::InvalidData,
1766                format!(
1767                    "proposal finalization run_id `{}` does not match requested run `{run_id}`",
1768                    pending.run_id
1769                ),
1770            ));
1771        }
1772        self.pending_provenance(&pending)?;
1773        Ok(Some(pending))
1774    }
1775
1776    pub fn all_pending_proposals(&self) -> std::io::Result<Vec<PendingProposalFinalization>> {
1777        let root = self.proposal_outbox_root();
1778        let entries = match std::fs::read_dir(root) {
1779            Ok(entries) => entries,
1780            Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
1781            Err(error) => return Err(error),
1782        };
1783        let mut pending = Vec::new();
1784        for entry in entries {
1785            let entry = entry?;
1786            if entry.path().extension().and_then(|ext| ext.to_str()) != Some("json") {
1787                continue;
1788            }
1789            let file = car_secrets::open_private_read(&entry.path())?;
1790            let row: PendingProposalFinalization = serde_json::from_reader(file)
1791                .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
1792            if entry.path() != self.proposal_outbox_path(&row.run_id) {
1793                return Err(std::io::Error::new(
1794                    std::io::ErrorKind::InvalidData,
1795                    format!(
1796                        "proposal finalization filename does not match internal run_id `{}`",
1797                        row.run_id
1798                    ),
1799                ));
1800            }
1801            self.pending_provenance(&row)?;
1802            pending.push(row);
1803        }
1804        Ok(pending)
1805    }
1806
1807    /// Remove only the exact, provenance-validated transaction whose terminal
1808    /// was acknowledged. The caller clears this before its matching execution
1809    /// marker so provenance remains available for the check.
1810    pub fn clear_pending_proposal(
1811        &self,
1812        expected: &PendingProposalFinalization,
1813    ) -> std::io::Result<()> {
1814        self.pending_provenance(expected)?;
1815        let existing = self.pending_proposal(&expected.run_id)?.ok_or_else(|| {
1816            std::io::Error::new(
1817                std::io::ErrorKind::NotFound,
1818                "proposal finalization outbox row is absent",
1819            )
1820        })?;
1821        if existing != *expected {
1822            return Err(std::io::Error::new(
1823                std::io::ErrorKind::InvalidInput,
1824                "proposal finalization outbox does not match the full expected transaction",
1825            ));
1826        }
1827        let root = self.proposal_outbox_root();
1828        std::fs::remove_file(self.proposal_outbox_path(&expected.run_id))?;
1829        sync_directory(root)
1830    }
1831
1832    /// Validate only the receipt's self-contained typed preimages. The sole
1833    /// creation path separately requires [`Self::pending_provenance`] before
1834    /// publishing, while retained reads cannot depend on a GC-eligible trace.
1835    fn validate_completed_response(
1836        &self,
1837        receipt: &CompletedProposalResponse,
1838    ) -> std::io::Result<()> {
1839        receipt.validate()?;
1840        Ok(())
1841    }
1842
1843    fn read_completed_proposal_owner(
1844        &self,
1845        requested_policy_session_id: Option<&str>,
1846        original_submission: &Value,
1847    ) -> std::io::Result<Option<CompletedProposalOwnership>> {
1848        let path =
1849            self.completed_response_owner_path(requested_policy_session_id, original_submission)?;
1850        let file = match car_secrets::open_private_read(&path) {
1851            Ok(file) => file,
1852            Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
1853            Err(error) => return Err(error),
1854        };
1855        let owner: CompletedProposalOwnership = serde_json::from_reader(file)
1856            .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
1857        if owner.requested_policy_session_id.as_deref() != requested_policy_session_id
1858            || owner.original_submission != *original_submission
1859            || path
1860                != self.completed_response_owner_path(
1861                    owner.requested_policy_session_id.as_deref(),
1862                    &owner.original_submission,
1863                )?
1864            || owner.run_id.is_empty()
1865            || owner.client_id.is_empty()
1866        {
1867            return Err(std::io::Error::new(
1868                std::io::ErrorKind::InvalidData,
1869                "completed proposal ownership index does not match its retry tuple",
1870            ));
1871        }
1872        Ok(Some(owner))
1873    }
1874
1875    /// Atomically reserve the process-wide retry tuple before any execution
1876    /// marker or runtime invocation. Direct exclusive creation is the linear
1877    /// point: one run/client wins, every concurrent or restarted contender
1878    /// observes that durable owner, and a partial claim fails closed.
1879    pub fn reserve_proposal_retry_owner(
1880        &self,
1881        run_id: &str,
1882        client_id: &str,
1883        requested_policy_session_id: Option<&str>,
1884        original_submission: &Value,
1885    ) -> std::io::Result<ProposalRetryReservation> {
1886        if run_id.is_empty() || client_id.is_empty() || requested_policy_session_id == Some("") {
1887            return Err(std::io::Error::new(
1888                std::io::ErrorKind::InvalidInput,
1889                "proposal retry reservation requires non-empty run/client/policy identities",
1890            ));
1891        }
1892        let expected = CompletedProposalOwnership::new(
1893            run_id,
1894            client_id,
1895            requested_policy_session_id,
1896            original_submission,
1897        );
1898        let root = self.completed_response_index_root();
1899        self.ensure_private_dir(&root)?;
1900        ensure_backup_excluded(&root)?;
1901        sync_directory(root.parent().unwrap_or_else(|| Path::new(".")))?;
1902        let path =
1903            self.completed_response_owner_path(requested_policy_session_id, original_submission)?;
1904        let lock = self.append_lock(&path);
1905        let _guard = lock
1906            .lock()
1907            .map_err(|_| std::io::Error::other("proposal retry reservation lock poisoned"))?;
1908        if let Some(existing) =
1909            self.read_completed_proposal_owner(requested_policy_session_id, original_submission)?
1910        {
1911            return Ok(ProposalRetryReservation::Existing {
1912                run_id: existing.run_id,
1913                client_id: existing.client_id,
1914            });
1915        }
1916
1917        let mut file = match self.create_private_file(&path) {
1918            Ok(file) => file,
1919            Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {
1920                let existing = self
1921                    .read_completed_proposal_owner(
1922                        requested_policy_session_id,
1923                        original_submission,
1924                    )?
1925                    .ok_or_else(|| {
1926                        std::io::Error::new(
1927                            std::io::ErrorKind::InvalidData,
1928                            "proposal retry reservation appeared without a readable owner",
1929                        )
1930                    })?;
1931                return Ok(ProposalRetryReservation::Existing {
1932                    run_id: existing.run_id,
1933                    client_id: existing.client_id,
1934                });
1935            }
1936            Err(error) => return Err(error),
1937        };
1938        serde_json::to_writer(&mut file, &expected)
1939            .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
1940        file.flush()?;
1941        file.sync_all()?;
1942        car_secrets::revalidate_private_path(&path, &file)?;
1943        sync_directory(&root)?;
1944        Ok(ProposalRetryReservation::Acquired)
1945    }
1946
1947    /// Release only the exact pre-dispatch retry owner. This is used when
1948    /// execution never became admissible (marker durability or policy bind
1949    /// failed), so a later scheduled run may safely submit the same proposal.
1950    /// The content-addressed owner is validated before unlink and the owner
1951    /// directory is fsynced before success is reported.
1952    pub fn release_proposal_retry_owner(
1953        &self,
1954        run_id: &str,
1955        client_id: &str,
1956        requested_policy_session_id: Option<&str>,
1957        original_submission: &Value,
1958    ) -> std::io::Result<()> {
1959        let path =
1960            self.completed_response_owner_path(requested_policy_session_id, original_submission)?;
1961        let lock = self.append_lock(&path);
1962        let _guard = lock
1963            .lock()
1964            .map_err(|_| std::io::Error::other("proposal retry release lock poisoned"))?;
1965        let Some(existing) =
1966            self.read_completed_proposal_owner(requested_policy_session_id, original_submission)?
1967        else {
1968            return Ok(());
1969        };
1970        if existing.run_id != run_id || existing.client_id != client_id {
1971            return Err(std::io::Error::new(
1972                std::io::ErrorKind::InvalidInput,
1973                "proposal retry owner does not match the exact rollback authority",
1974            ));
1975        }
1976        let root = self.completed_response_index_root();
1977        std::fs::remove_file(path)?;
1978        sync_directory(root)
1979    }
1980
1981    /// Verify or backfill the pre-execution owner before publishing a
1982    /// completed response. Startup uses the same operation to migrate legacy
1983    /// receipts that predate reservations.
1984    fn claim_completed_proposal_owner(
1985        &self,
1986        pending: &PendingProposalFinalization,
1987    ) -> std::io::Result<()> {
1988        match self.reserve_proposal_retry_owner(
1989            &pending.run_id,
1990            &pending.client_id,
1991            pending.requested_policy_session_id.as_deref(),
1992            &pending.original_submission,
1993        )? {
1994            ProposalRetryReservation::Acquired => Ok(()),
1995            ProposalRetryReservation::Existing { run_id, client_id }
1996                if run_id == pending.run_id && client_id == pending.client_id =>
1997            {
1998                Ok(())
1999            }
2000            ProposalRetryReservation::Existing { .. } => Err(std::io::Error::new(
2001                std::io::ErrorKind::AlreadyExists,
2002                "proposal retry tuple already belongs to a different completed response",
2003            )),
2004        }
2005    }
2006
2007    /// Persist an exact typed response only after the matching critical
2008    /// `proposal_completed` append acknowledged. The finalization outbox and
2009    /// execution marker must still be present and provenance-valid here, so a
2010    /// forged receipt cannot become a cleanup authority.
2011    pub fn write_completed_proposal(
2012        &self,
2013        pending: &PendingProposalFinalization,
2014    ) -> std::io::Result<CompletedProposalResponse> {
2015        self.pending_provenance(pending)?;
2016        let receipt = CompletedProposalResponse {
2017            finalization: pending.clone(),
2018        };
2019        self.validate_completed_response(&receipt)?;
2020        self.claim_completed_proposal_owner(pending)?;
2021        let root = self.completed_response_root();
2022        self.ensure_private_dir(&root)?;
2023        ensure_backup_excluded(&root)?;
2024        sync_directory(root.parent().unwrap_or_else(|| Path::new(".")))?;
2025        let run_root = self.completed_response_run_root(&pending.run_id);
2026        self.ensure_private_dir(&run_root)?;
2027        sync_directory(&root)?;
2028        let path = self.completed_response_path(
2029            &pending.run_id,
2030            &pending.client_id,
2031            pending.requested_policy_session_id.as_deref(),
2032            &pending.original_submission,
2033        )?;
2034        match car_secrets::open_private_read(&path) {
2035            Ok(file) => {
2036                let existing: CompletedProposalResponse = serde_json::from_reader(file)
2037                    .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
2038                self.validate_completed_response(&existing)?;
2039                if existing == receipt {
2040                    return Ok(existing);
2041                }
2042                return Err(std::io::Error::new(
2043                    std::io::ErrorKind::AlreadyExists,
2044                    "exact proposal retry key already has a different completed response",
2045                ));
2046            }
2047            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
2048            Err(error) => return Err(error),
2049        }
2050
2051        let temp = run_root.join(format!(".response.{}.tmp", uuid::Uuid::new_v4().simple()));
2052        let write_result = (|| {
2053            let mut file = self.create_private_file(&temp)?;
2054            serde_json::to_writer(&mut file, &receipt)
2055                .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
2056            file.flush()?;
2057            file.sync_all()?;
2058            car_secrets::revalidate_private_path(&temp, &file)?;
2059            drop(file);
2060            car_secrets::atomic_replace_private_file(&temp, &path)?;
2061            sync_directory(&run_root)
2062        })();
2063        if write_result.is_err() {
2064            let _ = std::fs::remove_file(&temp);
2065        }
2066        write_result.map(|()| receipt)
2067    }
2068
2069    /// Load only the receipt whose immutable retry tuple exactly matches this
2070    /// bound run/client/policy/raw submission. Absence permits a new proposal;
2071    /// a present corrupt or self-inconsistent receipt fails closed.
2072    pub fn completed_proposal(
2073        &self,
2074        run_id: &str,
2075        client_id: &str,
2076        requested_policy_session_id: Option<&str>,
2077        original_submission: &Value,
2078    ) -> std::io::Result<Option<CompletedProposalResponse>> {
2079        let path = self.completed_response_path(
2080            run_id,
2081            client_id,
2082            requested_policy_session_id,
2083            original_submission,
2084        )?;
2085        let file = match car_secrets::open_private_read(&path) {
2086            Ok(file) => file,
2087            Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
2088            Err(error) => return Err(error),
2089        };
2090        let receipt: CompletedProposalResponse = serde_json::from_reader(file)
2091            .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
2092        self.validate_completed_response(&receipt)?;
2093        let pending = &receipt.finalization;
2094        if pending.run_id != run_id
2095            || pending.client_id != client_id
2096            || pending.requested_policy_session_id.as_deref() != requested_policy_session_id
2097            || pending.original_submission != *original_submission
2098            || path
2099                != self.completed_response_path(
2100                    &pending.run_id,
2101                    &pending.client_id,
2102                    pending.requested_policy_session_id.as_deref(),
2103                    &pending.original_submission,
2104                )?
2105        {
2106            return Err(std::io::Error::new(
2107                std::io::ErrorKind::InvalidData,
2108                "completed proposal response does not match its exact retry key",
2109            ));
2110        }
2111        Ok(Some(receipt))
2112    }
2113
2114    /// Recover the sole retained response for an exact proposal after an
2115    /// authenticated run owner reconnects with a newly minted policy session.
2116    ///
2117    /// The lookup is deliberately bounded to one run and one durable client.
2118    /// It never treats the replacement policy-session id as durable identity,
2119    /// and ambiguity fails closed instead of selecting an arbitrary receipt.
2120    pub fn completed_proposal_for_resumed_owner(
2121        &self,
2122        run_id: &str,
2123        durable_client_id: &str,
2124        original_submission: &Value,
2125    ) -> std::io::Result<Option<CompletedProposalResponse>> {
2126        let run_root = self.completed_response_run_root(run_id);
2127        let entries = match std::fs::read_dir(&run_root) {
2128            Ok(entries) => entries,
2129            Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
2130            Err(error) => return Err(error),
2131        };
2132        car_secrets::ensure_private_dir(&run_root)?;
2133
2134        let mut receipt_paths = Vec::new();
2135        for entry in entries {
2136            let entry = entry?;
2137            if entry.path().extension().and_then(|ext| ext.to_str()) != Some("json") {
2138                continue;
2139            }
2140            if receipt_paths.len() >= MAX_RESUMED_PROPOSAL_RECEIPTS_PER_RUN {
2141                return Err(std::io::Error::new(
2142                    std::io::ErrorKind::InvalidData,
2143                    "resumed proposal recovery exceeded the per-run receipt count limit",
2144                ));
2145            }
2146            receipt_paths.push(entry.path());
2147        }
2148
2149        let mut matching = None;
2150        let mut scanned_bytes = 0_u64;
2151        for path in receipt_paths {
2152            let file = car_secrets::open_private_read(&path)?;
2153            scanned_bytes = scanned_bytes
2154                .checked_add(file.metadata()?.len())
2155                .filter(|total| *total <= MAX_RESUMED_PROPOSAL_RECEIPT_BYTES_PER_RUN)
2156                .ok_or_else(|| {
2157                    std::io::Error::new(
2158                        std::io::ErrorKind::InvalidData,
2159                        "resumed proposal recovery exceeded the per-run receipt byte limit",
2160                    )
2161                })?;
2162            let receipt: CompletedProposalResponse = serde_json::from_reader(file)
2163                .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
2164            self.validate_completed_response(&receipt)?;
2165            let pending = &receipt.finalization;
2166            if pending.run_id != run_id
2167                || path
2168                    != self.completed_response_path(
2169                        &pending.run_id,
2170                        &pending.client_id,
2171                        pending.requested_policy_session_id.as_deref(),
2172                        &pending.original_submission,
2173                    )?
2174            {
2175                return Err(std::io::Error::new(
2176                    std::io::ErrorKind::InvalidData,
2177                    "completed proposal response path does not match its typed identity",
2178                ));
2179            }
2180            if pending.client_id != durable_client_id
2181                || pending.original_submission != *original_submission
2182            {
2183                continue;
2184            }
2185            if matching.replace(receipt).is_some() {
2186                return Err(std::io::Error::new(
2187                    std::io::ErrorKind::InvalidData,
2188                    "resumed proposal recovery is ambiguous for the exact durable owner and submission",
2189                ));
2190            }
2191        }
2192        Ok(matching)
2193    }
2194
2195    pub fn all_completed_proposals(&self) -> std::io::Result<Vec<CompletedProposalResponse>> {
2196        let root = self.completed_response_root();
2197        let run_dirs = match std::fs::read_dir(&root) {
2198            Ok(entries) => entries,
2199            Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
2200            Err(error) => return Err(error),
2201        };
2202        let mut receipts = Vec::new();
2203        for run_dir in run_dirs {
2204            let run_dir = run_dir?;
2205            if !run_dir.file_type()?.is_dir() || run_dir.file_type()?.is_symlink() {
2206                continue;
2207            }
2208            car_secrets::ensure_private_dir(&run_dir.path())?;
2209            for entry in std::fs::read_dir(run_dir.path())? {
2210                let entry = entry?;
2211                if entry.path().extension().and_then(|ext| ext.to_str()) != Some("json") {
2212                    continue;
2213                }
2214                let file = car_secrets::open_private_read(&entry.path())?;
2215                let receipt: CompletedProposalResponse = serde_json::from_reader(file)
2216                    .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
2217                self.validate_completed_response(&receipt)?;
2218                let pending = &receipt.finalization;
2219                if run_dir.path() != self.completed_response_run_root(&pending.run_id)
2220                    || entry.path()
2221                        != self.completed_response_path(
2222                            &pending.run_id,
2223                            &pending.client_id,
2224                            pending.requested_policy_session_id.as_deref(),
2225                            &pending.original_submission,
2226                        )?
2227                {
2228                    return Err(std::io::Error::new(
2229                        std::io::ErrorKind::InvalidData,
2230                        "completed proposal response path does not match its typed identity",
2231                    ));
2232                }
2233                self.claim_completed_proposal_owner(pending)?;
2234                receipts.push(receipt);
2235            }
2236        }
2237        Ok(receipts)
2238    }
2239
2240    /// Backfill the content-addressed retry-owner index exactly once for
2241    /// receipts written before that index existed. Permanent receipts grow
2242    /// without bound by design, so normal daemon startup must never enumerate
2243    /// them. After this versioned checkpoint is durable, crash cleanup is
2244    /// driven by the bounded pending-finalization outbox instead.
2245    pub fn reconcile_completed_proposal_migration(&self) -> std::io::Result<()> {
2246        let checkpoint_path = self.completed_response_index_migration_path();
2247        match car_secrets::open_private_read(&checkpoint_path) {
2248            Ok(file) => {
2249                let checkpoint: CompletedProposalOwnerIndexMigration =
2250                    serde_json::from_reader(file).map_err(|error| {
2251                        std::io::Error::new(std::io::ErrorKind::InvalidData, error)
2252                    })?;
2253                if checkpoint.version > COMPLETED_PROPOSAL_OWNER_INDEX_MIGRATION_VERSION {
2254                    return Err(std::io::Error::new(
2255                        std::io::ErrorKind::InvalidData,
2256                        format!(
2257                            "unsupported completed-proposal owner-index migration version {}",
2258                            checkpoint.version
2259                        ),
2260                    ));
2261                }
2262                if checkpoint.version == COMPLETED_PROPOSAL_OWNER_INDEX_MIGRATION_VERSION {
2263                    return Ok(());
2264                }
2265                // An older checkpoint indexed owners but did not guarantee
2266                // trace backfill plus authorized guard cleanup. Re-run the
2267                // streamed migration and publish v2 only after both finish.
2268            }
2269            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
2270            Err(error) => return Err(error),
2271        }
2272
2273        // Stream permanent receipts one at a time. A corrupt or unexpectedly
2274        // large legacy row fails closed before the checkpoint, making the
2275        // migration crash-resumable without an unbounded all-receipts Vec.
2276        const MAX_LEGACY_RECEIPT_BYTES: u64 = 16 * 1024 * 1024;
2277        let receipts_root = self.completed_response_root();
2278        match std::fs::read_dir(&receipts_root) {
2279            Ok(run_dirs) => {
2280                for run_dir in run_dirs {
2281                    let run_dir = run_dir?;
2282                    if !run_dir.file_type()?.is_dir() || run_dir.file_type()?.is_symlink() {
2283                        continue;
2284                    }
2285                    car_secrets::ensure_private_dir(&run_dir.path())?;
2286                    for entry in std::fs::read_dir(run_dir.path())? {
2287                        let entry = entry?;
2288                        if entry.path().extension().and_then(|ext| ext.to_str()) != Some("json") {
2289                            continue;
2290                        }
2291                        let file = car_secrets::open_private_read(&entry.path())?;
2292                        if file.metadata()?.len() > MAX_LEGACY_RECEIPT_BYTES {
2293                            return Err(std::io::Error::new(
2294                                std::io::ErrorKind::InvalidData,
2295                                "completed proposal receipt exceeds migration byte limit",
2296                            ));
2297                        }
2298                        let receipt: CompletedProposalResponse = serde_json::from_reader(file)
2299                            .map_err(|error| {
2300                                std::io::Error::new(std::io::ErrorKind::InvalidData, error)
2301                            })?;
2302                        self.validate_completed_response(&receipt)?;
2303                        let pending = &receipt.finalization;
2304                        if run_dir.path() != self.completed_response_run_root(&pending.run_id)
2305                            || entry.path()
2306                                != self.completed_response_path(
2307                                    &pending.run_id,
2308                                    &pending.client_id,
2309                                    pending.requested_policy_session_id.as_deref(),
2310                                    &pending.original_submission,
2311                                )?
2312                        {
2313                            return Err(std::io::Error::new(
2314                                std::io::ErrorKind::InvalidData,
2315                                "completed proposal response path does not match its typed identity",
2316                            ));
2317                        }
2318                        self.claim_completed_proposal_owner(pending)?;
2319                        if self.run_started(&pending.run_id)?.is_some() {
2320                            self.ensure_proposal_turns(pending)?;
2321                        }
2322                        // Cleanup is part of the authorized migration. The
2323                        // checkpoint is deliberately published only after
2324                        // every receipt's guards have been durably cleared.
2325                        self.cleanup_completed_proposal_guards(&receipt)?;
2326                    }
2327                }
2328            }
2329            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
2330            Err(error) => return Err(error),
2331        }
2332        let root = self.completed_response_index_root();
2333        self.ensure_private_dir(&root)?;
2334        ensure_backup_excluded(&root)?;
2335        sync_directory(root.parent().unwrap_or_else(|| Path::new(".")))?;
2336        let temp = root.join(format!(
2337            ".owner-index-migration.{}.tmp",
2338            uuid::Uuid::new_v4().simple()
2339        ));
2340        let write_result = (|| {
2341            let mut file = self.create_private_file(&temp)?;
2342            serde_json::to_writer(
2343                &mut file,
2344                &CompletedProposalOwnerIndexMigration {
2345                    version: COMPLETED_PROPOSAL_OWNER_INDEX_MIGRATION_VERSION,
2346                },
2347            )
2348            .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
2349            file.flush()?;
2350            file.sync_all()?;
2351            car_secrets::revalidate_private_path(&temp, &file)?;
2352            drop(file);
2353            car_secrets::atomic_replace_private_file(&temp, &checkpoint_path)?;
2354            sync_directory(&root)
2355        })();
2356        if write_result.is_err() {
2357            let _ = std::fs::remove_file(&temp);
2358        }
2359        write_result
2360    }
2361
2362    pub fn completed_proposal_retry_owner(
2363        &self,
2364        requested_policy_session_id: Option<&str>,
2365        original_submission: &Value,
2366    ) -> std::io::Result<Option<(String, String)>> {
2367        Ok(self
2368            .read_completed_proposal_owner(requested_policy_session_id, original_submission)?
2369            .map(|owner| (owner.run_id, owner.client_id)))
2370    }
2371
2372    /// Remove leftover pre-response guards only under a completed receipt's
2373    /// exact typed authority. Each deletion is independently durable; a fault
2374    /// leaves the receipt and any remaining guard for safe retry/restart.
2375    pub fn cleanup_completed_proposal_guards(
2376        &self,
2377        receipt: &CompletedProposalResponse,
2378    ) -> std::io::Result<()> {
2379        self.validate_completed_response(receipt)?;
2380        let pending = &receipt.finalization;
2381        // Remove the execution marker first. If either unlink boundary is
2382        // interrupted, the pending-finalization outbox remains as the bounded
2383        // startup cleanup index. Removing pending first could leave a lone
2384        // marker that was discoverable only by rescanning every permanent
2385        // completed receipt.
2386        let expected_marker = receipt.execution_marker()?;
2387        if let Some(existing) = self.execution_marker(&pending.run_id)? {
2388            if existing != expected_marker {
2389                return Err(std::io::Error::new(
2390                    std::io::ErrorKind::InvalidData,
2391                    "completed response does not authorize this execution marker cleanup",
2392                ));
2393            }
2394            if self.failures.take(RunStoreFailurePoint::MarkerUnlink) {
2395                return Err(std::io::Error::other(
2396                    "injected execution marker unlink failure",
2397                ));
2398            }
2399            std::fs::remove_file(self.proposal_execution_path(&pending.run_id))?;
2400            self.sync_cleanup_directory(&self.proposal_execution_root())?;
2401        }
2402
2403        let pending_path = self.proposal_outbox_path(&pending.run_id);
2404        match car_secrets::open_private_read(&pending_path) {
2405            Ok(file) => {
2406                let existing: PendingProposalFinalization = serde_json::from_reader(file)
2407                    .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
2408                existing.validate()?;
2409                if existing != *pending {
2410                    return Err(std::io::Error::new(
2411                        std::io::ErrorKind::InvalidData,
2412                        "completed response does not authorize this pending finalization cleanup",
2413                    ));
2414                }
2415                if self.failures.take(RunStoreFailurePoint::PendingUnlink) {
2416                    return Err(std::io::Error::other(
2417                        "injected pending proposal unlink failure",
2418                    ));
2419                }
2420                std::fs::remove_file(&pending_path)?;
2421                self.sync_cleanup_directory(&self.proposal_outbox_root())?;
2422            }
2423            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
2424            Err(error) => return Err(error),
2425        }
2426        Ok(())
2427    }
2428
2429    fn sync_cleanup_directory(&self, root: &Path) -> std::io::Result<()> {
2430        if self.failures.take(RunStoreFailurePoint::DirectoryFsync) {
2431            return Err(std::io::Error::other(
2432                "injected proposal cleanup directory fsync failure",
2433            ));
2434        }
2435        sync_directory(root)
2436    }
2437
2438    pub fn completed_proposal_response_value(
2439        &self,
2440        receipt: &CompletedProposalResponse,
2441    ) -> std::io::Result<Value> {
2442        self.validate_completed_response(receipt)?;
2443        if self
2444            .failures
2445            .take(RunStoreFailurePoint::ResponseSerialization)
2446        {
2447            return Err(std::io::Error::other(
2448                "injected completed proposal response serialization failure",
2449            ));
2450        }
2451        let result = receipt.proposal_result();
2452        let mut value = serde_json::to_value(result)
2453            .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
2454
2455        // Keep rollback provenance explicit at the durable-response boundary.
2456        // `ActionResult`'s serde shape already carries this field, but this is
2457        // the second serialization after the engine result crossed the pending
2458        // finalization outbox. Project true markers from the typed receipt so a
2459        // response-shaping refactor cannot silently reduce rolled-back success
2460        // to an ordinary committed success. False remains omitted for backward
2461        // compatibility.
2462        let wire_results = value
2463            .get_mut("results")
2464            .and_then(Value::as_array_mut)
2465            .ok_or_else(|| {
2466                std::io::Error::new(
2467                    std::io::ErrorKind::InvalidData,
2468                    "completed proposal response is missing its results array",
2469                )
2470            })?;
2471        if wire_results.len() != result.results.len() {
2472            return Err(std::io::Error::new(
2473                std::io::ErrorKind::InvalidData,
2474                "completed proposal response result count changed during serialization",
2475            ));
2476        }
2477        for (wire, typed) in wire_results.iter_mut().zip(&result.results) {
2478            if typed.rolled_back {
2479                wire.as_object_mut()
2480                    .ok_or_else(|| {
2481                        std::io::Error::new(
2482                            std::io::ErrorKind::InvalidData,
2483                            "completed proposal response contains a non-object action result",
2484                        )
2485                    })?
2486                    .insert("rolled_back".to_string(), Value::Bool(true));
2487            }
2488        }
2489        Ok(value)
2490    }
2491
2492    /// Prepare the CAR-owned run tree for daemon startup. This is intentionally
2493    /// fallible: the daemon must refuse adoption/listening when the root or
2494    /// backup marker cannot be proven owner-private.
2495    pub fn prepare_storage(&self) -> std::io::Result<()> {
2496        self.ensure_root()
2497    }
2498
2499    /// Path to a run's JSONL file: `runs/{agent_id}/{run_id}.jsonl`.
2500    fn run_path(&self, agent_id: &str, run_id: &str) -> PathBuf {
2501        self.root
2502            .join(sanitize(agent_id))
2503            .join(format!("{}.jsonl", sanitize(run_id)))
2504    }
2505
2506    fn run_summary_index_path(agent_path: &Path) -> PathBuf {
2507        agent_path.join(RUN_SUMMARY_INDEX_FILE)
2508    }
2509
2510    fn run_summary_sidecar_root(agent_path: &Path) -> PathBuf {
2511        agent_path.join(RUN_SUMMARY_SIDECAR_DIR)
2512    }
2513
2514    fn run_summary_key(run_id: &str) -> [u8; RUN_SUMMARY_KEY_BYTES] {
2515        Sha256::digest(run_id.as_bytes()).into()
2516    }
2517
2518    fn run_summary_sidecar_path(agent_path: &Path, key: &[u8; 32]) -> PathBuf {
2519        let name = key
2520            .iter()
2521            .map(|byte| format!("{byte:02x}"))
2522            .collect::<String>();
2523        Self::run_summary_sidecar_root(agent_path).join(format!("{name}.json"))
2524    }
2525
2526    fn run_trace_corruption_sidecar_root(agent_path: &Path) -> PathBuf {
2527        agent_path.join(RUN_TRACE_CORRUPTION_SIDECAR_DIR)
2528    }
2529
2530    fn run_trace_corruption_sidecar_path(agent_path: &Path, key: &[u8; 32]) -> PathBuf {
2531        let name = key
2532            .iter()
2533            .map(|byte| format!("{byte:02x}"))
2534            .collect::<String>();
2535        Self::run_trace_corruption_sidecar_root(agent_path).join(format!("{name}.json"))
2536    }
2537
2538    fn read_run_trace_corruption_marker(
2539        &self,
2540        agent_path: &Path,
2541        agent_id: &str,
2542        run_id: &str,
2543    ) -> std::io::Result<Option<RunTraceCorruption>> {
2544        let key = Self::run_summary_key(run_id);
2545        let path = Self::run_trace_corruption_sidecar_path(agent_path, &key);
2546        let file = match car_secrets::open_private_read(&path) {
2547            Ok(file) => file,
2548            Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
2549            Err(error) => return Err(error),
2550        };
2551        let marker: RunTraceCorruptionMarker = serde_json::from_reader(file)
2552            .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
2553        if marker.run_id != run_id || marker.agent_id != agent_id {
2554            return Err(std::io::Error::new(
2555                std::io::ErrorKind::InvalidData,
2556                "run trace corruption marker identity does not match its path",
2557            ));
2558        }
2559        Ok(Some(marker.corruption))
2560    }
2561
2562    fn write_run_trace_corruption_marker(
2563        &self,
2564        agent_path: &Path,
2565        agent_id: &str,
2566        run_id: &str,
2567        corruption: &RunTraceCorruption,
2568    ) -> std::io::Result<()> {
2569        if let Some(existing) =
2570            self.read_run_trace_corruption_marker(agent_path, agent_id, run_id)?
2571        {
2572            if existing == *corruption {
2573                return Ok(());
2574            }
2575            return Err(std::io::Error::new(
2576                std::io::ErrorKind::InvalidData,
2577                "run trace corruption marker conflicts with an existing marker",
2578            ));
2579        }
2580        if self
2581            .failures
2582            .take(RunStoreFailurePoint::CorruptionMarkerWrite)
2583        {
2584            return Err(injected_storage_full());
2585        }
2586        let root = Self::run_trace_corruption_sidecar_root(agent_path);
2587        self.ensure_private_dir(&root)?;
2588        let path =
2589            Self::run_trace_corruption_sidecar_path(agent_path, &Self::run_summary_key(run_id));
2590        self.write_private_json_atomic(
2591            &root,
2592            &path,
2593            &RunTraceCorruptionMarker {
2594                run_id: run_id.to_string(),
2595                agent_id: agent_id.to_string(),
2596                corruption: corruption.clone(),
2597            },
2598        )
2599    }
2600
2601    fn persist_run_trace_corruption_marker(
2602        &self,
2603        agent_path: &Path,
2604        agent_id: &str,
2605        run_id: &str,
2606        corruption: &RunTraceCorruption,
2607    ) -> std::io::Result<()> {
2608        let error = match self
2609            .write_run_trace_corruption_marker(agent_path, agent_id, run_id, corruption)
2610        {
2611            Ok(()) => return Ok(()),
2612            Err(error) => error,
2613        };
2614        if self
2615            .read_run_trace_corruption_marker(agent_path, agent_id, run_id)
2616            .ok()
2617            .flatten()
2618            .is_some()
2619        {
2620            return Err(error);
2621        }
2622
2623        // If the independent marker cannot be made durable, invalidate the
2624        // derived healthy summary. The bounded list path then fails closed
2625        // instead of trusting stale state; the authoritative JSONL remains.
2626        let sidecar_root = Self::run_summary_sidecar_root(agent_path);
2627        let sidecar = Self::run_summary_sidecar_path(agent_path, &Self::run_summary_key(run_id));
2628        if self
2629            .failures
2630            .take(RunStoreFailurePoint::CorruptionSummaryInvalidate)
2631        {
2632            return Err(std::io::Error::new(
2633                error.kind(),
2634                format!("{error}; stale run summary invalidation also failed: permission denied"),
2635            ));
2636        }
2637        match std::fs::remove_file(&sidecar) {
2638            Ok(()) => sync_directory(&sidecar_root)?,
2639            Err(remove_error) if remove_error.kind() == std::io::ErrorKind::NotFound => {}
2640            Err(remove_error) => {
2641                return Err(std::io::Error::new(
2642                    error.kind(),
2643                    format!("{error}; stale run summary invalidation also failed: {remove_error}"),
2644                ));
2645            }
2646        }
2647        Err(error)
2648    }
2649
2650    fn remember_run_trace_corruption(
2651        &self,
2652        agent_id: &str,
2653        run_id: &str,
2654        corruption: &RunTraceCorruption,
2655    ) {
2656        self.trace_corruptions
2657            .lock()
2658            .expect("run trace corruption registry poisoned")
2659            .insert(
2660                (agent_id.to_string(), run_id.to_string()),
2661                corruption.clone(),
2662            );
2663    }
2664
2665    fn known_run_trace_corruption(
2666        &self,
2667        agent_id: &str,
2668        run_id: &str,
2669    ) -> Option<RunTraceCorruption> {
2670        self.trace_corruptions
2671            .lock()
2672            .expect("run trace corruption registry poisoned")
2673            .get(&(agent_id.to_string(), run_id.to_string()))
2674            .cloned()
2675    }
2676
2677    /// Publish corruption discovered by a trust-bearing read through the same
2678    /// marker/summary fail-closed boundary used by writers. The in-process
2679    /// registry is updated first so even marker ENOSPC cannot make a later
2680    /// read or write treat the run as healthy in this process.
2681    fn publish_strict_read_corruption(
2682        &self,
2683        agent_path: &Path,
2684        agent_id: &str,
2685        run_id: &str,
2686        error: std::io::Error,
2687    ) -> std::io::Error {
2688        let Some(corruption) = trace_corruption_from_error(&error) else {
2689            return error;
2690        };
2691        self.remember_run_trace_corruption(agent_id, run_id, &corruption);
2692        match self.persist_run_trace_corruption_marker(agent_path, agent_id, run_id, &corruption) {
2693            Ok(()) => error,
2694            Err(source) => summary_refresh_error(source, corruption),
2695        }
2696    }
2697
2698    fn apply_run_trace_corruption_marker(
2699        &self,
2700        agent_path: &Path,
2701        summary: &mut RunSummary,
2702    ) -> std::io::Result<()> {
2703        let corruption = match self.known_run_trace_corruption(&summary.agent_id, &summary.run_id) {
2704            Some(corruption) => Some(corruption),
2705            None => self.read_run_trace_corruption_marker(
2706                agent_path,
2707                &summary.agent_id,
2708                &summary.run_id,
2709            )?,
2710        };
2711        if let Some(corruption) = corruption {
2712            summary.status = RunStatus::Incomplete;
2713            summary.trace_corruption = Some(corruption);
2714        }
2715        Ok(())
2716    }
2717
2718    fn read_summary_sidecar(
2719        &self,
2720        agent_path: &Path,
2721        key: &[u8; 32],
2722    ) -> std::io::Result<RunSummary> {
2723        let path = Self::run_summary_sidecar_path(agent_path, key);
2724        let file = car_secrets::open_private_read(&path)?;
2725        let summary: RunSummary = serde_json::from_reader(file)
2726            .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
2727        if Self::run_summary_key(&summary.run_id) != *key {
2728            return Err(std::io::Error::new(
2729                std::io::ErrorKind::InvalidData,
2730                "run summary sidecar identity does not match its index key",
2731            ));
2732        }
2733        Ok(summary)
2734    }
2735
2736    fn write_summary_sidecar(
2737        &self,
2738        agent_path: &Path,
2739        summary: &RunSummary,
2740    ) -> std::io::Result<()> {
2741        if let Some(gate) = &self.summary_write_gate {
2742            gate.wait_if_armed();
2743        }
2744        if self.failures.take(RunStoreFailurePoint::SummaryWrite) {
2745            return Err(injected_storage_full());
2746        }
2747        let root = Self::run_summary_sidecar_root(agent_path);
2748        self.ensure_private_dir(&root)?;
2749        let path =
2750            Self::run_summary_sidecar_path(agent_path, &Self::run_summary_key(&summary.run_id));
2751        self.write_private_json_atomic(&root, &path, summary)
2752    }
2753
2754    fn write_summary_index(
2755        &self,
2756        agent_path: &Path,
2757        summaries: &[RunSummary],
2758    ) -> std::io::Result<()> {
2759        let mut ordered = summaries.to_vec();
2760        let mut next_sequence = ordered.iter().map(|row| row.sequence).max().unwrap_or(0);
2761        let mut missing = ordered
2762            .iter_mut()
2763            .filter(|row| row.sequence == 0)
2764            .collect::<Vec<_>>();
2765        missing.sort_by(|left, right| {
2766            left.started_at
2767                .cmp(&right.started_at)
2768                .then_with(|| right.run_id.cmp(&left.run_id))
2769        });
2770        for summary in missing {
2771            next_sequence = next_sequence.checked_add(1).ok_or_else(|| {
2772                std::io::Error::new(
2773                    std::io::ErrorKind::InvalidData,
2774                    "run summary sequence exhausted",
2775                )
2776            })?;
2777            summary.sequence = next_sequence;
2778        }
2779        ordered.sort_by(|left, right| right.sequence.cmp(&left.sequence));
2780        if ordered
2781            .windows(2)
2782            .any(|pair| pair[0].sequence == pair[1].sequence)
2783        {
2784            return Err(std::io::Error::new(
2785                std::io::ErrorKind::InvalidData,
2786                "run summary index contains duplicate sequences",
2787            ));
2788        }
2789        let sidecar_root = Self::run_summary_sidecar_root(agent_path);
2790        self.ensure_private_dir(&sidecar_root)?;
2791        let mut retained = std::collections::HashSet::new();
2792        let mut bytes = Vec::with_capacity(ordered.len() * RUN_SUMMARY_INDEX_RECORD_BYTES);
2793        for summary in &ordered {
2794            let key = Self::run_summary_key(&summary.run_id);
2795            retained.insert(key);
2796            self.write_summary_sidecar(agent_path, summary)?;
2797            bytes.extend_from_slice(&summary.sequence.to_be_bytes());
2798            bytes.extend_from_slice(&key);
2799        }
2800        if let Ok(entries) = std::fs::read_dir(&sidecar_root) {
2801            for entry in entries.flatten() {
2802                let path = entry.path();
2803                let Some(stem) = path.file_stem().and_then(|value| value.to_str()) else {
2804                    continue;
2805                };
2806                let is_retained = retained.iter().any(|key| {
2807                    key.iter()
2808                        .map(|byte| format!("{byte:02x}"))
2809                        .collect::<String>()
2810                        == stem
2811                });
2812                if !is_retained && path.extension().and_then(|value| value.to_str()) == Some("json")
2813                {
2814                    let _ = std::fs::remove_file(path);
2815                }
2816            }
2817        }
2818        let path = Self::run_summary_index_path(agent_path);
2819        let temp = agent_path.join(format!(
2820            ".run-summary-index-v2.{}.tmp",
2821            uuid::Uuid::new_v4().simple()
2822        ));
2823        let result = (|| {
2824            let mut file = self.create_private_file(&temp)?;
2825            file.write_all(&bytes)?;
2826            file.flush()?;
2827            file.sync_all()?;
2828            car_secrets::revalidate_private_path(&temp, &file)?;
2829            drop(file);
2830            car_secrets::atomic_replace_private_file(&temp, &path)?;
2831            sync_directory(agent_path)
2832        })();
2833        if result.is_err() {
2834            let _ = std::fs::remove_file(temp);
2835        }
2836        result
2837    }
2838
2839    fn refresh_run_summary(
2840        &self,
2841        agent_id: &str,
2842        run_id: &str,
2843        add_to_order: bool,
2844    ) -> std::io::Result<()> {
2845        let agent_path = self.open_agent_dir_for_read(agent_id)?;
2846        // Every summary refresh for an agent shares the index path as its
2847        // serialization key. Terminal refreshes update one sidecar; new-run
2848        // refreshes rewrite every retained sidecar plus the index. The lock
2849        // starts before the scan so a slower, older refresh cannot publish a
2850        // stale summary after a newer lifecycle boundary.
2851        let index_path = Self::run_summary_index_path(&agent_path);
2852        let index_lock = self.append_lock(&index_path);
2853        let _guard = index_lock
2854            .lock()
2855            .map_err(|_| std::io::Error::other("run summary index lock poisoned"))?;
2856        let path = self.run_path(agent_id, run_id);
2857        let scan = match summarize_file_checked(&path) {
2858            Ok(scan) => scan,
2859            Err(error) => {
2860                let Some(corruption) = trace_corruption_from_error(&error) else {
2861                    return Err(error);
2862                };
2863                self.remember_run_trace_corruption(agent_id, run_id, &corruption);
2864                self.persist_run_trace_corruption_marker(
2865                    &agent_path,
2866                    agent_id,
2867                    run_id,
2868                    &corruption,
2869                )
2870                .map_err(|source| summary_refresh_error(source, corruption))?;
2871                return Err(error);
2872            }
2873        };
2874        if let Some(corruption) = &scan.corruption {
2875            self.remember_run_trace_corruption(agent_id, run_id, corruption);
2876            self.persist_run_trace_corruption_marker(&agent_path, agent_id, run_id, corruption)
2877                .map_err(|source| summary_refresh_error(source, corruption.clone()))?;
2878        }
2879        let mut summary = scan.summary.ok_or_else(|| {
2880            let error = std::io::Error::new(
2881                std::io::ErrorKind::InvalidData,
2882                "durable trace cannot be summarized for the run index",
2883            );
2884            match scan.corruption.clone() {
2885                Some(corruption) => summary_refresh_error(error, corruption),
2886                None => error,
2887            }
2888        })?;
2889        self.apply_run_trace_corruption_marker(&agent_path, &mut summary)?;
2890        let corruption = summary.trace_corruption.clone();
2891        let update = (|| {
2892            if !add_to_order {
2893                let key = Self::run_summary_key(run_id);
2894                summary.sequence = self.read_summary_sidecar(&agent_path, &key)?.sequence;
2895                self.write_summary_sidecar(&agent_path, &summary)?;
2896                return Ok(());
2897            }
2898            let mut summaries = Vec::new();
2899            if let Ok(mut index) = car_secrets::open_private_read(&index_path) {
2900                let len = index.metadata()?.len() as usize;
2901                if !len.is_multiple_of(RUN_SUMMARY_INDEX_RECORD_BYTES) {
2902                    return Err(std::io::Error::new(
2903                        std::io::ErrorKind::InvalidData,
2904                        "run summary index has a partial record",
2905                    ));
2906                }
2907                let mut record = [0u8; RUN_SUMMARY_INDEX_RECORD_BYTES];
2908                while index.read_exact(&mut record).is_ok() {
2909                    let sequence = u64::from_be_bytes(record[..8].try_into().expect("eight bytes"));
2910                    let key: [u8; RUN_SUMMARY_KEY_BYTES] =
2911                        record[8..].try_into().expect("summary key bytes");
2912                    let existing = self.read_summary_sidecar(&agent_path, &key)?;
2913                    if existing.sequence != sequence {
2914                        return Err(std::io::Error::new(
2915                            std::io::ErrorKind::InvalidData,
2916                            "run summary sidecar sequence does not match its index record",
2917                        ));
2918                    }
2919                    if existing.run_id != run_id {
2920                        summaries.push(existing);
2921                    } else {
2922                        summary.sequence = sequence;
2923                    }
2924                }
2925            }
2926            summaries.push(summary.clone());
2927            self.write_summary_index(&agent_path, &summaries)
2928        })();
2929        preserve_summary_corruption(update, &corruption)?;
2930        reject_corrupt_summary(&summary)
2931    }
2932
2933    /// Ensure the `runs/` root exists with `0700` perms and is marked
2934    /// backup-excluded. Idempotent. Called lazily on the first append so
2935    /// constructing a `RunStore` is free (no disk touch until a run is
2936    /// actually recorded).
2937    fn ensure_root(&self) -> std::io::Result<()> {
2938        self.ensure_private_dir(&self.root)?;
2939        ensure_backup_excluded(&self.root)?;
2940        Ok(())
2941    }
2942
2943    /// Validate and harden an existing runs root without creating a missing
2944    /// store as a side effect of a replay/list request.
2945    fn open_root_for_read(&self) -> std::io::Result<()> {
2946        std::fs::symlink_metadata(&self.root)?;
2947        self.ensure_root()
2948    }
2949
2950    /// Ensure an agent's run dir exists `0700`.
2951    fn ensure_agent_dir(&self, agent_id: &str) -> std::io::Result<PathBuf> {
2952        self.ensure_root()?;
2953        let dir = self.root.join(sanitize(agent_id));
2954        self.ensure_private_dir(&dir)?;
2955        Ok(dir)
2956    }
2957
2958    /// Validate and harden an existing agent directory without creating an
2959    /// empty directory for an unknown replay/list key.
2960    fn open_agent_dir_for_read(&self, agent_id: &str) -> std::io::Result<PathBuf> {
2961        self.open_root_for_read()?;
2962        let dir = self.root.join(sanitize(agent_id));
2963        std::fs::symlink_metadata(&dir)?;
2964        self.ensure_private_dir(&dir)?;
2965        Ok(dir)
2966    }
2967
2968    /// Append one or more `RunRecord`s to a run's file, creating it `0600`
2969    /// on first write. Records are written one JSONL line each, in order.
2970    ///
2971    /// This is the single low-level flush primitive the wiring calls at
2972    /// each boundary: `RunStarted` on `runs.start`, `RunTurn`s as the
2973    /// recorder produces them, and the terminal `RunEnded`/`Incomplete` on
2974    /// `runs.complete`/disconnect.
2975    pub fn append_records(
2976        &self,
2977        agent_id: &str,
2978        run_id: &str,
2979        records: &[RunRecord],
2980    ) -> std::io::Result<()> {
2981        if records.is_empty() {
2982            return Ok(());
2983        }
2984        self.ensure_agent_dir(agent_id)?;
2985        let path = self.run_path(agent_id, run_id);
2986        let append_lock = self.append_lock(&path);
2987        let _guard = append_lock
2988            .lock()
2989            .map_err(|_| std::io::Error::other("run append lock poisoned"))?;
2990        let existed_before_open = path.exists();
2991        let mut file = self.open_private_append(&path)?;
2992        if self.failures.take(RunStoreFailurePoint::Write) {
2993            return Err(injected_storage_full());
2994        }
2995        append_jsonl_batch_to_path(&path, &mut file, records)?;
2996        self.durable_receipt(&path, &mut file, existed_before_open)
2997    }
2998
2999    /// Append the `RunStarted` line + create the run file (`runs.start`).
3000    pub fn write_started(&self, started: &car_proto::RunStarted) -> std::io::Result<()> {
3001        if started.run_id.is_empty()
3002            || started.agent_id.is_empty()
3003            || started.client_id.as_deref().is_none_or(str::is_empty)
3004        {
3005            return Err(std::io::Error::new(
3006                std::io::ErrorKind::InvalidInput,
3007                "active RunStarted requires client_id",
3008            ));
3009        }
3010        self.ensure_boundary_record(
3011            &started.agent_id,
3012            &started.run_id,
3013            RunRecord::Started(started.clone()),
3014        )?;
3015        self.refresh_run_summary(&started.agent_id, &started.run_id, true)
3016    }
3017
3018    /// Append `RunTurn` records (the recorder's per-proposal output).
3019    pub fn append_turns(
3020        &self,
3021        agent_id: &str,
3022        run_id: &str,
3023        turns: &[RunRecord],
3024    ) -> std::io::Result<()> {
3025        if let Some(gate) = &self.append_gate {
3026            gate.wait_if_armed();
3027        }
3028        self.append_records(agent_id, run_id, turns)?;
3029        if let Err(error) = self.refresh_run_summary(agent_id, run_id, false) {
3030            if is_trace_corruption_error(&error) {
3031                return Err(error);
3032            }
3033            tracing::error!(%agent_id, %run_id, %error, "run summary index update deferred until startup repair");
3034        }
3035        Ok(())
3036    }
3037
3038    /// Read the durable summary's corruption marker without materializing the
3039    /// trace. Active subscribe reads this off the async lock path, then
3040    /// revalidates live state before it trusts a snapshot or registers.
3041    pub fn run_trace_corruption_for(
3042        &self,
3043        agent_id: &str,
3044        run_id: &str,
3045    ) -> std::io::Result<Option<RunTraceCorruption>> {
3046        if let Some(gate) = &self.summary_read_gate {
3047            gate.wait_if_armed();
3048        }
3049        if let Some(corruption) = self.known_run_trace_corruption(agent_id, run_id) {
3050            return Ok(Some(corruption));
3051        }
3052        let agent_path = match self.open_agent_dir_for_read(agent_id) {
3053            Ok(path) => path,
3054            Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
3055            Err(error) => return Err(error),
3056        };
3057        if let Some(corruption) =
3058            self.read_run_trace_corruption_marker(&agent_path, agent_id, run_id)?
3059        {
3060            return Ok(Some(corruption));
3061        }
3062        let key = Self::run_summary_key(run_id);
3063        match self.read_summary_sidecar(&agent_path, &key) {
3064            Ok(summary) => {
3065                if summary.run_id != run_id || summary.agent_id != agent_id {
3066                    return Err(std::io::Error::new(
3067                        std::io::ErrorKind::InvalidData,
3068                        "run summary sidecar owner does not match requested run",
3069                    ));
3070                }
3071                Ok(summary.trace_corruption)
3072            }
3073            Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
3074            Err(error) => Err(error),
3075        }
3076    }
3077
3078    /// Ensure the exact action trace authenticated by a durable proposal
3079    /// finalization exists once before the terminal journal/receipt can
3080    /// advance. Existing exact rows make retries idempotent; partial or
3081    /// conflicting rows fail closed.
3082    pub fn ensure_proposal_turns(
3083        &self,
3084        pending: &PendingProposalFinalization,
3085    ) -> std::io::Result<ProposalTraceEnsure> {
3086        if let Some(gate) = &self.append_gate {
3087            gate.wait_if_armed();
3088        }
3089        pending.validate()?;
3090        let started = self.durable_run_started(&pending.run_id)?;
3091        match self.execution_marker(&pending.run_id)? {
3092            Some(marker) => pending.validate_provenance(&started, &marker)?,
3093            None => {
3094                let completed = self.completed_proposal(
3095                    &pending.run_id,
3096                    &pending.client_id,
3097                    pending.requested_policy_session_id.as_deref(),
3098                    &pending.original_submission,
3099                )?;
3100                if completed.as_ref().map(|receipt| &receipt.finalization) != Some(pending) {
3101                    return Err(std::io::Error::new(
3102                        std::io::ErrorKind::InvalidData,
3103                        "proposal trace has neither its execution marker nor an exact completed receipt",
3104                    ));
3105                }
3106            }
3107        }
3108        let mut expected = crate::run_trace::record_turns(
3109            &pending.final_proposal,
3110            &pending.proposal_result.results,
3111            0,
3112        );
3113        for record in &mut expected {
3114            let RunRecord::Turn(turn) = record else {
3115                return Err(std::io::Error::new(
3116                    std::io::ErrorKind::InvalidData,
3117                    "proposal trace generator returned a non-turn record",
3118                ));
3119            };
3120            if !crate::handler::enforce_proposal_turn_byte_cap(turn) {
3121                return Err(std::io::Error::new(
3122                    std::io::ErrorKind::InvalidData,
3123                    "proposal turn exceeds the durable byte limit without changing authenticated parameters",
3124                ));
3125            }
3126        }
3127        if expected.is_empty() {
3128            return Ok(ProposalTraceEnsure {
3129                records: expected,
3130                appended: false,
3131            });
3132        }
3133
3134        let path = self.run_path(&started.agent_id, &pending.run_id);
3135        let append_lock = self.append_lock(&path);
3136        let _guard = append_lock
3137            .lock()
3138            .map_err(|_| std::io::Error::other("proposal trace append lock poisoned"))?;
3139        let file = car_secrets::open_private_read(&path)?;
3140        let mut turn_count = 0usize;
3141        let mut matching = Vec::new();
3142        let scan = scan_records(&file, |record| {
3143            if let RunRecord::Turn(turn) = record {
3144                turn_count = turn_count.saturating_add(1);
3145                if turn.proposal_id.as_deref() == Some(pending.final_proposal_id.as_str()) {
3146                    matching.push(RunRecord::Turn(turn));
3147                }
3148            }
3149        });
3150        if let Err(error) = scan {
3151            let error = scan_error_to_io(error);
3152            let _ = self.refresh_run_summary(&started.agent_id, &pending.run_id, false);
3153            return Err(error);
3154        }
3155        car_secrets::revalidate_private_path(&path, &file)?;
3156
3157        let to_append = if !matching.is_empty() {
3158            let start = matching
3159                .first()
3160                .and_then(|record| match record {
3161                    RunRecord::Turn(turn) => Some(turn.index),
3162                    _ => None,
3163                })
3164                .unwrap_or(0);
3165            for (offset, record) in expected.iter_mut().enumerate() {
3166                if let RunRecord::Turn(turn) = record {
3167                    turn.index = start + offset;
3168                }
3169            }
3170            let exact_prefix = matching.len() <= expected.len()
3171                && matching.iter().zip(&expected).all(|(left, right)| {
3172                    serde_json::to_value(left).ok() == serde_json::to_value(right).ok()
3173                });
3174            if !exact_prefix || turn_count != start.saturating_add(matching.len()) {
3175                return Err(std::io::Error::new(
3176                    std::io::ErrorKind::InvalidData,
3177                    "durable proposal trace conflicts with final proposal identity or has later rows",
3178                ));
3179            }
3180            if matching.len() == expected.len() {
3181                return Ok(ProposalTraceEnsure {
3182                    records: matching,
3183                    appended: false,
3184                });
3185            }
3186            expected[matching.len()..].to_vec()
3187        } else {
3188            for (offset, record) in expected.iter_mut().enumerate() {
3189                if let RunRecord::Turn(turn) = record {
3190                    turn.index = turn_count + offset;
3191                }
3192            }
3193            expected.clone()
3194        };
3195
3196        if turn_count.saturating_add(to_append.len()) > crate::session::RECORD_TURNS_RUN_CEILING {
3197            return Err(std::io::Error::new(
3198                std::io::ErrorKind::InvalidData,
3199                "proposal trace would exceed the durable run turn ceiling",
3200            ));
3201        }
3202
3203        let existed_before_open = path.exists();
3204        let mut append = self.open_private_append(&path)?;
3205        if self.failures.take(RunStoreFailurePoint::Write) {
3206            return Err(injected_storage_full());
3207        }
3208        append_jsonl_batch_to_path(&path, &mut append, &to_append)?;
3209        self.durable_receipt(&path, &mut append, existed_before_open)?;
3210        if let Err(error) = self.refresh_run_summary(&started.agent_id, &pending.run_id, false) {
3211            if is_trace_corruption_error(&error) {
3212                return Err(error);
3213            }
3214            tracing::error!(run_id = %pending.run_id, %error, "proposal trace summary update deferred until startup repair");
3215        }
3216        Ok(ProposalTraceEnsure {
3217            records: expected,
3218            appended: true,
3219        })
3220    }
3221
3222    /// Append the terminal `RunEnded` line (`runs.complete` or the
3223    /// disconnect-`Incomplete` path).
3224    pub fn write_ended(&self, ended: &car_proto::RunEnded) -> std::io::Result<()> {
3225        if ended.run_id.is_empty()
3226            || ended.agent_id.is_empty()
3227            || ended.client_id.as_deref().is_none_or(str::is_empty)
3228            || ended.completion_digest.as_deref().is_none_or(str::is_empty)
3229        {
3230            return Err(std::io::Error::new(
3231                std::io::ErrorKind::InvalidInput,
3232                "active RunEnded requires client_id and completion_digest",
3233            ));
3234        }
3235        let expected_digest = crate::session::run_completion_digest(&ended.termination)
3236            .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
3237        if ended.completion_digest.as_deref() != Some(expected_digest.as_str()) {
3238            return Err(std::io::Error::new(
3239                std::io::ErrorKind::InvalidData,
3240                "RunEnded completion_digest does not match JCS RunTermination",
3241            ));
3242        }
3243        self.ensure_boundary_record(
3244            &ended.agent_id,
3245            &ended.run_id,
3246            RunRecord::Ended(ended.clone()),
3247        )?;
3248        self.refresh_run_summary(&ended.agent_id, &ended.run_id, false)
3249    }
3250
3251    /// Durably append the body-free cancellation request before attempting
3252    /// control. The exact key/preimage is idempotent; another key cannot race
3253    /// the active request.
3254    pub fn write_cancellation_requested(
3255        &self,
3256        agent_id: &str,
3257        requested: &car_proto::RunCancellationRequested,
3258    ) -> std::io::Result<()> {
3259        self.ensure_boundary_record(
3260            agent_id,
3261            &requested.run_id,
3262            RunRecord::CancellationRequested(requested.clone()),
3263        )?;
3264        self.refresh_run_summary(agent_id, &requested.run_id, false)
3265    }
3266
3267    /// Persist an unconfirmed cancellation receipt. Confirmed cancellation is
3268    /// represented by the terminal `RunEnded::Cancelled` row instead.
3269    pub fn write_cancellation_result(
3270        &self,
3271        agent_id: &str,
3272        result: &car_proto::RunCancelResponse,
3273    ) -> std::io::Result<()> {
3274        self.ensure_boundary_record(
3275            agent_id,
3276            &result.run_id,
3277            RunRecord::CancellationResult(result.clone()),
3278        )?;
3279        self.refresh_run_summary(agent_id, &result.run_id, false)
3280    }
3281
3282    fn append_lock(&self, path: &Path) -> Arc<Mutex<()>> {
3283        let mut locks = self
3284            .append_locks
3285            .lock()
3286            .expect("run append-lock registry poisoned");
3287        if let Some(lock) = locks.get(path).and_then(Weak::upgrade) {
3288            return lock;
3289        }
3290        locks.retain(|_, lock| lock.strong_count() > 0);
3291        let lock = Arc::new(Mutex::new(()));
3292        locks.insert(path.to_path_buf(), Arc::downgrade(&lock));
3293        lock
3294    }
3295
3296    fn ensure_boundary_record(
3297        &self,
3298        agent_id: &str,
3299        run_id: &str,
3300        record: RunRecord,
3301    ) -> std::io::Result<()> {
3302        self.ensure_agent_dir(agent_id)?;
3303        let path = self.run_path(agent_id, run_id);
3304        let append_lock = self.append_lock(&path);
3305        let _guard = append_lock
3306            .lock()
3307            .map_err(|_| std::io::Error::other("run append lock poisoned"))?;
3308        let existed_before_open = path.exists();
3309        let mut file = self.open_private_append(&path)?;
3310        car_secrets::revalidate_private_path(&path, &file)?;
3311        let existing = match load_records(&file) {
3312            Ok(records) => records,
3313            Err(error) => {
3314                // Publish the fail-closed summary marker before returning the
3315                // corruption error to the attempted boundary writer.
3316                let _ = self.refresh_run_summary(agent_id, run_id, false);
3317                return Err(error);
3318            }
3319        };
3320
3321        let exact_exists = match &record {
3322            RunRecord::Started(wanted) => {
3323                if let Some(found) = existing.iter().find_map(|row| match row {
3324                    RunRecord::Started(started) => Some(started),
3325                    _ => None,
3326                }) {
3327                    if found != wanted {
3328                        return Err(std::io::Error::new(
3329                            std::io::ErrorKind::AlreadyExists,
3330                            "run_id is already reserved by a different RunStarted owner/preimage",
3331                        ));
3332                    }
3333                    true
3334                } else {
3335                    if !existing.is_empty() {
3336                        return Err(std::io::Error::new(
3337                            std::io::ErrorKind::InvalidData,
3338                            "non-empty run trace is missing its RunStarted boundary",
3339                        ));
3340                    }
3341                    false
3342                }
3343            }
3344            RunRecord::Ended(wanted) => {
3345                let started = existing.iter().find_map(|row| match row {
3346                    RunRecord::Started(started) => Some(started),
3347                    _ => None,
3348                });
3349                let Some(started) = started else {
3350                    return Err(std::io::Error::new(
3351                        std::io::ErrorKind::InvalidData,
3352                        "RunEnded cannot precede RunStarted",
3353                    ));
3354                };
3355                if started.run_id != wanted.run_id
3356                    || started.agent_id != wanted.agent_id
3357                    || started.client_id != wanted.client_id
3358                {
3359                    return Err(std::io::Error::new(
3360                        std::io::ErrorKind::PermissionDenied,
3361                        "RunEnded owner does not match durable RunStarted",
3362                    ));
3363                }
3364                if let Some(found) = existing.iter().find_map(|row| match row {
3365                    RunRecord::Ended(ended) => Some(ended),
3366                    _ => None,
3367                }) {
3368                    if !same_run_ended(found, wanted) {
3369                        return Err(std::io::Error::new(
3370                            std::io::ErrorKind::AlreadyExists,
3371                            "run already has a different durable terminal",
3372                        ));
3373                    }
3374                    true
3375                } else {
3376                    false
3377                }
3378            }
3379            RunRecord::Turn(_) => false,
3380            RunRecord::CancellationRequested(wanted) => {
3381                let started = existing.iter().find_map(|row| match row {
3382                    RunRecord::Started(started) => Some(started),
3383                    _ => None,
3384                });
3385                let Some(started) = started else {
3386                    return Err(std::io::Error::new(
3387                        std::io::ErrorKind::InvalidData,
3388                        "cancellation request cannot precede RunStarted",
3389                    ));
3390                };
3391                if started.run_id != wanted.run_id || started.agent_id != agent_id {
3392                    return Err(std::io::Error::new(
3393                        std::io::ErrorKind::PermissionDenied,
3394                        "cancellation request owner does not match durable RunStarted",
3395                    ));
3396                }
3397                if existing
3398                    .iter()
3399                    .any(|row| matches!(row, RunRecord::Ended(_)))
3400                {
3401                    return Err(std::io::Error::new(
3402                        std::io::ErrorKind::AlreadyExists,
3403                        "run is already terminal",
3404                    ));
3405                }
3406                if let Some(found) = existing.iter().find_map(|row| match row {
3407                    RunRecord::CancellationRequested(row) => Some(row),
3408                    _ => None,
3409                }) {
3410                    if found != wanted {
3411                        return Err(std::io::Error::new(
3412                            std::io::ErrorKind::AlreadyExists,
3413                            "run already has a different cancellation request",
3414                        ));
3415                    }
3416                    true
3417                } else {
3418                    false
3419                }
3420            }
3421            RunRecord::CancellationResult(wanted) => {
3422                if wanted.status != car_proto::RunCancellationStatus::TerminationUnconfirmed
3423                    || wanted.terminal_digest.is_some()
3424                {
3425                    return Err(std::io::Error::new(
3426                        std::io::ErrorKind::InvalidInput,
3427                        "only body-free termination_unconfirmed receipts are nonterminal records",
3428                    ));
3429                }
3430                let request = existing.iter().find_map(|row| match row {
3431                    RunRecord::CancellationRequested(row) => Some(row),
3432                    _ => None,
3433                });
3434                let Some(request) = request else {
3435                    return Err(std::io::Error::new(
3436                        std::io::ErrorKind::InvalidData,
3437                        "cancellation result cannot precede its request",
3438                    ));
3439                };
3440                if request.run_id != wanted.run_id
3441                    || request.idempotency_key != wanted.idempotency_key
3442                    || request.reason_digest != wanted.reason_digest
3443                    || request.principal != wanted.principal
3444                    || request.action_id != wanted.action_id
3445                    || request.request_id != wanted.request_id
3446                {
3447                    return Err(std::io::Error::new(
3448                        std::io::ErrorKind::InvalidData,
3449                        "cancellation result conflicts with its durable request",
3450                    ));
3451                }
3452                if existing
3453                    .iter()
3454                    .any(|row| matches!(row, RunRecord::Ended(_)))
3455                {
3456                    return Err(std::io::Error::new(
3457                        std::io::ErrorKind::AlreadyExists,
3458                        "run is already terminal",
3459                    ));
3460                }
3461                if let Some(found) = existing.iter().find_map(|row| match row {
3462                    RunRecord::CancellationResult(row) => Some(row),
3463                    _ => None,
3464                }) {
3465                    if found != wanted {
3466                        return Err(std::io::Error::new(
3467                            std::io::ErrorKind::AlreadyExists,
3468                            "run already has a different cancellation result",
3469                        ));
3470                    }
3471                    true
3472                } else {
3473                    false
3474                }
3475            }
3476        };
3477
3478        if !exact_exists {
3479            if self.failures.take(RunStoreFailurePoint::Write) {
3480                return Err(injected_storage_full());
3481            }
3482            append_jsonl_batch_to_path(&path, &mut file, &[record])?;
3483        }
3484        self.durable_receipt(&path, &mut file, existed_before_open)
3485    }
3486
3487    fn durable_receipt(
3488        &self,
3489        path: &Path,
3490        file: &mut File,
3491        existed_before_open: bool,
3492    ) -> std::io::Result<()> {
3493        if self.failures.take(RunStoreFailurePoint::Flush) {
3494            return Err(std::io::Error::other("injected run-store flush failure"));
3495        }
3496        file.flush()?;
3497        if self.failures.take(RunStoreFailurePoint::Fsync) {
3498            return Err(std::io::Error::other("injected run-store fsync failure"));
3499        }
3500        file.sync_all()?;
3501        if !existed_before_open {
3502            if let Some(parent) = path.parent() {
3503                sync_directory(parent)?;
3504            }
3505        }
3506        car_secrets::revalidate_private_path(path, file)
3507    }
3508
3509    /// Load a run's full ordered trace from disk by `run_id` — the U5
3510    /// `runs.get_trace` read path. Works after a restart when memory is
3511    /// empty. Resolves `run_id -> agent_id` by scanning the tree, then
3512    /// reads the JSONL. A malformed unterminated final crash tail is ignored;
3513    /// committed corruption makes this compatibility wrapper return `None`.
3514    /// Returns `None` when no file exists for the `run_id`.
3515    pub fn get_run_trace(&self, run_id: &str) -> Option<Vec<RunRecord>> {
3516        self.get_run_trace_checked(run_id).ok().flatten()
3517    }
3518
3519    /// Strict replay read used by trust-bearing RPC surfaces. A malformed
3520    /// newline-terminated row rejects the trace; only a torn final row is
3521    /// ignored as an uncommitted crash tail.
3522    pub fn get_run_trace_checked(&self, run_id: &str) -> std::io::Result<Option<Vec<RunRecord>>> {
3523        let Some((path, file)) = self.resolve_run_file(run_id) else {
3524            return Ok(None);
3525        };
3526        let Some(agent_path) = path.parent() else {
3527            return Err(std::io::Error::new(
3528                std::io::ErrorKind::InvalidData,
3529                "run trace path has no owning agent directory",
3530            ));
3531        };
3532        let Some(agent_id) = agent_path.file_name().and_then(|value| value.to_str()) else {
3533            return Err(std::io::Error::new(
3534                std::io::ErrorKind::InvalidData,
3535                "run trace path has no valid owning agent id",
3536            ));
3537        };
3538        load_private_records(&path, &file)
3539            .map(Some)
3540            .map_err(|error| {
3541                self.publish_strict_read_corruption(agent_path, agent_id, run_id, error)
3542            })
3543    }
3544
3545    /// Load a run's trace given both keys (cheaper — no tree scan). Used
3546    /// by `list_runs` internally and available to callers that already
3547    /// know the owning agent.
3548    pub fn get_run_trace_for(&self, agent_id: &str, run_id: &str) -> Option<Vec<RunRecord>> {
3549        self.get_run_trace_for_checked(agent_id, run_id)
3550            .ok()
3551            .flatten()
3552    }
3553
3554    pub fn get_run_trace_for_checked(
3555        &self,
3556        agent_id: &str,
3557        run_id: &str,
3558    ) -> std::io::Result<Option<Vec<RunRecord>>> {
3559        let agent_path = match self.open_agent_dir_for_read(agent_id) {
3560            Ok(path) => path,
3561            Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
3562            Err(error) => return Err(error),
3563        };
3564        let path = self.run_path(agent_id, run_id);
3565        let file = match car_secrets::open_private_read(&path) {
3566            Ok(file) => file,
3567            Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
3568            Err(error) => return Err(error),
3569        };
3570        load_private_records(&path, &file)
3571            .map(Some)
3572            .map_err(|error| {
3573                self.publish_strict_read_corruption(&agent_path, agent_id, run_id, error)
3574            })
3575    }
3576
3577    /// Stream one bounded page from a run's JSONL trace. `limit + 1`
3578    /// records are retained only to determine whether a continuation exists;
3579    /// accumulated history is never materialized by the dashboard read path.
3580    pub fn get_run_trace_page_for(
3581        &self,
3582        agent_id: &str,
3583        run_id: &str,
3584        cursor: usize,
3585        limit: usize,
3586    ) -> std::io::Result<Option<(Vec<RunRecord>, Option<usize>)>> {
3587        let agent_path = match self.open_agent_dir_for_read(agent_id) {
3588            Ok(path) => path,
3589            Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
3590            Err(error) => return Err(error),
3591        };
3592        let path = self.run_path(agent_id, run_id);
3593        let file = match car_secrets::open_private_read(&path) {
3594            Ok(file) => file,
3595            Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
3596            Err(error) => return Err(error),
3597        };
3598        let mut records = Vec::with_capacity(limit.saturating_add(1));
3599        let mut valid_index = 0usize;
3600        if let Err(error) = scan_records(&file, |record| {
3601            if valid_index >= cursor && records.len() <= limit {
3602                records.push(record);
3603            }
3604            valid_index = valid_index.saturating_add(1);
3605        }) {
3606            let error = scan_error_to_io(error);
3607            return Err(self.publish_strict_read_corruption(&agent_path, agent_id, run_id, error));
3608        }
3609        car_secrets::revalidate_private_path(&path, &file)?;
3610        let has_more = records.len() > limit;
3611        records.truncate(limit);
3612        let next_cursor = has_more.then_some(cursor.saturating_add(records.len()));
3613        Ok(Some((records, next_cursor)))
3614    }
3615
3616    /// Stream a bounded turns-only page for `runs.subscribe`. The exact total
3617    /// comes from the durable summary sidecar, so returning `live_cursor` never
3618    /// requires materializing the trace.
3619    pub fn get_run_turn_page_for(
3620        &self,
3621        agent_id: &str,
3622        run_id: &str,
3623        cursor: usize,
3624        limit: usize,
3625    ) -> std::io::Result<Option<(Vec<RunRecord>, Option<usize>, usize, RunStatus)>> {
3626        let agent_path = match self.open_agent_dir_for_read(agent_id) {
3627            Ok(path) => path,
3628            Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
3629            Err(error) => return Err(error),
3630        };
3631        let key = Self::run_summary_key(run_id);
3632        let mut summary = match self.read_summary_sidecar(&agent_path, &key) {
3633            Ok(summary) if summary.run_id == run_id && summary.agent_id == agent_id => summary,
3634            Ok(_) => {
3635                return Err(std::io::Error::new(
3636                    std::io::ErrorKind::InvalidData,
3637                    "run summary identity does not match the requested trace",
3638                ))
3639            }
3640            Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
3641            Err(error) => return Err(error),
3642        };
3643        self.apply_run_trace_corruption_marker(&agent_path, &mut summary)?;
3644        reject_corrupt_summary(&summary)?;
3645        if cursor > summary.turn_count {
3646            return Err(std::io::Error::new(
3647                std::io::ErrorKind::InvalidInput,
3648                format!(
3649                    "runs.subscribe cursor {cursor} exceeds live_cursor {}",
3650                    summary.turn_count
3651                ),
3652            ));
3653        }
3654        let path = self.run_path(agent_id, run_id);
3655        let file = car_secrets::open_private_read(&path)?;
3656        let mut turns = Vec::with_capacity(limit.saturating_add(1));
3657        let mut turn_index = 0usize;
3658        if let Err(error) = scan_records(&file, |record| {
3659            if !matches!(record, RunRecord::Turn(_)) {
3660                return;
3661            }
3662            if turn_index >= cursor && turns.len() <= limit {
3663                turns.push(record);
3664            }
3665            turn_index = turn_index.saturating_add(1);
3666        }) {
3667            let error = scan_error_to_io(error);
3668            return Err(self.publish_strict_read_corruption(&agent_path, agent_id, run_id, error));
3669        }
3670        car_secrets::revalidate_private_path(&path, &file)?;
3671        if turn_index < summary.turn_count && turns.len() <= limit {
3672            return Err(std::io::Error::new(
3673                std::io::ErrorKind::InvalidData,
3674                "run trace ended before its durable summary turn count",
3675            ));
3676        }
3677        let has_more = turns.len() > limit;
3678        turns.truncate(limit);
3679        let next_cursor = has_more.then_some(cursor.saturating_add(turns.len()));
3680        Ok(Some((
3681            turns,
3682            next_cursor,
3683            summary.turn_count,
3684            summary.status,
3685        )))
3686    }
3687
3688    /// Read a newest-first run-summary page through the durable fixed-record
3689    /// order index. The page performs at most `limit + 1` sidecar reads and no
3690    /// run-directory enumeration or trace scan.
3691    pub fn list_runs_page(
3692        &self,
3693        agent_id: &str,
3694        cursor: usize,
3695        limit: usize,
3696    ) -> std::io::Result<(Vec<RunSummary>, Option<usize>)> {
3697        let agent_path = match self.open_agent_dir_for_read(agent_id) {
3698            Ok(path) => path,
3699            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
3700                return Ok((Vec::new(), None))
3701            }
3702            Err(error) => return Err(error),
3703        };
3704        let index_path = Self::run_summary_index_path(&agent_path);
3705        let mut index = car_secrets::open_private_read(&index_path)?;
3706        let byte_len = index.metadata()?.len() as usize;
3707        if !byte_len.is_multiple_of(RUN_SUMMARY_INDEX_RECORD_BYTES) {
3708            return Err(std::io::Error::new(
3709                std::io::ErrorKind::InvalidData,
3710                "run summary index has a partial record",
3711            ));
3712        }
3713        let total = byte_len / RUN_SUMMARY_INDEX_RECORD_BYTES;
3714        if total == 0 {
3715            return Ok((Vec::new(), None));
3716        }
3717
3718        // `0` means the current head. Every non-zero cursor is an exclusive
3719        // immutable sequence bound, so inserts at the head cannot shift a
3720        // continuation and duplicate/skip rows. Locate the first record with
3721        // sequence < cursor using O(log n) fixed-record seeks.
3722        let mut start = 0usize;
3723        if cursor != 0 {
3724            let cursor = u64::try_from(cursor).map_err(|_| {
3725                std::io::Error::new(std::io::ErrorKind::InvalidInput, "cursor overflow")
3726            })?;
3727            let mut low = 0usize;
3728            let mut high = total;
3729            while low < high {
3730                let mid = low + (high - low) / 2;
3731                index.seek(SeekFrom::Start(
3732                    (mid * RUN_SUMMARY_INDEX_RECORD_BYTES) as u64,
3733                ))?;
3734                let mut sequence = [0u8; 8];
3735                index.read_exact(&mut sequence)?;
3736                if u64::from_be_bytes(sequence) >= cursor {
3737                    low = mid + 1;
3738                } else {
3739                    high = mid;
3740                }
3741            }
3742            start = low;
3743        }
3744        if start >= total {
3745            return Ok((Vec::new(), None));
3746        }
3747        index.seek(SeekFrom::Start(
3748            (start * RUN_SUMMARY_INDEX_RECORD_BYTES) as u64,
3749        ))?;
3750        let count = limit.saturating_add(1).min(total - start);
3751        let mut summaries = Vec::with_capacity(count);
3752        for _ in 0..count {
3753            let mut record = [0u8; RUN_SUMMARY_INDEX_RECORD_BYTES];
3754            index.read_exact(&mut record)?;
3755            let sequence = u64::from_be_bytes(record[..8].try_into().expect("eight bytes"));
3756            let key: [u8; RUN_SUMMARY_KEY_BYTES] =
3757                record[8..].try_into().expect("summary key bytes");
3758            let mut summary = self.read_summary_sidecar(&agent_path, &key)?;
3759            if summary.agent_id != agent_id || summary.sequence != sequence {
3760                return Err(std::io::Error::new(
3761                    std::io::ErrorKind::InvalidData,
3762                    "run summary index does not match its durable sidecar",
3763                ));
3764            }
3765            self.apply_run_trace_corruption_marker(&agent_path, &mut summary)?;
3766            summaries.push(summary);
3767        }
3768        let has_more = summaries.len() > limit;
3769        summaries.truncate(limit);
3770        let next_cursor = if has_more {
3771            summaries
3772                .last()
3773                .map(|summary| {
3774                    usize::try_from(summary.sequence).map_err(|_| {
3775                        std::io::Error::new(
3776                            std::io::ErrorKind::InvalidData,
3777                            "run summary sequence cannot be represented by the wire cursor",
3778                        )
3779                    })
3780                })
3781                .transpose()?
3782        } else {
3783            None
3784        };
3785        Ok((summaries, next_cursor))
3786    }
3787
3788    /// List an agent's runs newest-first — the U5 `runs.list` read path.
3789    /// Each summary is built from the run file's records (head for
3790    /// `RunStarted`, tail for the terminal record, count of `Turn`s).
3791    /// Returns an empty Vec for an agent with no runs (the empty-state).
3792    pub fn list_runs(&self, agent_id: &str) -> Vec<RunSummary> {
3793        let Ok(dir) = self.open_agent_dir_for_read(agent_id) else {
3794            return Vec::new();
3795        };
3796        let mut out = Vec::new();
3797        let Ok(entries) = std::fs::read_dir(&dir) else {
3798            return out;
3799        };
3800        for entry in entries {
3801            let Ok(entry) = entry else {
3802                return Vec::new();
3803            };
3804            let path = entry.path();
3805            if path.extension().and_then(|e| e.to_str()) != Some("jsonl") {
3806                continue;
3807            }
3808            if let Some(mut summary) = summarize_file(&path) {
3809                if self
3810                    .apply_run_trace_corruption_marker(&dir, &mut summary)
3811                    .is_err()
3812                {
3813                    return Vec::new();
3814                }
3815                out.push(summary);
3816            }
3817        }
3818        // Newest first by start time.
3819        out.sort_by(|a, b| {
3820            b.started_at
3821                .cmp(&a.started_at)
3822                .then_with(|| a.run_id.cmp(&b.run_id))
3823        });
3824        out
3825    }
3826
3827    /// Visit only the durable lifecycle boundaries for one trace at a time.
3828    /// Startup reconciliation needs neither turns nor a process-wide snapshot;
3829    /// streaming each file prevents accumulated newsroom history from being
3830    /// materialized before the daemon can listen and schedule.
3831    pub fn visit_run_boundaries<F>(&self, mut visitor: F)
3832    where
3833        F: FnMut(
3834            car_proto::RunStarted,
3835            Option<car_proto::RunEnded>,
3836            Option<car_proto::RunCancellationRequested>,
3837            Option<car_proto::RunCancelResponse>,
3838        ),
3839    {
3840        if self.open_root_for_read().is_err() {
3841            return;
3842        }
3843        let Ok(agent_dirs) = std::fs::read_dir(&self.root) else {
3844            return;
3845        };
3846        for agent in agent_dirs.flatten() {
3847            let Ok(file_type) = agent.file_type() else {
3848                continue;
3849            };
3850            if !file_type.is_dir() || file_type.is_symlink() {
3851                continue;
3852            }
3853            let Ok(entries) = std::fs::read_dir(agent.path()) else {
3854                continue;
3855            };
3856            for entry in entries.flatten() {
3857                let path = entry.path();
3858                if path.extension().and_then(|ext| ext.to_str()) != Some("jsonl") {
3859                    continue;
3860                }
3861                let Ok(file) = car_secrets::open_private_read(&path) else {
3862                    continue;
3863                };
3864                if let Some((started, ended, requested, result)) =
3865                    load_private_run_boundaries(&path, &file)
3866                {
3867                    visitor(started, ended, requested, result);
3868                }
3869            }
3870        }
3871    }
3872
3873    /// Resolve `run_id -> path` by scanning `runs/*/{run_id}.jsonl`. U5's
3874    /// `runs.get_trace` takes only a `run_id`, so the owning agent must be
3875    /// discovered. Returns the first match (run ids are uuids — unique).
3876    fn resolve_run_file(&self, run_id: &str) -> Option<(PathBuf, File)> {
3877        self.open_root_for_read().ok()?;
3878        let file_name = format!("{}.jsonl", sanitize(run_id));
3879        let agent_dirs = std::fs::read_dir(&self.root).ok()?;
3880        for agent in agent_dirs {
3881            let agent = agent.ok()?;
3882            let file_type = agent.file_type().ok()?;
3883            if !file_type.is_dir() || file_type.is_symlink() {
3884                continue;
3885            }
3886            car_secrets::ensure_private_dir(&agent.path()).ok()?;
3887            let candidate = agent.path().join(&file_name);
3888            match car_secrets::open_private_read(&candidate) {
3889                Ok(file) => return Some((candidate, file)),
3890                Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue,
3891                Err(_) => return None,
3892            }
3893        }
3894        None
3895    }
3896
3897    /// Resolve the owning `agent_id` for a `run_id` from disk. Mirrors
3898    /// [`Self::resolve_run_file`] but returns the agent dir name — U5's
3899    /// authorization check (KTD10) needs `run_id -> agent_id` to verify
3900    /// ownership before serving a trace.
3901    pub fn agent_for_run(&self, run_id: &str) -> Option<String> {
3902        let (path, _file) = self.resolve_run_file(run_id)?;
3903        path.parent()
3904            .and_then(Path::file_name)
3905            .and_then(|s| s.to_str())
3906            .map(str::to_string)
3907    }
3908
3909    /// Retention GC (R6) — call on daemon boot. Per agent: keep the most
3910    /// recent `max_per_agent` **completed** runs and drop any completed
3911    /// run older than `max_age_days`, whichever is more restrictive. An
3912    /// in-progress run (no terminal record) is NEVER evicted. Returns the
3913    /// number of run files removed.
3914    pub fn gc(&self) -> usize {
3915        let mut removed = 0;
3916        if self.open_root_for_read().is_err() {
3917            return 0;
3918        }
3919        let Ok(agent_dirs) = std::fs::read_dir(&self.root) else {
3920            return 0;
3921        };
3922        let cutoff = self.retention.age_cutoff(Utc::now());
3923        for agent in agent_dirs {
3924            let Ok(agent) = agent else {
3925                return removed;
3926            };
3927            let Ok(file_type) = agent.file_type() else {
3928                return removed;
3929            };
3930            if !file_type.is_dir() || file_type.is_symlink() {
3931                continue;
3932            }
3933            let agent_path = agent.path();
3934            if car_secrets::ensure_private_dir(&agent_path).is_err() {
3935                return removed;
3936            }
3937            removed += self.gc_agent_dir(&agent_path, cutoff);
3938        }
3939        removed
3940    }
3941
3942    /// Adopt crash-orphaned runs at boot (FIX 4). A daemon crash mid-run
3943    /// leaves an on-disk run with `RunStarted` (+ `Turn`s) but no terminal
3944    /// `RunEnded`, so it reads `InProgress` forever and the GC — which never
3945    /// evicts an in-progress run — can never reclaim it. The file leaks
3946    /// across every crash.
3947    ///
3948    /// This runs at store construction/boot, BEFORE `gc()`. At that moment
3949    /// the in-memory `runs` map is always empty, so any run that is
3950    /// `InProgress` on disk cannot have a live harness writing to it — it is
3951    /// necessarily a crashed prior process. We append an `Incomplete`
3952    /// terminal `RunEnded` marker to adopt it, making it terminal and thus
3953    /// age-GC-eligible (so a later `gc()` in the same boot can reclaim it).
3954    ///
3955    /// Returns the number of runs adopted. Best-effort: an unwritable file is
3956    /// skipped rather than failing startup.
3957    pub fn adopt_orphans(&self) -> usize {
3958        let mut adopted = 0;
3959        if self.open_root_for_read().is_err() {
3960            return 0;
3961        }
3962        let Ok(agent_dirs) = std::fs::read_dir(&self.root) else {
3963            return 0;
3964        };
3965        let now = Utc::now();
3966        for agent in agent_dirs {
3967            let Ok(agent) = agent else {
3968                return adopted;
3969            };
3970            let Ok(file_type) = agent.file_type() else {
3971                return adopted;
3972            };
3973            if !file_type.is_dir() || file_type.is_symlink() {
3974                continue;
3975            }
3976            let agent_path = agent.path();
3977            if car_secrets::ensure_private_dir(&agent_path).is_err() {
3978                return adopted;
3979            }
3980            let Ok(entries) = std::fs::read_dir(&agent_path) else {
3981                continue;
3982            };
3983            for entry in entries {
3984                let Ok(entry) = entry else {
3985                    return adopted;
3986                };
3987                let path = entry.path();
3988                if path.extension().and_then(|e| e.to_str()) != Some("jsonl") {
3989                    continue;
3990                }
3991                let Some(summary) = summarize_file(&path) else {
3992                    continue;
3993                };
3994                if summary.status != RunStatus::InProgress {
3995                    continue;
3996                }
3997                // A pre-dispatch marker with no exact result means the daemon
3998                // may have died after an irreversible effect. Preserve the
3999                // open run as quarantined/outcome-unknown; an automatic
4000                // Incomplete terminal would hide the unresolved proposal and
4001                // invite an unsafe retry.
4002                let pending_state = self.pending_proposal(&summary.run_id);
4003                let execution_state = self.execution_marker(&summary.run_id);
4004                if !matches!(pending_state, Ok(None)) || !matches!(execution_state, Ok(None)) {
4005                    continue;
4006                }
4007                // No terminal record + memory empty at boot => crash orphan.
4008                let client_id = self
4009                    .get_run_trace_for(&summary.agent_id, &summary.run_id)
4010                    .and_then(|records| {
4011                        records.into_iter().find_map(|record| match record {
4012                            RunRecord::Started(started) => started.client_id,
4013                            _ => None,
4014                        })
4015                    });
4016                let termination = RunTermination::Incomplete;
4017                let completion_digest = crate::session::run_completion_digest(&termination).ok();
4018                let incomplete = RunRecord::Ended(car_proto::RunEnded {
4019                    run_id: summary.run_id.clone(),
4020                    client_id,
4021                    agent_id: summary.agent_id.clone(),
4022                    termination,
4023                    completion_digest,
4024                    ended_at: now,
4025                });
4026                if self
4027                    .append_records(&summary.agent_id, &summary.run_id, &[incomplete])
4028                    .is_ok()
4029                {
4030                    adopted += 1;
4031                }
4032            }
4033        }
4034        adopted
4035    }
4036
4037    /// GC one agent's run dir against the retention caps.
4038    fn gc_agent_dir(&self, agent_path: &Path, age_cutoff: Option<DateTime<Utc>>) -> usize {
4039        // Collect (path, summary) for every run file, ignoring unreadable
4040        // ones (a malformed file with no RunStarted can't be summarized;
4041        // leave it rather than risk evicting something we can't classify).
4042        let mut runs: Vec<(PathBuf, RunSummary)> = Vec::new();
4043        let Ok(entries) = std::fs::read_dir(agent_path) else {
4044            return 0;
4045        };
4046        for entry in entries {
4047            let Ok(entry) = entry else {
4048                return 0;
4049            };
4050            let path = entry.path();
4051            if path.extension().and_then(|e| e.to_str()) != Some("jsonl") {
4052                continue;
4053            }
4054            let file_run_id = path
4055                .file_stem()
4056                .and_then(|value| value.to_str())
4057                .unwrap_or_default();
4058            let key = Self::run_summary_key(file_run_id);
4059            let existing = self.read_summary_sidecar(agent_path, &key).ok();
4060            let scan = summarize_file_checked(&path);
4061            let mut summary = match scan {
4062                Ok(scan) => match scan.summary {
4063                    Some(summary) => {
4064                        if let Some(corruption) = &scan.corruption {
4065                            self.remember_run_trace_corruption(
4066                                &summary.agent_id,
4067                                file_run_id,
4068                                corruption,
4069                            );
4070                        }
4071                        summary
4072                    }
4073                    None => {
4074                        let Some(mut existing) = existing.clone() else {
4075                            continue;
4076                        };
4077                        let corruption = scan
4078                            .corruption
4079                            .or_else(|| {
4080                                self.known_run_trace_corruption(&existing.agent_id, file_run_id)
4081                            })
4082                            .or_else(|| {
4083                                self.read_run_trace_corruption_marker(
4084                                    agent_path,
4085                                    &existing.agent_id,
4086                                    file_run_id,
4087                                )
4088                                .ok()
4089                                .flatten()
4090                            });
4091                        let Some(corruption) = corruption else {
4092                            continue;
4093                        };
4094                        self.remember_run_trace_corruption(
4095                            &existing.agent_id,
4096                            file_run_id,
4097                            &corruption,
4098                        );
4099                        existing.status = RunStatus::Incomplete;
4100                        existing.trace_corruption = Some(corruption);
4101                        existing
4102                    }
4103                },
4104                Err(error) => {
4105                    let Some(corruption) = trace_corruption_from_error(&error) else {
4106                        continue;
4107                    };
4108                    let Some(mut existing) = existing.clone() else {
4109                        continue;
4110                    };
4111                    self.remember_run_trace_corruption(
4112                        &existing.agent_id,
4113                        file_run_id,
4114                        &corruption,
4115                    );
4116                    existing.status = RunStatus::Incomplete;
4117                    existing.trace_corruption = Some(corruption);
4118                    existing
4119                }
4120            };
4121            if let Some(existing) = existing {
4122                summary.sequence = existing.sequence;
4123            }
4124            if self
4125                .apply_run_trace_corruption_marker(agent_path, &mut summary)
4126                .is_err()
4127            {
4128                continue;
4129            }
4130            runs.push((path, summary));
4131        }
4132        // Newest-first so the most recent completed runs rank first.
4133        runs.sort_by(|a, b| b.1.started_at.cmp(&a.1.started_at));
4134
4135        let mut removed = 0;
4136        // Rank among COMPLETED/Incomplete runs only — an in-progress run
4137        // must not consume a keeper slot, so the count cap is measured by
4138        // completed-run rank, not the combined sorted index (R6).
4139        let mut completed_rank = 0usize;
4140        for (path, summary) in runs.iter() {
4141            // Corruption is durable audit evidence, not an ordinary incomplete
4142            // run. Never age/count-evict it until an operator repairs it.
4143            if summary.trace_corruption.is_some() {
4144                continue;
4145            }
4146            // NEVER evict an in-progress run — it has no terminal record
4147            // and a live harness may still be writing to it (R6).
4148            if matches!(
4149                summary.status,
4150                RunStatus::InProgress | RunStatus::CancellationPending
4151            ) {
4152                continue;
4153            }
4154            let over_count = self
4155                .retention
4156                .count_cap()
4157                .is_some_and(|cap| completed_rank >= cap);
4158            completed_rank += 1;
4159            // Age the run by its TERMINAL time, not its start. A long run
4160            // that started >max_age_days ago but completed recently is still
4161            // a recent result and must not be evicted (FIX 2). Completed and
4162            // Incomplete runs always have an `ended_at`; fall back to
4163            // `started_at` only if a terminal record somehow lacks one.
4164            let term_time = summary.ended_at.unwrap_or(summary.started_at);
4165            let too_old = age_cutoff.is_some_and(|cut| term_time < cut);
4166            if (over_count || too_old) && std::fs::remove_file(path).is_ok() {
4167                removed += 1;
4168            }
4169        }
4170        let retained: Vec<RunSummary> = runs
4171            .into_iter()
4172            .filter_map(|(path, summary)| path.exists().then_some(summary))
4173            .collect();
4174        if let Err(error) = self.write_summary_index(agent_path, &retained) {
4175            tracing::error!(path = %agent_path.display(), %error, "run summary index startup repair failed");
4176        }
4177        removed
4178    }
4179}
4180
4181fn same_run_ended(left: &car_proto::RunEnded, right: &car_proto::RunEnded) -> bool {
4182    serde_json::to_value(left).ok() == serde_json::to_value(right).ok()
4183}
4184
4185/// Serialize and append a complete batch through one already-validated file
4186/// descriptor. Tail inspection and repair deliberately use the same
4187/// descriptor as the `O_APPEND` write: swapping the path after open cannot
4188/// redirect any bytes into a symlink target or replacement inode.
4189fn append_jsonl_batch_on_descriptor(file: &mut File, records: &[RunRecord]) -> std::io::Result<()> {
4190    car_secrets::revalidate_private_file(file)?;
4191    // FIX 6/7: build the WHOLE batch into one buffer and write it with a
4192    // single `write_all`. With `O_APPEND` a single write is positioned and
4193    // appended atomically relative to other appenders, so two concurrent
4194    // batches for the same file can't interleave mid-record. A leading '\n'
4195    // repairs a torn tail as part of the same atomic write.
4196    let mut buf = Vec::new();
4197    if last_byte_is_not_newline(file)? {
4198        buf.push(b'\n');
4199    }
4200    for rec in records {
4201        let line = serde_json::to_string(rec)
4202            .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
4203        buf.extend_from_slice(line.as_bytes());
4204        buf.push(b'\n');
4205    }
4206    file.write_all(&buf)?;
4207    car_secrets::revalidate_private_file(file)
4208}
4209
4210/// Bind the descriptor-safe append to the canonical path before and after the
4211/// write. The second check converts a concurrent rename/substitution into a
4212/// failed append receipt while guaranteeing the replacement path was never
4213/// opened for writing.
4214fn append_jsonl_batch_to_path(
4215    path: &Path,
4216    file: &mut File,
4217    records: &[RunRecord],
4218) -> std::io::Result<()> {
4219    car_secrets::revalidate_private_path(path, file)?;
4220    append_jsonl_batch_on_descriptor(file, records)?;
4221    car_secrets::revalidate_private_path(path, file)
4222}
4223
4224/// Return `true` when the opened file's last byte is not `\n` — i.e. the
4225/// previous append was torn. An empty file returns `false`. The caller passes
4226/// the already-validated read+append descriptor so no path lookup occurs
4227/// between inspection and repair.
4228fn last_byte_is_not_newline(file: &mut File) -> std::io::Result<bool> {
4229    let len = file.seek(SeekFrom::End(0))?;
4230    if len == 0 {
4231        return Ok(false);
4232    }
4233    file.seek(SeekFrom::End(-1))?;
4234    let mut buf = [0u8; 1];
4235    file.read_exact(&mut buf)?;
4236    Ok(buf[0] != b'\n')
4237}
4238
4239#[derive(Debug)]
4240enum RecordScanError {
4241    Io(std::io::Error),
4242    Malformed { line: usize, detail: String },
4243}
4244
4245#[derive(Debug)]
4246struct StrictRunTraceCorruptionError {
4247    corruption: RunTraceCorruption,
4248    detail: String,
4249}
4250
4251impl std::fmt::Display for StrictRunTraceCorruptionError {
4252    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
4253        write!(
4254            formatter,
4255            "malformed run trace record at line {}: {}",
4256            self.corruption.line, self.detail
4257        )
4258    }
4259}
4260
4261impl std::error::Error for StrictRunTraceCorruptionError {}
4262
4263fn scan_error_to_io(error: RecordScanError) -> std::io::Error {
4264    match error {
4265        RecordScanError::Io(error) => error,
4266        RecordScanError::Malformed { line, detail } => std::io::Error::new(
4267            std::io::ErrorKind::InvalidData,
4268            StrictRunTraceCorruptionError {
4269                corruption: RunTraceCorruption {
4270                    kind: RunTraceCorruptionKind::MalformedRecord,
4271                    line,
4272                },
4273                detail,
4274            },
4275        ),
4276    }
4277}
4278
4279/// Visit every committed JSONL row. A malformed newline-terminated row is
4280/// durable corruption and rejects the trace. A malformed final row without a
4281/// newline is an uncommitted crash tail and is ignored.
4282fn scan_records<F>(file: &File, mut visit: F) -> Result<(), RecordScanError>
4283where
4284    F: FnMut(RunRecord),
4285{
4286    let cloned = file.try_clone().map_err(RecordScanError::Io)?;
4287    let mut reader = std::io::BufReader::new(cloned);
4288    let mut bytes = Vec::new();
4289    let mut line = 0usize;
4290    loop {
4291        bytes.clear();
4292        let read = reader
4293            .read_until(b'\n', &mut bytes)
4294            .map_err(RecordScanError::Io)?;
4295        if read == 0 {
4296            break;
4297        }
4298        line = line.saturating_add(1);
4299        let terminated = bytes.last() == Some(&b'\n');
4300        if terminated {
4301            bytes.pop();
4302            if bytes.last() == Some(&b'\r') {
4303                bytes.pop();
4304            }
4305        }
4306        if bytes.iter().all(u8::is_ascii_whitespace) {
4307            continue;
4308        }
4309        match serde_json::from_slice::<RunRecord>(&bytes) {
4310            Ok(record) => visit(record),
4311            Err(_) if !terminated => break,
4312            Err(error) => {
4313                return Err(RecordScanError::Malformed {
4314                    line,
4315                    detail: error.to_string(),
4316                })
4317            }
4318        }
4319    }
4320    Ok(())
4321}
4322
4323fn load_records(file: &File) -> std::io::Result<Vec<RunRecord>> {
4324    let mut records = Vec::new();
4325    scan_records(file, |record| records.push(record)).map_err(scan_error_to_io)?;
4326    Ok(records)
4327}
4328
4329fn load_private_records(path: &Path, file: &File) -> std::io::Result<Vec<RunRecord>> {
4330    let records = load_records(file)?;
4331    car_secrets::revalidate_private_path(path, file)?;
4332    Ok(records)
4333}
4334
4335/// Stream a trace and retain only its lifecycle boundary rows. This preserves
4336/// the JSONL reader's torn-line tolerance without allocating every recorded
4337/// turn during daemon startup.
4338fn load_private_run_boundaries(
4339    path: &Path,
4340    file: &File,
4341) -> Option<(
4342    car_proto::RunStarted,
4343    Option<car_proto::RunEnded>,
4344    Option<car_proto::RunCancellationRequested>,
4345    Option<car_proto::RunCancelResponse>,
4346)> {
4347    let mut started = None;
4348    let mut ended = None;
4349    let mut requested = None;
4350    let mut result = None;
4351    scan_records(file, |record| match record {
4352        RunRecord::Started(row) if started.is_none() => started = Some(row),
4353        RunRecord::Ended(row) if ended.is_none() => ended = Some(row),
4354        RunRecord::CancellationRequested(row) if requested.is_none() => requested = Some(row),
4355        RunRecord::CancellationResult(row) if result.is_none() => result = Some(row),
4356        _ => {}
4357    })
4358    .ok()?;
4359    car_secrets::revalidate_private_path(path, file).ok()?;
4360    started.map(|started| (started, ended, requested, result))
4361}
4362
4363fn reject_corrupt_summary(summary: &RunSummary) -> std::io::Result<()> {
4364    if let Some(corruption) = &summary.trace_corruption {
4365        return Err(std::io::Error::new(
4366            std::io::ErrorKind::InvalidData,
4367            format!("malformed run trace record at line {}", corruption.line),
4368        ));
4369    }
4370    Ok(())
4371}
4372
4373#[derive(Debug)]
4374struct RunSummaryRefreshError {
4375    source: std::io::Error,
4376    corruption: RunTraceCorruption,
4377}
4378
4379impl std::fmt::Display for RunSummaryRefreshError {
4380    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
4381        write!(
4382            formatter,
4383            "malformed run trace record at line {}; summary persistence also failed: {}",
4384            self.corruption.line, self.source
4385        )
4386    }
4387}
4388
4389impl std::error::Error for RunSummaryRefreshError {
4390    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
4391        Some(&self.source)
4392    }
4393}
4394
4395fn summary_refresh_error(source: std::io::Error, corruption: RunTraceCorruption) -> std::io::Error {
4396    std::io::Error::new(source.kind(), RunSummaryRefreshError { source, corruption })
4397}
4398
4399fn preserve_summary_corruption<T>(
4400    result: std::io::Result<T>,
4401    corruption: &Option<RunTraceCorruption>,
4402) -> std::io::Result<T> {
4403    match corruption {
4404        Some(corruption) => {
4405            result.map_err(|source| summary_refresh_error(source, corruption.clone()))
4406        }
4407        None => result,
4408    }
4409}
4410
4411pub(crate) fn trace_corruption_from_error(error: &std::io::Error) -> Option<RunTraceCorruption> {
4412    let source = error.get_ref()?;
4413    if let Some(failure) = source.downcast_ref::<RunSummaryRefreshError>() {
4414        return Some(failure.corruption.clone());
4415    }
4416    source
4417        .downcast_ref::<StrictRunTraceCorruptionError>()
4418        .map(|failure| failure.corruption.clone())
4419}
4420
4421pub(crate) fn is_trace_corruption_error(error: &std::io::Error) -> bool {
4422    trace_corruption_from_error(error).is_some()
4423        || (error.kind() == std::io::ErrorKind::InvalidData
4424            && error.to_string().contains("malformed run trace record"))
4425}
4426
4427fn summary_from_records(
4428    started: Option<car_proto::RunStarted>,
4429    ended: Option<car_proto::RunEnded>,
4430    turn_count: usize,
4431    cancellation_pending: bool,
4432    trace_corruption: Option<RunTraceCorruption>,
4433) -> Option<RunSummary> {
4434    let started = started?;
4435    let (status, ended_at) = if trace_corruption.is_some() {
4436        (RunStatus::Incomplete, None)
4437    } else {
4438        match &ended {
4439            Some(e) => {
4440                let status = match &e.termination {
4441                    RunTermination::Outcome { .. } => RunStatus::Completed,
4442                    RunTermination::Incomplete => RunStatus::Incomplete,
4443                    RunTermination::Cancelled { .. } => RunStatus::Cancelled,
4444                };
4445                (status, Some(e.ended_at))
4446            }
4447            None if cancellation_pending => (RunStatus::CancellationPending, None),
4448            None => (RunStatus::InProgress, None),
4449        }
4450    };
4451    Some(RunSummary {
4452        run_id: started.run_id,
4453        agent_id: started.agent_id,
4454        intent: started.intent,
4455        started_at: started.started_at,
4456        ended_at,
4457        status,
4458        turn_count,
4459        sequence: 0,
4460        trace_corruption,
4461    })
4462}
4463
4464/// Build a [`RunSummary`] from a run's JSONL file: `RunStarted` from the
4465/// first valid record, terminal status from the last, and `turn_count`
4466/// from the `Turn`s in between. Returns `None` when the file has no
4467/// `RunStarted` (it can't be keyed/summarized).
4468struct RunSummaryScan {
4469    summary: Option<RunSummary>,
4470    corruption: Option<RunTraceCorruption>,
4471}
4472
4473fn summarize_file_checked(path: &Path) -> std::io::Result<RunSummaryScan> {
4474    let file = car_secrets::open_private_read(path)?;
4475    let mut started: Option<car_proto::RunStarted> = None;
4476    let mut ended: Option<car_proto::RunEnded> = None;
4477    let mut turn_count = 0usize;
4478    let mut cancellation_pending = false;
4479    let scan = scan_records(&file, |record| match record {
4480        RunRecord::Started(s) => started = Some(s),
4481        RunRecord::Ended(e) => ended = Some(e),
4482        RunRecord::Turn(_) => turn_count += 1,
4483        RunRecord::CancellationRequested(_) | RunRecord::CancellationResult(_) => {
4484            cancellation_pending = true
4485        }
4486    });
4487    let trace_corruption = match scan {
4488        Ok(()) => None,
4489        Err(RecordScanError::Malformed { line, .. }) => Some(RunTraceCorruption {
4490            kind: RunTraceCorruptionKind::MalformedRecord,
4491            line,
4492        }),
4493        Err(RecordScanError::Io(error)) => return Err(error),
4494    };
4495    preserve_summary_corruption(
4496        car_secrets::revalidate_private_path(path, &file),
4497        &trace_corruption,
4498    )?;
4499    let summary = summary_from_records(
4500        started,
4501        ended,
4502        turn_count,
4503        cancellation_pending,
4504        trace_corruption.clone(),
4505    );
4506    Ok(RunSummaryScan {
4507        summary,
4508        corruption: trace_corruption,
4509    })
4510}
4511
4512fn summarize_file(path: &Path) -> Option<RunSummary> {
4513    summarize_file_checked(path).ok()?.summary
4514}
4515
4516/// Sanitize an id for use as a portable path segment — replace path
4517/// separators, Windows-reserved punctuation, and control characters, then
4518/// strip `..` so a hostile `agent_id`/`run_id` can't escape the `runs/` tree
4519/// or make an otherwise valid trace unpublishable on Windows. Ids are uuids /
4520/// slugs in practice; this is defense in depth.
4521fn sanitize(id: &str) -> String {
4522    let cleaned: String = id
4523        .chars()
4524        .map(|c| {
4525            if c.is_control() || matches!(c, '<' | '>' | ':' | '"' | '/' | '\\' | '|' | '?' | '*') {
4526                '_'
4527            } else {
4528                c
4529            }
4530        })
4531        .collect();
4532    let trimmed = cleaned.trim_matches(['.', ' ']);
4533    if trimmed.is_empty() {
4534        "_".to_string()
4535    } else {
4536        trimmed.to_string()
4537    }
4538}
4539
4540/// Mark a directory backup-excluded so Time Machine / iCloud don't copy
4541/// plaintext traces off the machine (R14). The portable `.nobackup` marker is
4542/// fail-closed CAR-owned state at `0600`; only the supplementary macOS xattr
4543/// remains best-effort.
4544fn ensure_backup_excluded(dir: &Path) -> std::io::Result<()> {
4545    let marker = dir.join(".nobackup");
4546    match car_secrets::create_private_file(&marker) {
4547        Ok(mut file) => {
4548            file.write_all(b"car run traces - excluded from backup\n")?;
4549            car_secrets::revalidate_private_file(&file)?;
4550            car_secrets::revalidate_private_path(&marker, &file)?;
4551        }
4552        Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {
4553            let file = car_secrets::open_private_read(&marker)?;
4554            car_secrets::revalidate_private_path(&marker, &file)?;
4555        }
4556        Err(error) => return Err(error),
4557    }
4558    #[cfg(target_os = "macos")]
4559    set_macos_backup_excluded(dir);
4560    car_secrets::ensure_private_dir(dir)?;
4561    Ok(())
4562}
4563
4564/// Set the macOS Time Machine exclusion xattr on `dir`. Uses the `xattr`
4565/// CLI (always present on macOS) rather than linking a native crate. Time
4566/// Machine treats any non-empty value on this attr as "exclude". Best-
4567/// effort — a missing `xattr` or a failure is ignored.
4568#[cfg(target_os = "macos")]
4569fn set_macos_backup_excluded(dir: &Path) {
4570    // `xattr -w` writes a string value; Time Machine only checks for the
4571    // attr's presence/non-emptiness, not its exact bytes.
4572    let _ = std::process::Command::new("xattr")
4573        .args(["-w", "com.apple.metadata:com_apple_backup_excludeItem", "1"])
4574        .arg(dir)
4575        .output();
4576}
4577
4578#[cfg(test)]
4579mod tests {
4580    use super::*;
4581    use car_ir::{AgentOutcome, CostSummary, OutcomeMetrics, OutcomeStatus, ProposalLineageEntry};
4582    use car_proto::{RunEnded, RunStarted, RunTurn, VerifierVerdict};
4583    use serde_json::json;
4584
4585    #[test]
4586    fn run_path_segments_replace_windows_reserved_characters() {
4587        assert_eq!(sanitize("name:bulldozer-agent"), "name_bulldozer-agent");
4588        assert_eq!(sanitize("<>:\"/\\|?*\0"), "__________");
4589        assert_eq!(sanitize(".. "), "_");
4590    }
4591
4592    fn store(root: PathBuf) -> RunStore {
4593        RunStore::new(root, RetentionConfig::default())
4594    }
4595
4596    fn started(run_id: &str, agent_id: &str, when: DateTime<Utc>) -> RunStarted {
4597        RunStarted {
4598            run_id: run_id.to_string(),
4599            client_id: Some("test-client".to_string()),
4600            agent_id: agent_id.to_string(),
4601            intent: "do the thing".to_string(),
4602            outcome_description: None,
4603            started_at: when,
4604        }
4605    }
4606
4607    fn turn(index: usize, prompt: &str) -> RunRecord {
4608        RunRecord::Turn(RunTurn {
4609            index,
4610            proposal_id: None,
4611            action_id: None,
4612            action_status: None,
4613            action_duration_ms: None,
4614            action_completed_at: None,
4615            depends_on: None,
4616            state_dependencies: None,
4617            prompt: Some(prompt.to_string()),
4618            tool: Some("drive_cli".to_string()),
4619            parameters: json!({ "prompt": prompt }),
4620            output: Some(json!({ "exit_code": 0 })),
4621            cli_outcome: None,
4622            verifier_verdict: VerifierVerdict::NotRun,
4623            policy_rejected: None,
4624        })
4625    }
4626
4627    fn ended(run_id: &str, agent_id: &str, status: OutcomeStatus) -> RunRecord {
4628        let outcome = AgentOutcome {
4629            status,
4630            summary: "done".to_string(),
4631            evidence: vec![],
4632            metrics: OutcomeMetrics::default(),
4633            timestamp: Utc::now(),
4634        };
4635        let termination = RunTermination::Outcome { status, outcome };
4636        let completion_digest = crate::session::run_completion_digest(&termination).unwrap();
4637        RunRecord::Ended(RunEnded {
4638            run_id: run_id.to_string(),
4639            client_id: Some("test-client".to_string()),
4640            agent_id: agent_id.to_string(),
4641            termination,
4642            completion_digest: Some(completion_digest),
4643            ended_at: Utc::now(),
4644        })
4645    }
4646
4647    fn valid_pending(run_id: &str) -> PendingProposalFinalization {
4648        let proposal = ActionProposal {
4649            id: format!("proposal-{run_id}"),
4650            source: "run-store-test".to_string(),
4651            actions: Vec::new(),
4652            timestamp: Utc::now(),
4653            context: HashMap::new(),
4654        };
4655        let digest = proposal_digest(&proposal).unwrap();
4656        let proposal_result = ProposalResult {
4657            proposal_id: proposal.id.clone(),
4658            original_proposal_id: proposal.id.clone(),
4659            final_proposal: Some(proposal.clone()),
4660            replan_lineage: vec![ProposalLineageEntry {
4661                generation: 0,
4662                proposal_id: proposal.id.clone(),
4663                proposal_digest: Some(digest.clone()),
4664                status: ProposalLineageStatus::Accepted,
4665                rejection_reason: None,
4666            }],
4667            accepted_proposal_preimages: vec![car_ir::AcceptedProposalPreimage {
4668                generation: 0,
4669                proposal_digest: digest,
4670                proposal: proposal.clone(),
4671            }],
4672            results: Vec::new(),
4673            cost: CostSummary::default(),
4674        };
4675        let result_value = serde_json::to_value(&proposal_result).unwrap();
4676        let canonical = car_inference::catalog_identity::canonical_json(&result_value).unwrap();
4677        PendingProposalFinalization {
4678            run_id: run_id.to_string(),
4679            client_id: "test-client".to_string(),
4680            requested_policy_session_id: None,
4681            policy_session_id: None,
4682            original_proposal_id: proposal.id.clone(),
4683            final_proposal_id: proposal.id.clone(),
4684            original_submission: json!({
4685                "id": proposal.id.clone(),
4686                "source": "run-store-test",
4687                "actions": []
4688            }),
4689            original_proposal: proposal.clone(),
4690            final_proposal: proposal.clone(),
4691            accepted_proposal_preimages: vec![AcceptedProposalPreimage {
4692                generation: 0,
4693                proposal,
4694            }],
4695            proposal_result,
4696            result_digest: format!("{:x}", Sha256::digest(canonical.as_bytes())),
4697        }
4698    }
4699
4700    fn refresh_pending_result_digest(pending: &mut PendingProposalFinalization) {
4701        let value = serde_json::to_value(&pending.proposal_result).unwrap();
4702        let canonical = car_inference::catalog_identity::canonical_json(&value).unwrap();
4703        pending.result_digest = format!("{:x}", Sha256::digest(canonical.as_bytes()));
4704    }
4705
4706    fn marker_for_pending(pending: &PendingProposalFinalization) -> ProposalExecutionMarker {
4707        ProposalExecutionMarker {
4708            run_id: pending.run_id.clone(),
4709            client_id: pending.client_id.clone(),
4710            requested_policy_session_id: pending.requested_policy_session_id.clone(),
4711            policy_session_id: pending.policy_session_id.clone(),
4712            original_proposal_id: pending.original_proposal_id.clone(),
4713            original_submission: pending.original_submission.clone(),
4714            original_proposal: serde_json::to_value(&pending.original_proposal).unwrap(),
4715            proposal_digest: proposal_digest(&pending.original_proposal).unwrap(),
4716        }
4717    }
4718
4719    fn rebind_pending_original(pending: &mut PendingProposalFinalization) {
4720        pending.original_submission["source"] = json!("pending-self-claim");
4721        pending.original_proposal.source = "pending-self-claim".to_string();
4722        pending.final_proposal = pending.original_proposal.clone();
4723        pending.accepted_proposal_preimages[0].proposal = pending.original_proposal.clone();
4724        pending.proposal_result.final_proposal = Some(pending.original_proposal.clone());
4725        pending.proposal_result.replan_lineage[0].proposal_digest =
4726            Some(proposal_digest(&pending.original_proposal).unwrap());
4727        refresh_pending_result_digest(pending);
4728    }
4729
4730    fn write_pending_provenance(store: &RunStore, pending: &PendingProposalFinalization) {
4731        store
4732            .write_started(&started(&pending.run_id, "agent-a", Utc::now()))
4733            .unwrap();
4734        store
4735            .write_execution_marker(&marker_for_pending(pending))
4736            .unwrap();
4737    }
4738
4739    fn write_completed_receipt_fixture(
4740        store: &RunStore,
4741        pending: PendingProposalFinalization,
4742        exact_bytes: Option<usize>,
4743        invalid_tail: bool,
4744    ) -> CompletedProposalResponse {
4745        let receipt = CompletedProposalResponse {
4746            finalization: pending,
4747        };
4748        receipt.validate().unwrap();
4749        let pending = &receipt.finalization;
4750        let run_root = store.completed_response_run_root(&pending.run_id);
4751        store.ensure_private_dir(&run_root).unwrap();
4752        let path = store
4753            .completed_response_path(
4754                &pending.run_id,
4755                &pending.client_id,
4756                pending.requested_policy_session_id.as_deref(),
4757                &pending.original_submission,
4758            )
4759            .unwrap();
4760        let mut bytes = serde_json::to_vec(&receipt).unwrap();
4761        if let Some(exact_bytes) = exact_bytes {
4762            assert!(bytes.len() <= exact_bytes);
4763            bytes.resize(exact_bytes, b' ');
4764        }
4765        if invalid_tail {
4766            bytes.push(b'!');
4767        }
4768        let mut file = store.create_private_file(&path).unwrap();
4769        file.write_all(&bytes).unwrap();
4770        file.sync_all().unwrap();
4771        receipt
4772    }
4773
4774    #[test]
4775    fn first_use_agent_receipt_parent_sync_failure_is_not_acknowledged_and_retry_is_exact() {
4776        let tmp = tempfile::TempDir::new().unwrap();
4777        let failures = car_secrets::PrivatePathDurabilityFailureInjector::default();
4778        let store =
4779            store(tmp.path().join("runs")).with_private_path_failure_injector(failures.clone());
4780        store.prepare_storage().unwrap();
4781        failures.fail_next(car_secrets::PrivatePathDurabilityFailurePoint::ParentDirectorySync);
4782        let boundary = started("private-first-use", "new-agent", Utc::now());
4783
4784        let error = store.write_started(&boundary).unwrap_err();
4785        assert!(error
4786            .to_string()
4787            .contains("injected private-path parent directory sync failure"));
4788        assert!(
4789            store.get_run_trace(&boundary.run_id).is_none(),
4790            "an unacknowledged agent-directory entry must not contain a run boundary"
4791        );
4792
4793        store.write_started(&boundary).unwrap();
4794        let trace = store.get_run_trace(&boundary.run_id).unwrap();
4795        assert_eq!(
4796            trace
4797                .iter()
4798                .filter(|row| matches!(row, RunRecord::Started(_)))
4799                .count(),
4800            1,
4801            "retry must persist exactly one RunStarted boundary"
4802        );
4803    }
4804
4805    #[test]
4806    fn proposal_sidecar_parent_sync_failures_are_not_acknowledged_and_retry_exactly() {
4807        let tmp = tempfile::TempDir::new().unwrap();
4808        let failures = car_secrets::PrivatePathDurabilityFailureInjector::default();
4809        let store =
4810            store(tmp.path().join("runs")).with_private_path_failure_injector(failures.clone());
4811        let pending = valid_pending("private-sidecar-first-use");
4812        store
4813            .write_started(&started(&pending.run_id, "agent-a", Utc::now()))
4814            .unwrap();
4815        let marker = marker_for_pending(&pending);
4816
4817        failures.fail_next(car_secrets::PrivatePathDurabilityFailurePoint::ParentDirectorySync);
4818        assert!(store.write_execution_marker(&marker).is_err());
4819        assert_eq!(store.execution_marker(&pending.run_id).unwrap(), None);
4820        store.write_execution_marker(&marker).unwrap();
4821        assert_eq!(
4822            store.execution_marker(&pending.run_id).unwrap(),
4823            Some(marker)
4824        );
4825
4826        failures.fail_next(car_secrets::PrivatePathDurabilityFailurePoint::ParentDirectorySync);
4827        assert!(store.write_pending_proposal(&pending).is_err());
4828        assert_eq!(store.pending_proposal(&pending.run_id).unwrap(), None);
4829        store.write_pending_proposal(&pending).unwrap();
4830        assert_eq!(
4831            store.pending_proposal(&pending.run_id).unwrap(),
4832            Some(pending)
4833        );
4834    }
4835
4836    /// A completed run is readable from a brand-new store instance — the
4837    /// "simulated daemon restart" (new store, empty memory): the trace
4838    /// must come back from disk (R4).
4839    #[test]
4840    fn completed_run_readable_after_restart() {
4841        let tmp = tempfile::TempDir::new().unwrap();
4842        let root = tmp.path().join("runs");
4843        let s1 = store(root.clone());
4844        s1.write_started(&started("run-1", "agent-a", Utc::now()))
4845            .unwrap();
4846        s1.append_turns("agent-a", "run-1", &[turn(0, "first")])
4847            .unwrap();
4848        s1.append_records(
4849            "agent-a",
4850            "run-1",
4851            &[ended("run-1", "agent-a", OutcomeStatus::Success)],
4852        )
4853        .unwrap();
4854
4855        // Brand-new store over the same root = simulated restart with empty memory.
4856        let s2 = store(root);
4857        let trace = s2
4858            .get_run_trace("run-1")
4859            .expect("trace readable after restart");
4860        assert!(matches!(trace.first(), Some(RunRecord::Started(_))));
4861        assert!(matches!(trace.last(), Some(RunRecord::Ended(_))));
4862        let turns = trace
4863            .iter()
4864            .filter(|r| matches!(r, RunRecord::Turn(_)))
4865            .count();
4866        assert_eq!(turns, 1);
4867    }
4868
4869    /// Each run persists to its own `(agent_id, run_id)` file — no
4870    /// cross-contamination between runs or agents (R1).
4871    #[test]
4872    fn runs_isolated_per_agent_and_run() {
4873        let tmp = tempfile::TempDir::new().unwrap();
4874        let s = store(tmp.path().join("runs"));
4875        s.write_started(&started("run-1", "agent-a", Utc::now()))
4876            .unwrap();
4877        s.append_turns("agent-a", "run-1", &[turn(0, "a-first")])
4878            .unwrap();
4879        s.write_started(&started("run-2", "agent-a", Utc::now()))
4880            .unwrap();
4881        s.append_turns("agent-a", "run-2", &[turn(0, "a-second")])
4882            .unwrap();
4883        s.write_started(&started("run-3", "agent-b", Utc::now()))
4884            .unwrap();
4885        s.append_turns("agent-b", "run-3", &[turn(0, "b-first")])
4886            .unwrap();
4887
4888        // Distinct files; each holds only its own turn.
4889        let t1 = s.get_run_trace("run-1").unwrap();
4890        let t2 = s.get_run_trace("run-2").unwrap();
4891        let t3 = s.get_run_trace("run-3").unwrap();
4892        assert_eq!(turn_prompt(&t1), "a-first");
4893        assert_eq!(turn_prompt(&t2), "a-second");
4894        assert_eq!(turn_prompt(&t3), "b-first");
4895        // run_id -> agent_id resolution works for replay authz.
4896        assert_eq!(s.agent_for_run("run-1").as_deref(), Some("agent-a"));
4897        assert_eq!(s.agent_for_run("run-3").as_deref(), Some("agent-b"));
4898        // agent-a lists 2 runs, agent-b lists 1.
4899        assert_eq!(s.list_runs("agent-a").len(), 2);
4900        assert_eq!(s.list_runs("agent-b").len(), 1);
4901    }
4902
4903    #[test]
4904    fn summary_index_pages_in_stable_newest_first_order() {
4905        let tmp = tempfile::TempDir::new().unwrap();
4906        let s = store(tmp.path().join("runs"));
4907        let now = Utc::now();
4908        s.write_started(&started(
4909            "run-old",
4910            "agent-page",
4911            now - chrono::Duration::seconds(2),
4912        ))
4913        .unwrap();
4914        s.append_turns("agent-page", "run-old", &[turn(0, "old")])
4915            .unwrap();
4916        s.write_started(&started("run-new", "agent-page", now))
4917            .unwrap();
4918        s.append_turns(
4919            "agent-page",
4920            "run-new",
4921            &[turn(0, "new-0"), turn(1, "new-1")],
4922        )
4923        .unwrap();
4924
4925        let (first, next) = s.list_runs_page("agent-page", 0, 1).unwrap();
4926        assert_eq!(first.len(), 1);
4927        assert_eq!(first[0].run_id, "run-new");
4928        assert_eq!(first[0].turn_count, 2);
4929        let next = next.expect("older run remains");
4930
4931        // A new run arriving at the head must not shift an offset cursor and
4932        // cause the previous first row to be returned twice.
4933        s.write_started(&started(
4934            "run-newest",
4935            "agent-page",
4936            now + chrono::Duration::seconds(1),
4937        ))
4938        .unwrap();
4939
4940        let (second, next) = s.list_runs_page("agent-page", next, 1).unwrap();
4941        assert_eq!(second.len(), 1);
4942        assert_eq!(second[0].run_id, "run-old");
4943        assert_eq!(second[0].turn_count, 1);
4944        assert_eq!(next, None);
4945
4946        let (past_end, next) = s.list_runs_page("agent-page", 1, 1).unwrap();
4947        assert!(past_end.is_empty());
4948        assert_eq!(next, None);
4949    }
4950
4951    #[test]
4952    fn newline_terminated_malformed_record_quarantines_completed_trace() {
4953        let tmp = tempfile::TempDir::new().unwrap();
4954        let root = tmp.path().join("runs");
4955        let s = store(root.clone());
4956        s.write_started(&started("corrupt", "agent-a", Utc::now()))
4957            .unwrap();
4958        s.append_turns("agent-a", "corrupt", &[turn(0, "before")])
4959            .unwrap();
4960
4961        let path = root.join("agent-a").join("corrupt.jsonl");
4962        let mut file = car_secrets::open_private_append(&path).unwrap();
4963        file.write_all(b"{malformed-middle}\n").unwrap();
4964        file.flush().unwrap();
4965        file.sync_all().unwrap();
4966        drop(file);
4967
4968        let RunRecord::Ended(terminal) = ended("corrupt", "agent-a", OutcomeStatus::Success) else {
4969            unreachable!()
4970        };
4971        let error = s
4972            .write_ended(&terminal)
4973            .expect_err("a durable malformed row must prevent completion");
4974        assert!(error.to_string().contains("line 3"), "{error}");
4975
4976        let (summaries, _) = s.list_runs_page("agent-a", 0, 1).unwrap();
4977        assert_eq!(summaries[0].status, RunStatus::Incomplete);
4978        assert_eq!(
4979            serde_json::to_value(&summaries[0]).unwrap()["trace_corruption"],
4980            json!({"kind":"malformed_record","line":3})
4981        );
4982        assert!(s
4983            .get_run_turn_page_for("agent-a", "corrupt", 0, 10)
4984            .unwrap_err()
4985            .to_string()
4986            .contains("line 3"));
4987    }
4988
4989    fn turn_prompt(trace: &[RunRecord]) -> String {
4990        trace
4991            .iter()
4992            .find_map(|r| match r {
4993                RunRecord::Turn(t) => t.prompt.clone(),
4994                _ => None,
4995            })
4996            .unwrap_or_default()
4997    }
4998
4999    /// Files are `0600` and dirs are `0700` (R14 — assert the modes).
5000    #[cfg(unix)]
5001    #[test]
5002    fn perms_are_0600_files_0700_dirs() {
5003        use std::os::unix::fs::PermissionsExt;
5004        let tmp = tempfile::TempDir::new().unwrap();
5005        let root = tmp.path().join("runs");
5006        let s = store(root.clone());
5007        s.write_started(&started("run-1", "agent-a", Utc::now()))
5008            .unwrap();
5009
5010        let file = root.join("agent-a").join("run-1.jsonl");
5011        let fmode = std::fs::metadata(&file).unwrap().permissions().mode() & 0o777;
5012        assert_eq!(fmode, 0o600, "run file must be 0600, got {:o}", fmode);
5013
5014        let root_mode = std::fs::metadata(&root).unwrap().permissions().mode() & 0o777;
5015        assert_eq!(
5016            root_mode, 0o700,
5017            "runs/ dir must be 0700, got {:o}",
5018            root_mode
5019        );
5020        let agent_mode = std::fs::metadata(root.join("agent-a"))
5021            .unwrap()
5022            .permissions()
5023            .mode()
5024            & 0o777;
5025        assert_eq!(
5026            agent_mode, 0o700,
5027            "agent dir must be 0700, got {:o}",
5028            agent_mode
5029        );
5030
5031        // The backup-exclusion marker is present.
5032        let marker = root.join(".nobackup");
5033        assert!(marker.exists(), ".nobackup marker written");
5034        let marker_mode = std::fs::metadata(marker).unwrap().permissions().mode() & 0o777;
5035        assert_eq!(
5036            marker_mode, 0o600,
5037            ".nobackup marker must be 0600, got {:o}",
5038            marker_mode
5039        );
5040    }
5041
5042    #[cfg(unix)]
5043    #[test]
5044    fn replay_hardens_owned_legacy_tree_without_changing_content() {
5045        use std::os::unix::fs::PermissionsExt;
5046
5047        let tmp = tempfile::TempDir::new().unwrap();
5048        let root = tmp.path().join("runs");
5049        let agent = root.join("agent-a");
5050        std::fs::create_dir_all(&agent).unwrap();
5051        let path = agent.join("run-1.jsonl");
5052        let line =
5053            serde_json::to_string(&RunRecord::Started(started("run-1", "agent-a", Utc::now())))
5054                .unwrap();
5055        let original = format!("{line}\n");
5056        std::fs::write(&path, original.as_bytes()).unwrap();
5057        std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o755)).unwrap();
5058        std::fs::set_permissions(&agent, std::fs::Permissions::from_mode(0o755)).unwrap();
5059        std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644)).unwrap();
5060
5061        let trace = store(root.clone())
5062            .get_run_trace_for("agent-a", "run-1")
5063            .expect("legacy run remains replayable");
5064        assert_eq!(trace.len(), 1);
5065        assert_eq!(std::fs::read_to_string(&path).unwrap(), original);
5066        assert_eq!(
5067            std::fs::metadata(root).unwrap().permissions().mode() & 0o777,
5068            0o700
5069        );
5070        assert_eq!(
5071            std::fs::metadata(agent).unwrap().permissions().mode() & 0o777,
5072            0o700
5073        );
5074        assert_eq!(
5075            std::fs::metadata(path).unwrap().permissions().mode() & 0o777,
5076            0o600
5077        );
5078    }
5079
5080    #[cfg(unix)]
5081    #[test]
5082    fn run_file_symlinks_and_hardlinks_are_rejected_without_touching_victim() {
5083        use std::os::unix::fs::symlink;
5084
5085        let tmp = tempfile::TempDir::new().unwrap();
5086        let root = tmp.path().join("runs");
5087        let agent = root.join("agent-a");
5088        std::fs::create_dir_all(&agent).unwrap();
5089        let victim = tmp.path().join("victim.jsonl");
5090        std::fs::write(&victim, b"victim\n").unwrap();
5091
5092        let symlink_path = agent.join("symlink.jsonl");
5093        symlink(&victim, &symlink_path).unwrap();
5094        let s = store(root.clone());
5095        assert!(s.get_run_trace_for("agent-a", "symlink").is_none());
5096        assert!(s
5097            .append_records(
5098                "agent-a",
5099                "symlink",
5100                &[RunRecord::Started(started(
5101                    "symlink",
5102                    "agent-a",
5103                    Utc::now()
5104                ))],
5105            )
5106            .is_err());
5107        assert_eq!(std::fs::read(&victim).unwrap(), b"victim\n");
5108
5109        std::fs::remove_file(&symlink_path).unwrap();
5110        let hardlink_path = agent.join("hardlink.jsonl");
5111        std::fs::hard_link(&victim, &hardlink_path).unwrap();
5112        assert!(s.get_run_trace_for("agent-a", "hardlink").is_none());
5113        assert!(s
5114            .append_records(
5115                "agent-a",
5116                "hardlink",
5117                &[RunRecord::Started(started(
5118                    "hardlink",
5119                    "agent-a",
5120                    Utc::now()
5121                ))],
5122            )
5123            .is_err());
5124        assert_eq!(std::fs::read(&victim).unwrap(), b"victim\n");
5125    }
5126
5127    #[cfg(unix)]
5128    #[test]
5129    fn symlink_agent_directory_and_special_run_file_are_rejected() {
5130        use std::os::unix::fs::symlink;
5131        use std::os::unix::net::UnixListener;
5132
5133        let tmp = tempfile::TempDir::new().unwrap();
5134        let root = tmp.path().join("runs");
5135        std::fs::create_dir(&root).unwrap();
5136        let victim_dir = tmp.path().join("victim-agent");
5137        std::fs::create_dir(&victim_dir).unwrap();
5138        symlink(&victim_dir, root.join("agent-link")).unwrap();
5139
5140        let s = store(root.clone());
5141        assert!(s.list_runs("agent-link").is_empty());
5142        assert!(s
5143            .append_records(
5144                "agent-link",
5145                "run-1",
5146                &[RunRecord::Started(started(
5147                    "run-1",
5148                    "agent-link",
5149                    Utc::now()
5150                ))],
5151            )
5152            .is_err());
5153        assert!(std::fs::read_dir(&victim_dir).unwrap().next().is_none());
5154
5155        let agent = root.join("agent-a");
5156        std::fs::create_dir(&agent).unwrap();
5157        let socket_path = agent.join("socket.jsonl");
5158        let _listener = UnixListener::bind(&socket_path).unwrap();
5159        assert!(s.get_run_trace_for("agent-a", "socket").is_none());
5160        assert!(s
5161            .append_records(
5162                "agent-a",
5163                "socket",
5164                &[RunRecord::Started(started("socket", "agent-a", Utc::now()))],
5165            )
5166            .is_err());
5167    }
5168
5169    #[cfg(unix)]
5170    #[test]
5171    fn append_uses_the_validated_descriptor_after_path_substitution() {
5172        use std::os::unix::fs::symlink;
5173
5174        let tmp = tempfile::TempDir::new().unwrap();
5175        let path = tmp.path().join("run.jsonl");
5176        std::fs::write(&path, b"torn").unwrap();
5177        let mut file = car_secrets::open_private_append(&path).unwrap();
5178        let victim = tmp.path().join("victim");
5179        std::fs::write(&victim, b"victim\n").unwrap();
5180
5181        std::fs::remove_file(&path).unwrap();
5182        symlink(&victim, &path).unwrap();
5183        assert!(append_jsonl_batch_to_path(&path, &mut file, &[turn(0, "safe")]).is_err());
5184
5185        assert_eq!(std::fs::read(&victim).unwrap(), b"victim\n");
5186    }
5187
5188    #[cfg(unix)]
5189    #[test]
5190    fn replay_rejects_path_substitution_after_open() {
5191        use std::os::unix::fs::symlink;
5192
5193        let tmp = tempfile::TempDir::new().unwrap();
5194        let path = tmp.path().join("run.jsonl");
5195        let line =
5196            serde_json::to_string(&RunRecord::Started(started("run-1", "agent-a", Utc::now())))
5197                .unwrap();
5198        std::fs::write(&path, format!("{line}\n")).unwrap();
5199        let file = car_secrets::open_private_read(&path).unwrap();
5200        let victim = tmp.path().join("victim");
5201        std::fs::write(&victim, b"victim\n").unwrap();
5202
5203        std::fs::rename(&path, tmp.path().join("moved.jsonl")).unwrap();
5204        symlink(&victim, &path).unwrap();
5205
5206        assert!(load_private_records(&path, &file).is_err());
5207        assert_eq!(std::fs::read(&victim).unwrap(), b"victim\n");
5208    }
5209
5210    /// A run with no `RunEnded` reads `InProgress`; once the disconnect
5211    /// path writes the `Incomplete` terminal it reads `Incomplete` — never
5212    /// silently `Completed`. This is the R5 distinguishability the
5213    /// dashboard renders.
5214    #[test]
5215    fn orphan_run_status_distinguishes_inprogress_from_incomplete() {
5216        let tmp = tempfile::TempDir::new().unwrap();
5217        let s = store(tmp.path().join("runs"));
5218        // Stale start time (no harness writing — an orphan).
5219        let stale = Utc::now() - chrono::Duration::hours(6);
5220        s.write_started(&started("run-1", "agent-a", stale))
5221            .unwrap();
5222        s.append_turns("agent-a", "run-1", &[turn(0, "first")])
5223            .unwrap();
5224
5225        // Still open: no terminal record yet.
5226        let open = &s.list_runs("agent-a")[0];
5227        assert_eq!(open.status, RunStatus::InProgress);
5228
5229        // Disconnect path writes the Incomplete terminal.
5230        let incomplete = RunRecord::Ended(RunEnded {
5231            run_id: "run-1".to_string(),
5232            client_id: Some("test-client".to_string()),
5233            agent_id: "agent-a".to_string(),
5234            termination: RunTermination::Incomplete,
5235            completion_digest: Some("test-digest".to_string()),
5236            ended_at: Utc::now(),
5237        });
5238        s.append_records("agent-a", "run-1", &[incomplete]).unwrap();
5239        let closed = &s.list_runs("agent-a")[0];
5240        assert_eq!(closed.status, RunStatus::Incomplete);
5241    }
5242
5243    /// Retention evicts beyond the per-agent cap, never an in-progress
5244    /// run (R6).
5245    #[test]
5246    fn gc_evicts_beyond_per_agent_cap_but_never_in_progress() {
5247        let tmp = tempfile::TempDir::new().unwrap();
5248        let root = tmp.path().join("runs");
5249        let s = RunStore::new(root, RetentionConfig::new(3, 30));
5250        // 5 completed runs with increasing start times.
5251        let base = Utc::now() - chrono::Duration::days(1);
5252        for i in 0..5 {
5253            let id = format!("c{i}");
5254            let when = base + chrono::Duration::minutes(i);
5255            s.write_started(&started(&id, "agent-a", when)).unwrap();
5256            s.append_records(
5257                "agent-a",
5258                &id,
5259                &[ended(&id, "agent-a", OutcomeStatus::Success)],
5260            )
5261            .unwrap();
5262        }
5263        // 1 in-progress run (no terminal) — must survive GC.
5264        s.write_started(&started("live", "agent-a", Utc::now()))
5265            .unwrap();
5266
5267        let removed = s.gc();
5268        // Keep 3 most-recent completed; evict the 2 oldest completed. The
5269        // in-progress run is never counted/evicted.
5270        assert_eq!(removed, 2, "should evict the 2 oldest completed runs");
5271        let remaining = s.list_runs("agent-a");
5272        // 3 kept completed + the live one = 4.
5273        assert_eq!(remaining.len(), 4);
5274        assert!(
5275            remaining.iter().any(|r| r.run_id == "live"),
5276            "in-progress run must never be evicted"
5277        );
5278        // The two oldest (c0, c1) are gone.
5279        assert!(!remaining.iter().any(|r| r.run_id == "c0"));
5280        assert!(!remaining.iter().any(|r| r.run_id == "c1"));
5281    }
5282
5283    /// `0` disables a cap. It used to delete everything (car#1338).
5284    ///
5285    /// Read literally, `max_per_agent = 0` makes `completed_rank >= 0` true for
5286    /// every run and `max_age_days = 0` puts the cutoff at now — so either value
5287    /// wiped every collectable trace on the next boot, unrecoverably. Nobody
5288    /// types `0` meaning "keep nothing"; they mean "no cap", which is what it
5289    /// means for `max_session_wall_secs` and for the coder's own retention in
5290    /// the neighbouring config file.
5291    #[test]
5292    fn a_zero_cap_keeps_everything_instead_of_deleting_it() {
5293        // Each case disables ONE cap and keeps the run set clear of the other,
5294        // so a pass means the disabled cap stood down rather than the other one
5295        // simply having nothing to take.
5296        let recent = Utc::now();
5297        let ancient = Utc::now() - chrono::Duration::days(400);
5298        for (retention, count, when) in [
5299            // No count cap: 60 runs, well past the default 50, all recent.
5300            (RetentionConfig::new(0, 30), 60, recent),
5301            // No age cap: ancient runs, but fewer than the count cap.
5302            (RetentionConfig::new(50, 0), 5, ancient),
5303            // A negative age is not a cap either; it would put the cutoff in the
5304            // future and evict runs that have not happened yet.
5305            (RetentionConfig::new(50, -1), 5, recent),
5306            // Neither cap: both dimensions blown well past the defaults.
5307            (RetentionConfig::new(0, 0), 60, ancient),
5308        ] {
5309            let tmp = tempfile::TempDir::new().unwrap();
5310            let s = RunStore::new(tmp.path().join("runs"), retention);
5311            for i in 0..count {
5312                let id = format!("r{i}");
5313                s.write_started(&started(&id, "agent-a", when)).unwrap();
5314                s.append_records(
5315                    "agent-a",
5316                    &id,
5317                    &[ended_at(&id, "agent-a", OutcomeStatus::Success, when)],
5318                )
5319                .unwrap();
5320            }
5321
5322            assert_eq!(s.gc(), 0, "{retention:?} must evict nothing");
5323            assert_eq!(s.list_runs("agent-a").len(), count, "{retention:?}");
5324        }
5325    }
5326
5327    /// An operator typo must not take the daemon down at boot.
5328    ///
5329    /// `chrono::Duration::days` PANICS rather than saturating on a value that
5330    /// overflows, and `max_age_days` is an unvalidated `i64` straight from
5331    /// `config.toml`. An unrepresentable cutoff reads as no cap, which is the
5332    /// safe direction — the alternative is deleting on a number nobody meant.
5333    #[test]
5334    fn an_absurd_age_cap_disables_itself_rather_than_panicking() {
5335        // Asserted on the cap, not only on `gc() == 0`: a cutoff at year
5336        // -292_277_022_365 would also evict nothing, so the weaker assertion
5337        // would pass without the cap actually being disabled.
5338        assert_eq!(RetentionConfig::new(50, i64::MAX).age_cap_days(), None);
5339        assert_eq!(RetentionConfig::new(50, -1).age_cap_days(), None);
5340        assert_eq!(RetentionConfig::new(50, 30).age_cap_days(), Some(30));
5341
5342        let tmp = tempfile::TempDir::new().unwrap();
5343        let s = RunStore::new(tmp.path().join("runs"), RetentionConfig::new(50, i64::MAX));
5344        let old = Utc::now() - chrono::Duration::days(400);
5345        s.write_started(&started("r0", "agent-a", old)).unwrap();
5346        s.append_records(
5347            "agent-a",
5348            "r0",
5349            &[ended_at("r0", "agent-a", OutcomeStatus::Success, old)],
5350        )
5351        .unwrap();
5352
5353        assert_eq!(s.gc(), 0);
5354        assert_eq!(s.list_runs("agent-a").len(), 1);
5355    }
5356
5357    /// A present `0` is honored, not silently replaced by the default. Filtering
5358    /// it at load would turn "keep everything" back into the 50/30 caps.
5359    #[test]
5360    fn a_configured_zero_survives_loading() {
5361        let dir = tempfile::TempDir::new().unwrap();
5362        std::fs::write(
5363            dir.path().join("config.toml"),
5364            "[runs]\nmax_per_agent = 0\nmax_age_days = 0\n",
5365        )
5366        .unwrap();
5367        let cfg = RetentionConfig::from_car_dir(dir.path());
5368        assert_eq!(cfg.count_cap(), None, "a configured 0 must not become 50");
5369        assert_eq!(
5370            cfg.age_cap_days(),
5371            None,
5372            "a configured 0 must not become 30"
5373        );
5374    }
5375
5376    /// Retention evicts completed runs older than the age cap (R6).
5377    #[test]
5378    fn gc_evicts_runs_older_than_age_cap() {
5379        let tmp = tempfile::TempDir::new().unwrap();
5380        let s = RunStore::new(tmp.path().join("runs"), RetentionConfig::new(50, 30));
5381        // One old completed run — started AND ended 40 days ago (its
5382        // terminal time is what the age cap measures, FIX 2) — and one
5383        // fresh run.
5384        let old = Utc::now() - chrono::Duration::days(40);
5385        s.write_started(&started("old", "agent-a", old)).unwrap();
5386        s.append_records(
5387            "agent-a",
5388            "old",
5389            &[ended_at("old", "agent-a", OutcomeStatus::Success, old)],
5390        )
5391        .unwrap();
5392        s.write_started(&started("fresh", "agent-a", Utc::now()))
5393            .unwrap();
5394        s.append_records(
5395            "agent-a",
5396            "fresh",
5397            &[ended("fresh", "agent-a", OutcomeStatus::Success)],
5398        )
5399        .unwrap();
5400
5401        let removed = s.gc();
5402        assert_eq!(removed, 1, "the 40-day-old run should be evicted");
5403        let remaining = s.list_runs("agent-a");
5404        assert_eq!(remaining.len(), 1);
5405        assert_eq!(remaining[0].run_id, "fresh");
5406    }
5407
5408    /// An old run that is still in progress is NOT evicted by the age cap
5409    /// (R6 — never evict an open run, even a stale one).
5410    #[test]
5411    fn gc_never_evicts_stale_in_progress_run() {
5412        let tmp = tempfile::TempDir::new().unwrap();
5413        let s = RunStore::new(tmp.path().join("runs"), RetentionConfig::new(1, 1));
5414        let old = Utc::now() - chrono::Duration::days(40);
5415        // Old + in-progress (no terminal).
5416        s.write_started(&started("stale-live", "agent-a", old))
5417            .unwrap();
5418        let removed = s.gc();
5419        assert_eq!(removed, 0);
5420        assert!(s
5421            .list_runs("agent-a")
5422            .iter()
5423            .any(|r| r.run_id == "stale-live"));
5424    }
5425
5426    /// A corrupt/partial trailing JSONL line loads the prior valid
5427    /// records rather than failing the whole run (error path).
5428    #[test]
5429    fn corrupt_trailing_line_loads_prior_records() {
5430        let tmp = tempfile::TempDir::new().unwrap();
5431        let root = tmp.path().join("runs");
5432        let s = store(root.clone());
5433        s.write_started(&started("run-1", "agent-a", Utc::now()))
5434            .unwrap();
5435        s.append_turns("agent-a", "run-1", &[turn(0, "first"), turn(1, "second")])
5436            .unwrap();
5437
5438        // Append a partial/garbage line directly (simulating a crash
5439        // mid-append).
5440        let path = root.join("agent-a").join("run-1.jsonl");
5441        let mut f = std::fs::OpenOptions::new()
5442            .append(true)
5443            .open(&path)
5444            .unwrap();
5445        f.write_all(b"{\"record\":\"turn\",\"index\":2,\"prom")
5446            .unwrap();
5447
5448        let trace = s.get_run_trace("run-1").expect("trace still loads");
5449        // RunStarted + 2 valid turns; the garbage trailing line is dropped.
5450        let turns = trace
5451            .iter()
5452            .filter(|r| matches!(r, RunRecord::Turn(_)))
5453            .count();
5454        assert_eq!(turns, 2, "prior valid turns load; corrupt line skipped");
5455        assert!(matches!(trace.first(), Some(RunRecord::Started(_))));
5456    }
5457
5458    /// `list_runs` for an agent with no runs returns empty (empty-state).
5459    #[test]
5460    fn list_runs_empty_for_unknown_agent() {
5461        let tmp = tempfile::TempDir::new().unwrap();
5462        let s = store(tmp.path().join("runs"));
5463        assert!(s.list_runs("nobody").is_empty());
5464        assert!(s.get_run_trace("nope").is_none());
5465        assert!(s.agent_for_run("nope").is_none());
5466    }
5467
5468    /// `from_journal_dir` roots the store at `<car_dir>/runs` as a sibling
5469    /// of the journal dir.
5470    #[test]
5471    fn from_journal_dir_roots_at_car_runs() {
5472        let s = RunStore::from_journal_dir(Path::new("/home/u/.car/journals"));
5473        assert_eq!(s.root(), Path::new("/home/u/.car/runs"));
5474    }
5475
5476    /// Retention config loads `[runs]` overrides from config.toml; missing
5477    /// keys keep the restrictive default.
5478    #[test]
5479    fn retention_config_reads_overrides() {
5480        let tmp = tempfile::TempDir::new().unwrap();
5481        std::fs::write(
5482            tmp.path().join("config.toml"),
5483            "[runs]\nmax_per_agent = 10\n",
5484        )
5485        .unwrap();
5486        let cfg = RetentionConfig::from_car_dir(tmp.path());
5487        assert_eq!(cfg.count_cap(), Some(10));
5488        // Missing key keeps the default.
5489        assert_eq!(cfg.age_cap_days(), Some(DEFAULT_MAX_AGE_DAYS));
5490    }
5491
5492    /// A missing/malformed config.toml falls back to the restrictive
5493    /// default — never refuses.
5494    #[test]
5495    fn retention_config_defaults_on_missing_file() {
5496        let tmp = tempfile::TempDir::new().unwrap();
5497        let cfg = RetentionConfig::from_car_dir(tmp.path());
5498        assert_eq!(cfg.count_cap(), Some(DEFAULT_MAX_RUNS_PER_AGENT));
5499        assert_eq!(cfg.age_cap_days(), Some(DEFAULT_MAX_AGE_DAYS));
5500    }
5501
5502    /// Append the terminal `RunEnded` with an explicit `ended_at` (the
5503    /// disconnect/complete path uses `Utc::now()`, but GC ages by terminal
5504    /// time, so tests need to control it).
5505    fn ended_at(
5506        run_id: &str,
5507        agent_id: &str,
5508        status: OutcomeStatus,
5509        when: DateTime<Utc>,
5510    ) -> RunRecord {
5511        let outcome = AgentOutcome {
5512            status,
5513            summary: "done".to_string(),
5514            evidence: vec![],
5515            metrics: OutcomeMetrics::default(),
5516            timestamp: when,
5517        };
5518        RunRecord::Ended(RunEnded {
5519            run_id: run_id.to_string(),
5520            client_id: Some("test-client".to_string()),
5521            agent_id: agent_id.to_string(),
5522            termination: RunTermination::Outcome { status, outcome },
5523            completion_digest: Some("test-digest".to_string()),
5524            ended_at: when,
5525        })
5526    }
5527
5528    /// FIX 2: a run that STARTED >max_age_days ago but COMPLETED recently is
5529    /// a fresh result and must NOT be evicted by the age cap. GC must age by
5530    /// the terminal time, not the start time.
5531    #[test]
5532    fn gc_age_cap_uses_terminal_time_not_start() {
5533        let tmp = tempfile::TempDir::new().unwrap();
5534        let s = RunStore::new(tmp.path().join("runs"), RetentionConfig::new(50, 30));
5535        // Long-running run: started 40 days ago, completed 1 day ago.
5536        let started_40d = Utc::now() - chrono::Duration::days(40);
5537        let ended_1d = Utc::now() - chrono::Duration::days(1);
5538        s.write_started(&started("long", "agent-a", started_40d))
5539            .unwrap();
5540        s.append_records(
5541            "agent-a",
5542            "long",
5543            &[ended_at(
5544                "long",
5545                "agent-a",
5546                OutcomeStatus::Success,
5547                ended_1d,
5548            )],
5549        )
5550        .unwrap();
5551
5552        let removed = s.gc();
5553        assert_eq!(
5554            removed, 0,
5555            "a run completed 1 day ago must survive the 30-day age cap, \
5556             even if it started 40 days ago"
5557        );
5558        let remaining = s.list_runs("agent-a");
5559        assert_eq!(remaining.len(), 1);
5560        assert_eq!(remaining[0].run_id, "long");
5561    }
5562
5563    /// FIX 4: a crash-orphaned run (RunStarted + Turn, no RunEnded) on disk
5564    /// reads `InProgress`. A fresh store's `adopt_orphans()` at boot — when
5565    /// the in-memory map is empty — appends an `Incomplete` terminal so the
5566    /// run becomes terminal (and thus age-GC-eligible), no longer leaking.
5567    #[test]
5568    fn adopt_orphans_marks_crashed_inprogress_runs_incomplete() {
5569        let tmp = tempfile::TempDir::new().unwrap();
5570        let root = tmp.path().join("runs");
5571        // Prior process wrote a start + a turn, then crashed (no terminal).
5572        let s1 = store(root.clone());
5573        s1.write_started(&started("orphan", "agent-a", Utc::now()))
5574            .unwrap();
5575        s1.append_turns("agent-a", "orphan", &[turn(0, "first")])
5576            .unwrap();
5577        assert_eq!(
5578            s1.list_runs("agent-a")[0].status,
5579            RunStatus::InProgress,
5580            "precondition: orphan reads InProgress before adoption"
5581        );
5582
5583        // Fresh store = new process boot, empty in-memory map.
5584        let s2 = store(root);
5585        let adopted = s2.adopt_orphans();
5586        assert_eq!(adopted, 1, "the crash orphan should be adopted");
5587
5588        let after = &s2.list_runs("agent-a")[0];
5589        assert_eq!(
5590            after.status,
5591            RunStatus::Incomplete,
5592            "adopted orphan now reads Incomplete (terminal)"
5593        );
5594        assert!(after.ended_at.is_some(), "terminal record has an ended_at");
5595
5596        // Idempotent: a second boot finds no orphans (already terminal).
5597        assert_eq!(s2.adopt_orphans(), 0);
5598    }
5599
5600    fn write_truncated_private_file(path: &Path) {
5601        let parent = path.parent().unwrap();
5602        car_secrets::ensure_private_dir(parent).unwrap();
5603        let mut file = car_secrets::create_private_file(path).unwrap();
5604        file.write_all(b"{").unwrap();
5605        file.sync_all().unwrap();
5606    }
5607
5608    #[test]
5609    fn corrupt_execution_marker_quarantines_orphan_adoption() {
5610        let tmp = tempfile::TempDir::new().unwrap();
5611        let root = tmp.path().join("runs");
5612        let s = store(root);
5613        s.write_started(&started("marker-corrupt", "agent-a", Utc::now()))
5614            .unwrap();
5615        write_truncated_private_file(&s.proposal_execution_path("marker-corrupt"));
5616
5617        assert!(s.execution_marker("absent-marker").unwrap().is_none());
5618        assert!(s.execution_marker("marker-corrupt").is_err());
5619
5620        assert_eq!(
5621            s.adopt_orphans(),
5622            0,
5623            "present-but-invalid marker is outcome-unknown, never an adoptable absence"
5624        );
5625        assert_eq!(s.list_runs("agent-a")[0].status, RunStatus::InProgress);
5626    }
5627
5628    #[test]
5629    fn corrupt_finalization_outbox_quarantines_orphan_adoption() {
5630        let tmp = tempfile::TempDir::new().unwrap();
5631        let root = tmp.path().join("runs");
5632        let s = store(root);
5633        s.write_started(&started("outbox-corrupt", "agent-a", Utc::now()))
5634            .unwrap();
5635        write_truncated_private_file(&s.proposal_outbox_path("outbox-corrupt"));
5636
5637        assert!(s.pending_proposal("absent-outbox").unwrap().is_none());
5638        assert!(s.pending_proposal("outbox-corrupt").is_err());
5639
5640        assert_eq!(
5641            s.adopt_orphans(),
5642            0,
5643            "present-but-invalid outbox is outcome-unknown, never an adoptable absence"
5644        );
5645        assert_eq!(s.list_runs("agent-a")[0].status, RunStatus::InProgress);
5646    }
5647
5648    #[test]
5649    fn proposal_sidecars_reject_wrong_run_identity() {
5650        let tmp = tempfile::TempDir::new().unwrap();
5651        let s = store(tmp.path().join("runs"));
5652        let marker = ProposalExecutionMarker {
5653            run_id: "other-run".to_string(),
5654            client_id: "client".to_string(),
5655            requested_policy_session_id: None,
5656            policy_session_id: None,
5657            original_proposal_id: "proposal".to_string(),
5658            original_submission: json!({"id":"proposal","source":"test","actions":[]}),
5659            original_proposal: json!({"id":"proposal","source":"test","actions":[]}),
5660            proposal_digest: "a".repeat(64),
5661        };
5662        let path = s.proposal_execution_path("requested-run");
5663        car_secrets::ensure_private_dir(path.parent().unwrap()).unwrap();
5664        let mut file = car_secrets::create_private_file(&path).unwrap();
5665        serde_json::to_writer(&mut file, &marker).unwrap();
5666        file.sync_all().unwrap();
5667
5668        let error = s.execution_marker("requested-run").unwrap_err();
5669        assert_eq!(error.kind(), std::io::ErrorKind::InvalidData);
5670        assert!(error.to_string().contains("does not match requested run"));
5671
5672        let mut pending = valid_pending("other-outbox-run");
5673        pending.client_id = "other-client".to_string();
5674        let path = s.proposal_outbox_path("requested-outbox-run");
5675        car_secrets::ensure_private_dir(path.parent().unwrap()).unwrap();
5676        let mut file = car_secrets::create_private_file(&path).unwrap();
5677        serde_json::to_writer(&mut file, &pending).unwrap();
5678        file.sync_all().unwrap();
5679        let error = s.pending_proposal("requested-outbox-run").unwrap_err();
5680        assert_eq!(error.kind(), std::io::ErrorKind::InvalidData);
5681        assert!(error.to_string().contains("does not match requested run"));
5682    }
5683
5684    #[test]
5685    fn execution_marker_reader_rejects_semantically_invalid_preimage() {
5686        let tmp = tempfile::TempDir::new().unwrap();
5687        let s = store(tmp.path().join("runs"));
5688        let pending = valid_pending("invalid-marker");
5689        let marker = ProposalExecutionMarker {
5690            run_id: pending.run_id.clone(),
5691            client_id: pending.client_id.clone(),
5692            requested_policy_session_id: None,
5693            policy_session_id: None,
5694            original_proposal_id: pending.original_proposal_id.clone(),
5695            original_submission: pending.original_submission.clone(),
5696            original_proposal: serde_json::to_value(&pending.original_proposal).unwrap(),
5697            proposal_digest: "A".repeat(64),
5698        };
5699        let path = s.proposal_execution_path(&pending.run_id);
5700        car_secrets::ensure_private_dir(path.parent().unwrap()).unwrap();
5701        let mut file = car_secrets::create_private_file(&path).unwrap();
5702        serde_json::to_writer(&mut file, &marker).unwrap();
5703        file.sync_all().unwrap();
5704
5705        let error = s.execution_marker(&pending.run_id).unwrap_err();
5706        assert_eq!(error.kind(), std::io::ErrorKind::InvalidData);
5707        assert!(error.to_string().contains("does not bind"));
5708    }
5709
5710    #[test]
5711    fn execution_marker_cleanup_requires_the_full_identity_tuple() {
5712        let tmp = tempfile::TempDir::new().unwrap();
5713        let store = store(tmp.path().join("runs"));
5714        let pending = valid_pending("marker-cleanup-tuple");
5715        let marker = marker_for_pending(&pending);
5716        store.write_execution_marker(&marker).unwrap();
5717
5718        let mut mismatched = marker.clone();
5719        mismatched.requested_policy_session_id = Some("self-claimed-policy".to_string());
5720        assert!(store.clear_execution_marker(&mismatched).is_err());
5721        assert_eq!(
5722            store.execution_marker(&marker.run_id).unwrap(),
5723            Some(marker.clone()),
5724            "tuple mismatch must preserve the exact durable marker"
5725        );
5726
5727        store.clear_execution_marker(&marker).unwrap();
5728        assert!(store.execution_marker(&marker.run_id).unwrap().is_none());
5729    }
5730
5731    #[test]
5732    fn typed_finalization_rejects_invalid_result_and_lineage() {
5733        let tmp = tempfile::TempDir::new().unwrap();
5734        let s = store(tmp.path().join("runs"));
5735        let valid = valid_pending("typed-valid");
5736        valid.validate().unwrap();
5737        write_pending_provenance(&s, &valid);
5738        s.write_pending_proposal(&valid).unwrap();
5739        assert_eq!(s.pending_proposal("typed-valid").unwrap(), Some(valid));
5740
5741        let mut missing_final = valid_pending("typed-missing-final");
5742        missing_final.proposal_result.final_proposal = None;
5743        assert!(missing_final.validate().is_err());
5744
5745        let mut wrong_digest = valid_pending("typed-wrong-digest");
5746        wrong_digest.proposal_result.replan_lineage[0].proposal_digest = Some("A".repeat(64));
5747        assert!(wrong_digest.validate().is_err());
5748
5749        let mut wrong_final = valid_pending("typed-wrong-final");
5750        wrong_final.final_proposal_id = "another-final".to_string();
5751        assert!(wrong_final.validate().is_err());
5752
5753        let mut rejected_tail = valid_pending("typed-rejected-tail");
5754        rejected_tail
5755            .proposal_result
5756            .replan_lineage
5757            .push(ProposalLineageEntry {
5758                generation: 1,
5759                proposal_id: "rejected-candidate".to_string(),
5760                proposal_digest: Some("d".repeat(64)),
5761                status: ProposalLineageStatus::Rejected,
5762                rejection_reason: Some("candidate failed quality gate".to_string()),
5763            });
5764        let value = serde_json::to_value(&rejected_tail.proposal_result).unwrap();
5765        let canonical = car_inference::catalog_identity::canonical_json(&value).unwrap();
5766        rejected_tail.result_digest = format!("{:x}", Sha256::digest(canonical.as_bytes()));
5767        rejected_tail.validate().unwrap();
5768
5769        let mut rejected_without_reason = valid_pending("typed-rejected-no-reason");
5770        rejected_without_reason
5771            .proposal_result
5772            .replan_lineage
5773            .push(ProposalLineageEntry {
5774                generation: 1,
5775                proposal_id: "rejected-candidate".to_string(),
5776                proposal_digest: None,
5777                status: ProposalLineageStatus::Rejected,
5778                rejection_reason: None,
5779            });
5780        assert!(rejected_without_reason.validate().is_err());
5781
5782        let mut rejected_zero_then_accepted = valid_pending("typed-rejected-zero");
5783        rejected_zero_then_accepted.proposal_result.replan_lineage[0].status =
5784            ProposalLineageStatus::Rejected;
5785        rejected_zero_then_accepted.proposal_result.replan_lineage[0].rejection_reason =
5786            Some("original rejected".to_string());
5787        rejected_zero_then_accepted
5788            .proposal_result
5789            .replan_lineage
5790            .push(ProposalLineageEntry {
5791                generation: 1,
5792                proposal_id: rejected_zero_then_accepted.final_proposal_id.clone(),
5793                proposal_digest: Some(
5794                    proposal_digest(&rejected_zero_then_accepted.final_proposal).unwrap(),
5795                ),
5796                status: ProposalLineageStatus::Accepted,
5797                rejection_reason: None,
5798            });
5799        rejected_zero_then_accepted
5800            .accepted_proposal_preimages
5801            .push(AcceptedProposalPreimage {
5802                generation: 1,
5803                proposal: rejected_zero_then_accepted.final_proposal.clone(),
5804            });
5805        refresh_pending_result_digest(&mut rejected_zero_then_accepted);
5806        assert!(rejected_zero_then_accepted.validate().is_err());
5807    }
5808
5809    #[test]
5810    fn active_pending_requires_exact_raw_submission_shape() {
5811        let valid = valid_pending("raw-valid");
5812        valid.validate().unwrap();
5813
5814        for raw in [
5815            Value::Null,
5816            json!([]),
5817            json!({"id": valid.original_proposal_id, "source": "run-store-test"}),
5818            json!({
5819                "id": valid.original_proposal_id,
5820                "source": "run-store-test",
5821                "actions": [{"id":"not-the-typed-action","type":"state_read"}]
5822            }),
5823        ] {
5824            let mut invalid = valid.clone();
5825            invalid.original_submission = raw;
5826            assert!(
5827                invalid.validate().is_err(),
5828                "active pending accepted invalid raw submission: {}",
5829                invalid.original_submission
5830            );
5831        }
5832
5833        let mut with_extra = valid;
5834        with_extra.original_submission["caller_extension"] = json!({"kept": true});
5835        with_extra.validate().unwrap();
5836        assert_eq!(
5837            with_extra.event_data()["original_submission"]["caller_extension"],
5838            json!({"kept": true})
5839        );
5840    }
5841
5842    #[test]
5843    fn fresh_pending_write_requires_exact_run_and_marker_provenance() {
5844        let tmp = tempfile::TempDir::new().unwrap();
5845        let store = store(tmp.path().join("runs"));
5846        let pending = valid_pending("fresh-provenance");
5847
5848        assert!(
5849            store.write_pending_proposal(&pending).is_err(),
5850            "pending without durable RunStarted and execution marker must be rejected"
5851        );
5852        write_pending_provenance(&store, &pending);
5853        store.write_pending_proposal(&pending).unwrap();
5854        assert_eq!(
5855            store.pending_proposal(&pending.run_id).unwrap(),
5856            Some(pending.clone())
5857        );
5858
5859        store.clear_pending_proposal(&pending).unwrap();
5860        let mut client_mismatch = pending.clone();
5861        client_mismatch.client_id = "self-claimed-client".to_string();
5862        let mut requested_policy_mismatch = pending.clone();
5863        requested_policy_mismatch.requested_policy_session_id =
5864            Some("self-claimed-policy".to_string());
5865        let mut run_mismatch = pending.clone();
5866        run_mismatch.run_id = "self-claimed-run".to_string();
5867        let mut original_mismatch = pending;
5868        rebind_pending_original(&mut original_mismatch);
5869        for (name, mismatched) in [
5870            ("client", client_mismatch),
5871            ("requested policy", requested_policy_mismatch),
5872            ("run", run_mismatch),
5873            ("original proposal", original_mismatch),
5874        ] {
5875            assert!(
5876                store.write_pending_proposal(&mismatched).is_err(),
5877                "pending {name} self-claim must not override durable run/marker provenance"
5878            );
5879        }
5880
5881        let mut authenticated = valid_pending("fresh-auth-provenance");
5882        authenticated.requested_policy_session_id = Some("live-policy".to_string());
5883        authenticated.policy_session_id = Some("live-policy".to_string());
5884        write_pending_provenance(&store, &authenticated);
5885        authenticated.policy_session_id = None;
5886        assert!(
5887            store.write_pending_proposal(&authenticated).is_err(),
5888            "pending authenticated policy self-claim must match the durable marker"
5889        );
5890    }
5891
5892    #[test]
5893    fn outbox_reader_requires_active_v3_typed_result_fields() {
5894        let tmp = tempfile::TempDir::new().unwrap();
5895        let s = store(tmp.path().join("runs"));
5896        let pending = valid_pending("strict-result");
5897        let mut value = serde_json::to_value(&pending).unwrap();
5898        value["proposal_result"]
5899            .as_object_mut()
5900            .unwrap()
5901            .remove("cost");
5902        let path = s.proposal_outbox_path("strict-result");
5903        car_secrets::ensure_private_dir(path.parent().unwrap()).unwrap();
5904        let mut file = car_secrets::create_private_file(&path).unwrap();
5905        serde_json::to_writer(&mut file, &value).unwrap();
5906        file.sync_all().unwrap();
5907
5908        let error = s.pending_proposal("strict-result").unwrap_err();
5909        assert_eq!(error.kind(), std::io::ErrorKind::InvalidData);
5910        assert!(error.to_string().contains("missing `cost`"));
5911    }
5912
5913    /// FIX 4 (corollary): a completed run is NOT adopted (it already has a
5914    /// terminal record).
5915    #[test]
5916    fn adopt_orphans_leaves_completed_runs_alone() {
5917        let tmp = tempfile::TempDir::new().unwrap();
5918        let root = tmp.path().join("runs");
5919        let s = store(root);
5920        s.write_started(&started("done", "agent-a", Utc::now()))
5921            .unwrap();
5922        s.append_records(
5923            "agent-a",
5924            "done",
5925            &[ended("done", "agent-a", OutcomeStatus::Success)],
5926        )
5927        .unwrap();
5928        assert_eq!(s.adopt_orphans(), 0);
5929        assert_eq!(s.list_runs("agent-a")[0].status, RunStatus::Completed);
5930    }
5931
5932    /// A torn final tail is tolerated only until another committed record
5933    /// follows it. The separator makes the prior fragment a durable malformed
5934    /// middle row, so the trace must then be quarantined rather than filtered.
5935    #[test]
5936    fn appending_after_torn_tail_quarantines_trace() {
5937        let tmp = tempfile::TempDir::new().unwrap();
5938        let root = tmp.path().join("runs");
5939        let s = store(root.clone());
5940        s.write_started(&started("run-1", "agent-a", Utc::now()))
5941            .unwrap();
5942        s.append_turns("agent-a", "run-1", &[turn(0, "first")])
5943            .unwrap();
5944
5945        // Simulate a torn append: a partial line with NO trailing newline.
5946        let path = root.join("agent-a").join("run-1.jsonl");
5947        {
5948            let mut f = std::fs::OpenOptions::new()
5949                .append(true)
5950                .open(&path)
5951                .unwrap();
5952            // No writeln! — deliberately leaves the file's last byte != '\n'.
5953            f.write_all(b"{\"record\":\"turn\",\"index\":1,\"prom")
5954                .unwrap();
5955        }
5956        let mut opened = car_secrets::open_private_append(&path).unwrap();
5957        assert!(
5958            last_byte_is_not_newline(&mut opened).unwrap(),
5959            "precondition: tail is torn (no trailing newline)"
5960        );
5961        drop(opened);
5962
5963        // A following durable record turns the torn fragment into a committed
5964        // middle row. The append itself is durable but the trace is no longer
5965        // eligible for trusted replay/completion.
5966        let append_error = s
5967            .append_turns("agent-a", "run-1", &[turn(2, "third")])
5968            .expect_err("committed middle corruption must fail the append receipt");
5969        assert!(
5970            append_error.to_string().contains("line 3"),
5971            "{append_error}"
5972        );
5973
5974        let error = s.get_run_trace_checked("run-1").unwrap_err();
5975        assert!(error.to_string().contains("line 3"), "{error}");
5976        let summary = &s.list_runs("agent-a")[0];
5977        assert_eq!(summary.status, RunStatus::Incomplete);
5978        assert_eq!(summary.trace_corruption.as_ref().unwrap().line, 3);
5979    }
5980
5981    #[test]
5982    fn durable_boundaries_fail_closed_and_retry_without_duplicate_rows() {
5983        for point in [
5984            RunStoreFailurePoint::Write,
5985            RunStoreFailurePoint::Flush,
5986            RunStoreFailurePoint::Fsync,
5987        ] {
5988            let tmp = tempfile::TempDir::new().unwrap();
5989            let root = tmp.path().join("runs");
5990            let failures = RunStoreFailureInjector::default();
5991            let store = RunStore::new(root.clone(), RetentionConfig::default())
5992                .with_failure_injector(failures.clone());
5993            let started = started("durable", "agent-a", Utc::now());
5994
5995            failures.fail_next(point);
5996            assert!(
5997                store.write_started(&started).is_err(),
5998                "{point:?} start must not acknowledge"
5999            );
6000            store
6001                .write_started(&started)
6002                .expect("exact start retry reaches durability");
6003
6004            let ended = match ended("durable", "agent-a", OutcomeStatus::Success) {
6005                RunRecord::Ended(ended) => ended,
6006                _ => unreachable!(),
6007            };
6008            failures.fail_next(point);
6009            assert!(
6010                store.write_ended(&ended).is_err(),
6011                "{point:?} terminal must not acknowledge"
6012            );
6013            store
6014                .write_ended(&ended)
6015                .expect("exact terminal retry reaches durability");
6016
6017            let restarted = RunStore::new(root, RetentionConfig::default());
6018            let trace = restarted.get_run_trace("durable").unwrap();
6019            assert_eq!(
6020                trace
6021                    .iter()
6022                    .filter(|row| matches!(row, RunRecord::Started(_)))
6023                    .count(),
6024                1,
6025                "{point:?} retry duplicated RunStarted"
6026            );
6027            assert_eq!(
6028                trace
6029                    .iter()
6030                    .filter(|row| matches!(row, RunRecord::Ended(_)))
6031                    .count(),
6032                1,
6033                "{point:?} retry duplicated RunEnded"
6034            );
6035        }
6036    }
6037
6038    #[test]
6039    fn retained_completed_proposal_survives_run_gc_and_restart() {
6040        let tmp = tempfile::TempDir::new().unwrap();
6041        let root = tmp.path().join("runs");
6042        // An ordinary age cap, with the run stamped old enough to trip it.
6043        // This used `max_per_agent: 0` to evict everything — which worked only
6044        // because a zero cap was read as zero tolerance, the destructive
6045        // reading car#1338 removed. A test leaning on that is part of why it
6046        // stayed invisible: the footgun looked like a feature from in here.
6047        let retention = RetentionConfig::new(DEFAULT_MAX_RUNS_PER_AGENT, DEFAULT_MAX_AGE_DAYS);
6048        let long_ago = Utc::now() - chrono::Duration::days(DEFAULT_MAX_AGE_DAYS + 10);
6049        let store = RunStore::new(root.clone(), retention);
6050        let pending = valid_pending("completed-after-gc");
6051        write_pending_provenance(&store, &pending);
6052        store.write_pending_proposal(&pending).unwrap();
6053        let receipt = store.write_completed_proposal(&pending).unwrap();
6054        store.cleanup_completed_proposal_guards(&receipt).unwrap();
6055        let mut ended = match ended(&pending.run_id, "agent-a", OutcomeStatus::Success) {
6056            RunRecord::Ended(ended) => ended,
6057            _ => unreachable!(),
6058        };
6059        // Backdate the terminal so the age cap takes it. `write_ended` verifies
6060        // the digest, which covers the `RunTermination` and not this stamp, so
6061        // it stays valid — and this field is exactly what the age cap reads.
6062        ended.ended_at = long_ago;
6063        store.write_ended(&ended).unwrap();
6064
6065        assert_eq!(store.gc(), 1, "the completed run trace must be evicted");
6066        assert!(store.get_run_trace(&pending.run_id).is_none());
6067        assert_eq!(
6068            store
6069                .completed_proposal(
6070                    &pending.run_id,
6071                    &pending.client_id,
6072                    pending.requested_policy_session_id.as_deref(),
6073                    &pending.original_submission,
6074                )
6075                .unwrap(),
6076            Some(receipt.clone()),
6077            "receipt validity must not depend on a GC-eligible RunStarted"
6078        );
6079
6080        let restarted = RunStore::new(root, retention);
6081        assert_eq!(
6082            restarted
6083                .completed_proposal_retry_owner(
6084                    pending.requested_policy_session_id.as_deref(),
6085                    &pending.original_submission,
6086                )
6087                .unwrap(),
6088            Some((pending.run_id.clone(), pending.client_id.clone())),
6089            "the retry tuple owner must remain recoverable after restart"
6090        );
6091        assert_eq!(restarted.all_completed_proposals().unwrap(), vec![receipt]);
6092    }
6093
6094    #[test]
6095    fn resumed_completed_proposal_lookup_requires_one_exact_receipt() {
6096        let tmp = tempfile::TempDir::new().unwrap();
6097        let store = store(tmp.path().join("runs"));
6098        let mut first = valid_pending("resumed-policy-rotation");
6099        first.requested_policy_session_id = Some("original-policy-one".to_string());
6100        first.policy_session_id = first.requested_policy_session_id.clone();
6101        write_pending_provenance(&store, &first);
6102        store.write_pending_proposal(&first).unwrap();
6103        let first_receipt = store.write_completed_proposal(&first).unwrap();
6104        store
6105            .cleanup_completed_proposal_guards(&first_receipt)
6106            .unwrap();
6107
6108        assert_eq!(
6109            store
6110                .completed_proposal_for_resumed_owner(
6111                    &first.run_id,
6112                    &first.client_id,
6113                    &first.original_submission,
6114                )
6115                .unwrap(),
6116            Some(first_receipt),
6117            "a single exact receipt is recoverable without the closed policy-session id"
6118        );
6119        assert!(
6120            store
6121                .completed_proposal_for_resumed_owner(
6122                    &first.run_id,
6123                    &first.client_id,
6124                    &json!({"id":"different"}),
6125                )
6126                .unwrap()
6127                .is_none(),
6128            "a different proposal must not match"
6129        );
6130
6131        let mut second = first.clone();
6132        second.requested_policy_session_id = Some("original-policy-two".to_string());
6133        second.policy_session_id = second.requested_policy_session_id.clone();
6134        store
6135            .write_execution_marker(&marker_for_pending(&second))
6136            .unwrap();
6137        store.write_pending_proposal(&second).unwrap();
6138        let second_receipt = store.write_completed_proposal(&second).unwrap();
6139        store
6140            .cleanup_completed_proposal_guards(&second_receipt)
6141            .unwrap();
6142
6143        let error = store
6144            .completed_proposal_for_resumed_owner(
6145                &first.run_id,
6146                &first.client_id,
6147                &first.original_submission,
6148            )
6149            .unwrap_err();
6150        assert_eq!(error.kind(), std::io::ErrorKind::InvalidData);
6151        assert!(error.to_string().contains("ambiguous"));
6152    }
6153
6154    #[test]
6155    fn resumed_completed_proposal_lookup_enforces_receipt_count_before_deserialization() {
6156        let tmp = tempfile::TempDir::new().unwrap();
6157        let store = store(tmp.path().join("runs"));
6158        let target = valid_pending("resumed-receipt-count-limit");
6159        let target_receipt = write_completed_receipt_fixture(&store, target.clone(), None, false);
6160        for index in 1..MAX_RESUMED_PROPOSAL_RECEIPTS_PER_RUN {
6161            let mut unrelated = target.clone();
6162            unrelated.client_id = format!("unrelated-client-{index}");
6163            write_completed_receipt_fixture(&store, unrelated, None, false);
6164        }
6165
6166        assert_eq!(
6167            store
6168                .completed_proposal_for_resumed_owner(
6169                    &target.run_id,
6170                    &target.client_id,
6171                    &target.original_submission,
6172                )
6173                .unwrap(),
6174            Some(target_receipt),
6175            "the exact receipt-count limit remains recoverable"
6176        );
6177
6178        let corrupt_path = store
6179            .completed_response_run_root(&target.run_id)
6180            .join("count-plus-one-is-never-deserialized.json");
6181        let mut corrupt = store.create_private_file(&corrupt_path).unwrap();
6182        corrupt.write_all(b"not-json").unwrap();
6183        corrupt.sync_all().unwrap();
6184        let error = store
6185            .completed_proposal_for_resumed_owner(
6186                &target.run_id,
6187                &target.client_id,
6188                &target.original_submission,
6189            )
6190            .unwrap_err();
6191        assert_eq!(error.kind(), std::io::ErrorKind::InvalidData);
6192        assert!(error.to_string().contains("receipt count limit"));
6193    }
6194
6195    #[test]
6196    fn resumed_completed_proposal_lookup_enforces_byte_limit_before_deserialization() {
6197        let at_limit = tempfile::TempDir::new().unwrap();
6198        let at_limit_store = store(at_limit.path().join("runs"));
6199        let target = valid_pending("resumed-receipt-byte-limit");
6200        let first_receipt_bytes = MAX_RESUMED_PROPOSAL_RECEIPT_BYTES_PER_RUN as usize / 2;
6201        let target_receipt = write_completed_receipt_fixture(
6202            &at_limit_store,
6203            target.clone(),
6204            Some(first_receipt_bytes),
6205            false,
6206        );
6207        let mut unrelated = target.clone();
6208        unrelated.client_id = "unrelated-byte-limit-client".to_string();
6209        write_completed_receipt_fixture(
6210            &at_limit_store,
6211            unrelated,
6212            Some(MAX_RESUMED_PROPOSAL_RECEIPT_BYTES_PER_RUN as usize - first_receipt_bytes),
6213            false,
6214        );
6215        assert_eq!(
6216            at_limit_store
6217                .completed_proposal_for_resumed_owner(
6218                    &target.run_id,
6219                    &target.client_id,
6220                    &target.original_submission,
6221                )
6222                .unwrap(),
6223            Some(target_receipt),
6224            "the exact cumulative-byte limit remains recoverable"
6225        );
6226
6227        let cumulative_over_limit = tempfile::TempDir::new().unwrap();
6228        let cumulative_over_limit_store = store(cumulative_over_limit.path().join("runs"));
6229        write_completed_receipt_fixture(
6230            &cumulative_over_limit_store,
6231            target.clone(),
6232            Some(first_receipt_bytes),
6233            false,
6234        );
6235        let mut unrelated = target.clone();
6236        unrelated.client_id = "unrelated-byte-over-limit-client".to_string();
6237        write_completed_receipt_fixture(
6238            &cumulative_over_limit_store,
6239            unrelated,
6240            Some(MAX_RESUMED_PROPOSAL_RECEIPT_BYTES_PER_RUN as usize - first_receipt_bytes + 1),
6241            false,
6242        );
6243        let error = cumulative_over_limit_store
6244            .completed_proposal_for_resumed_owner(
6245                &target.run_id,
6246                &target.client_id,
6247                &target.original_submission,
6248            )
6249            .unwrap_err();
6250        assert_eq!(error.kind(), std::io::ErrorKind::InvalidData);
6251        assert!(error.to_string().contains("receipt byte limit"));
6252
6253        let invalid_over_limit = tempfile::TempDir::new().unwrap();
6254        let invalid_over_limit_store = store(invalid_over_limit.path().join("runs"));
6255        write_completed_receipt_fixture(
6256            &invalid_over_limit_store,
6257            target.clone(),
6258            Some(MAX_RESUMED_PROPOSAL_RECEIPT_BYTES_PER_RUN as usize),
6259            true,
6260        );
6261        let error = invalid_over_limit_store
6262            .completed_proposal_for_resumed_owner(
6263                &target.run_id,
6264                &target.client_id,
6265                &target.original_submission,
6266            )
6267            .unwrap_err();
6268        assert_eq!(error.kind(), std::io::ErrorKind::InvalidData);
6269        assert!(
6270            error.to_string().contains("receipt byte limit"),
6271            "the byte ceiling must reject an oversized corrupt receipt before deserialization"
6272        );
6273    }
6274
6275    #[test]
6276    fn resumed_completed_proposal_lookup_rejects_corrupt_receipt_under_limits() {
6277        let tmp = tempfile::TempDir::new().unwrap();
6278        let store = store(tmp.path().join("runs"));
6279        let target = valid_pending("resumed-corrupt-receipt");
6280        let run_root = store.completed_response_run_root(&target.run_id);
6281        store.ensure_private_dir(&run_root).unwrap();
6282        let path = store
6283            .completed_response_path(
6284                &target.run_id,
6285                &target.client_id,
6286                target.requested_policy_session_id.as_deref(),
6287                &target.original_submission,
6288            )
6289            .unwrap();
6290        let mut corrupt = store.create_private_file(&path).unwrap();
6291        corrupt.write_all(b"not-json").unwrap();
6292        corrupt.sync_all().unwrap();
6293
6294        let error = store
6295            .completed_proposal_for_resumed_owner(
6296                &target.run_id,
6297                &target.client_id,
6298                &target.original_submission,
6299            )
6300            .unwrap_err();
6301        assert_eq!(error.kind(), std::io::ErrorKind::InvalidData);
6302        assert!(!error.to_string().contains("receipt count limit"));
6303        assert!(!error.to_string().contains("receipt byte limit"));
6304    }
6305
6306    #[test]
6307    fn completed_proposal_owner_lookup_does_not_scan_unrelated_receipts() {
6308        let tmp = tempfile::TempDir::new().unwrap();
6309        let store = store(tmp.path().join("runs"));
6310        let unrelated_dir = store.completed_response_root().join("unrelated-run");
6311        car_secrets::ensure_private_dir(&unrelated_dir).unwrap();
6312        let path = unrelated_dir.join("corrupt.json");
6313        let mut file = car_secrets::create_private_file(&path).unwrap();
6314        file.write_all(b"not-json").unwrap();
6315        file.sync_all().unwrap();
6316
6317        assert_eq!(
6318            store
6319                .completed_proposal_retry_owner(None, &json!({"id": "absent"}))
6320                .unwrap(),
6321            None,
6322            "a bounded content-addressed miss must not enumerate unrelated receipts"
6323        );
6324    }
6325
6326    #[test]
6327    fn proposal_retry_reservation_is_first_writer_wins_and_exact_owner_idempotent() {
6328        let tmp = tempfile::TempDir::new().unwrap();
6329        let store = store(tmp.path().join("runs"));
6330        let submission = json!({"id":"reserved","source":"test","actions":[]});
6331        assert_eq!(
6332            store
6333                .reserve_proposal_retry_owner("first-run", "first-client", None, &submission)
6334                .unwrap(),
6335            ProposalRetryReservation::Acquired
6336        );
6337        for (run_id, client_id) in [
6338            ("first-run", "first-client"),
6339            ("second-run", "second-client"),
6340        ] {
6341            assert_eq!(
6342                store
6343                    .reserve_proposal_retry_owner(run_id, client_id, None, &submission)
6344                    .unwrap(),
6345                ProposalRetryReservation::Existing {
6346                    run_id: "first-run".to_string(),
6347                    client_id: "first-client".to_string(),
6348                }
6349            );
6350        }
6351        assert_eq!(
6352            store
6353                .completed_proposal_retry_owner(None, &submission)
6354                .unwrap(),
6355            Some(("first-run".to_string(), "first-client".to_string()))
6356        );
6357    }
6358
6359    #[test]
6360    fn partial_proposal_retry_reservation_fails_closed_without_replacement() {
6361        let tmp = tempfile::TempDir::new().unwrap();
6362        let store = store(tmp.path().join("runs"));
6363        let submission = json!({"id":"partial","source":"test","actions":[]});
6364        let root = store.completed_response_index_root();
6365        car_secrets::ensure_private_dir(&root).unwrap();
6366        let path = store
6367            .completed_response_owner_path(None, &submission)
6368            .unwrap();
6369        let mut file = car_secrets::create_private_file(&path).unwrap();
6370        file.write_all(b"{").unwrap();
6371        file.sync_all().unwrap();
6372
6373        let error = store
6374            .reserve_proposal_retry_owner("partial-run", "partial-client", None, &submission)
6375            .unwrap_err();
6376        assert_eq!(error.kind(), std::io::ErrorKind::InvalidData);
6377        assert_eq!(std::fs::read(&path).unwrap(), b"{");
6378    }
6379
6380    #[test]
6381    fn completed_proposal_owner_claim_rejects_a_different_run_for_the_same_retry_tuple() {
6382        let tmp = tempfile::TempDir::new().unwrap();
6383        let store = store(tmp.path().join("runs"));
6384        let first = valid_pending("owner-first-run");
6385        write_pending_provenance(&store, &first);
6386        store.write_pending_proposal(&first).unwrap();
6387        store.write_completed_proposal(&first).unwrap();
6388
6389        let mut conflicting = first.clone();
6390        conflicting.run_id = "owner-conflicting-run".to_string();
6391        write_pending_provenance(&store, &conflicting);
6392        store.write_pending_proposal(&conflicting).unwrap();
6393        let error = store.write_completed_proposal(&conflicting).unwrap_err();
6394        assert_eq!(error.kind(), std::io::ErrorKind::AlreadyExists);
6395        assert!(error.to_string().contains("different completed response"));
6396        assert_eq!(
6397            store
6398                .completed_proposal_retry_owner(
6399                    first.requested_policy_session_id.as_deref(),
6400                    &first.original_submission,
6401                )
6402                .unwrap(),
6403            Some((first.run_id.clone(), first.client_id.clone())),
6404            "a conflicting writer must not replace the first durable owner"
6405        );
6406        assert!(
6407            store
6408                .completed_proposal(
6409                    &conflicting.run_id,
6410                    &conflicting.client_id,
6411                    conflicting.requested_policy_session_id.as_deref(),
6412                    &conflicting.original_submission,
6413                )
6414                .unwrap()
6415                .is_none(),
6416            "the rejected owner must not publish a receipt"
6417        );
6418    }
6419
6420    #[test]
6421    fn startup_enumeration_backfills_a_legacy_completed_proposal_owner() {
6422        let tmp = tempfile::TempDir::new().unwrap();
6423        let root = tmp.path().join("runs");
6424        let initial = store(root.clone());
6425        let pending = valid_pending("legacy-owner-backfill");
6426        write_pending_provenance(&initial, &pending);
6427        initial.write_pending_proposal(&pending).unwrap();
6428        let receipt = initial.write_completed_proposal(&pending).unwrap();
6429        let owner_path = initial
6430            .completed_response_owner_path(
6431                pending.requested_policy_session_id.as_deref(),
6432                &pending.original_submission,
6433            )
6434            .unwrap();
6435        std::fs::remove_file(&owner_path).unwrap();
6436        sync_directory(owner_path.parent().unwrap()).unwrap();
6437
6438        let restarted = store(root);
6439        assert!(
6440            restarted
6441                .completed_proposal_retry_owner(
6442                    pending.requested_policy_session_id.as_deref(),
6443                    &pending.original_submission,
6444                )
6445                .unwrap()
6446                .is_none(),
6447            "pre-index receipts begin without an ownership claim"
6448        );
6449        assert_eq!(restarted.all_completed_proposals().unwrap(), vec![receipt]);
6450        assert_eq!(
6451            restarted
6452                .completed_proposal_retry_owner(
6453                    pending.requested_policy_session_id.as_deref(),
6454                    &pending.original_submission,
6455                )
6456                .unwrap(),
6457            Some((pending.run_id, pending.client_id)),
6458            "startup receipt enumeration must durably backfill the owner claim"
6459        );
6460    }
6461}