1use car_ir::{ActionProposal, ProposalLineageStatus, ProposalResult};
54use car_proto::{RunRecord, RunTermination};
55use chrono::{DateTime, Utc};
56use serde::{Deserialize, Serialize};
57use serde_json::Value;
58use sha2::{Digest, Sha256};
59use std::collections::{HashMap, VecDeque};
60use std::fs::File;
61use std::io::{BufRead, Read, Seek, SeekFrom, Write};
62use std::path::{Path, PathBuf};
63use std::sync::{Arc, Condvar, Mutex, Weak};
64
65#[cfg(not(target_os = "windows"))]
66fn sync_directory(path: impl AsRef<Path>) -> std::io::Result<()> {
67 File::open(path)?.sync_all()
68}
69
70#[cfg(target_os = "windows")]
71fn sync_directory(_path: impl AsRef<Path>) -> std::io::Result<()> {
72 Ok(())
76}
77
78const COMPLETED_PROPOSAL_OWNER_INDEX_MIGRATION_VERSION: u32 = 2;
79const RUN_SUMMARY_KEY_BYTES: usize = 32;
80const RUN_SUMMARY_INDEX_RECORD_BYTES: usize = 8 + RUN_SUMMARY_KEY_BYTES;
81const RUN_SUMMARY_INDEX_FILE: &str = ".run-summary-index-v2";
82const RUN_SUMMARY_SIDECAR_DIR: &str = ".run-summaries-v2";
83const RUN_TRACE_CORRUPTION_SIDECAR_DIR: &str = ".run-trace-corruption-v1";
84#[cfg(not(test))]
85const MAX_RESUMED_PROPOSAL_RECEIPTS_PER_RUN: usize = 1024;
86#[cfg(test)]
87const MAX_RESUMED_PROPOSAL_RECEIPTS_PER_RUN: usize = 4;
88#[cfg(not(test))]
89const MAX_RESUMED_PROPOSAL_RECEIPT_BYTES_PER_RUN: u64 = 16 * 1024 * 1024;
90#[cfg(test)]
91const MAX_RESUMED_PROPOSAL_RECEIPT_BYTES_PER_RUN: u64 = 64 * 1024;
92
93#[derive(Debug, Serialize, Deserialize)]
94struct CompletedProposalOwnerIndexMigration {
95 version: u32,
96}
97
98pub const DEFAULT_MAX_RUNS_PER_AGENT: usize = 50;
100pub const DEFAULT_MAX_AGE_DAYS: i64 = 30;
102
103#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
110#[serde(rename_all = "snake_case")]
111pub enum RunStatus {
112 InProgress,
114 Completed,
117 Cancelled,
119 Incomplete,
121 CancellationPending,
123}
124
125#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
126#[serde(rename_all = "snake_case")]
127pub enum RunTraceCorruptionKind {
128 MalformedRecord,
129}
130
131#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
132pub struct RunTraceCorruption {
133 pub kind: RunTraceCorruptionKind,
134 pub line: usize,
136}
137
138#[derive(Debug, Clone, Serialize, Deserialize)]
139struct RunTraceCorruptionMarker {
140 run_id: String,
141 agent_id: String,
142 corruption: RunTraceCorruption,
143}
144
145#[derive(Debug, Clone, Serialize, Deserialize)]
148pub struct RunSummary {
149 pub run_id: String,
150 pub agent_id: String,
151 pub intent: String,
152 pub started_at: DateTime<Utc>,
153 #[serde(default, skip_serializing_if = "Option::is_none")]
155 pub ended_at: Option<DateTime<Utc>>,
156 pub status: RunStatus,
157 pub turn_count: usize,
159 #[serde(default)]
162 pub sequence: u64,
163 #[serde(default, skip_serializing_if = "Option::is_none")]
165 pub trace_corruption: Option<RunTraceCorruption>,
166}
167
168#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
176pub struct PendingProposalFinalization {
177 pub run_id: String,
178 pub client_id: String,
179 #[serde(default, skip_serializing_if = "Option::is_none")]
183 pub requested_policy_session_id: Option<String>,
184 #[serde(default, skip_serializing_if = "Option::is_none")]
185 pub policy_session_id: Option<String>,
186 pub original_proposal_id: String,
187 pub final_proposal_id: String,
188 pub original_submission: Value,
191 pub original_proposal: ActionProposal,
193 pub final_proposal: ActionProposal,
197 #[serde(default)]
200 pub accepted_proposal_preimages: Vec<AcceptedProposalPreimage>,
201 #[serde(deserialize_with = "deserialize_active_proposal_result")]
202 pub proposal_result: ProposalResult,
203 pub result_digest: String,
204}
205
206#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
213pub struct CompletedProposalResponse {
214 pub finalization: PendingProposalFinalization,
215}
216
217#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
222struct CompletedProposalOwnership {
223 run_id: String,
224 client_id: String,
225 #[serde(default, skip_serializing_if = "Option::is_none")]
226 requested_policy_session_id: Option<String>,
227 original_submission: Value,
228}
229
230impl CompletedProposalOwnership {
231 fn new(
232 run_id: &str,
233 client_id: &str,
234 requested_policy_session_id: Option<&str>,
235 original_submission: &Value,
236 ) -> Self {
237 Self {
238 run_id: run_id.to_string(),
239 client_id: client_id.to_string(),
240 requested_policy_session_id: requested_policy_session_id.map(str::to_string),
241 original_submission: original_submission.clone(),
242 }
243 }
244}
245
246#[derive(Debug, Clone, PartialEq, Eq)]
248pub enum ProposalRetryReservation {
249 Acquired,
250 Existing { run_id: String, client_id: String },
251}
252
253#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
260pub struct ProposalRetryRollback {
261 pub run_id: String,
262 pub client_id: String,
263 #[serde(default, skip_serializing_if = "Option::is_none")]
264 pub requested_policy_session_id: Option<String>,
265 pub original_submission: Value,
266}
267
268impl ProposalRetryRollback {
269 fn validate(&self) -> std::io::Result<()> {
270 if self.run_id.is_empty()
271 || self.client_id.is_empty()
272 || self.requested_policy_session_id.as_deref() == Some("")
273 {
274 return Err(std::io::Error::new(
275 std::io::ErrorKind::InvalidInput,
276 "proposal retry rollback requires non-empty run/client/policy identities",
277 ));
278 }
279 Ok(())
280 }
281}
282
283#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
284struct ProposalIdClaim {
285 run_id: String,
286 client_id: String,
287 proposal_id: String,
288 original_submission: Value,
289}
290
291#[derive(Debug, Clone, Copy, PartialEq, Eq)]
292pub enum ProposalIdClaimOutcome {
293 Acquired,
294 ExistingExact,
295}
296
297impl CompletedProposalResponse {
298 pub fn proposal_result(&self) -> &ProposalResult {
299 &self.finalization.proposal_result
300 }
301
302 fn validate(&self) -> std::io::Result<()> {
303 self.finalization.validate()
304 }
305
306 fn execution_marker(&self) -> std::io::Result<ProposalExecutionMarker> {
307 let pending = &self.finalization;
308 let original_proposal = serde_json::to_value(&pending.original_proposal)
309 .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
310 let marker = ProposalExecutionMarker {
311 run_id: pending.run_id.clone(),
312 client_id: pending.client_id.clone(),
313 requested_policy_session_id: pending.requested_policy_session_id.clone(),
314 policy_session_id: pending.policy_session_id.clone(),
315 original_proposal_id: pending.original_proposal_id.clone(),
316 original_submission: pending.original_submission.clone(),
317 original_proposal,
318 proposal_digest: proposal_digest(&pending.original_proposal)
319 .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?,
320 };
321 marker.validate()?;
322 Ok(marker)
323 }
324}
325
326#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
327pub struct AcceptedProposalPreimage {
328 pub generation: u32,
329 pub proposal: ActionProposal,
330}
331
332fn deserialize_active_proposal_result<'de, D>(deserializer: D) -> Result<ProposalResult, D::Error>
333where
334 D: serde::Deserializer<'de>,
335{
336 let value = Value::deserialize(deserializer)?;
337 let object = value
338 .as_object()
339 .ok_or_else(|| serde::de::Error::custom("proposal_result must be an object"))?;
340 for field in [
341 "proposal_id",
342 "original_proposal_id",
343 "final_proposal",
344 "replan_lineage",
345 "results",
346 "cost",
347 ] {
348 if !object.contains_key(field) {
349 return Err(serde::de::Error::custom(format!(
350 "active v3 proposal_result is missing `{field}`"
351 )));
352 }
353 }
354 serde_json::from_value(value).map_err(serde::de::Error::custom)
355}
356
357fn is_lowercase_sha256(value: &str) -> bool {
358 value.len() == 64
359 && value
360 .bytes()
361 .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
362}
363
364fn proposal_digest(proposal: &ActionProposal) -> Result<String, String> {
365 let value = serde_json::to_value(proposal).map_err(|error| error.to_string())?;
366 let canonical = car_inference::catalog_identity::canonical_json(&value)?;
367 Ok(format!("{:x}", Sha256::digest(canonical.as_bytes())))
368}
369
370fn validate_original_submission(
371 submission: &Value,
372 proposal: &ActionProposal,
373) -> std::io::Result<()> {
374 let invalid = |message: String| std::io::Error::new(std::io::ErrorKind::InvalidData, message);
375 let object = submission.as_object().ok_or_else(|| {
376 invalid("active proposal original submission must be an object".to_string())
377 })?;
378 if object.get("id").and_then(Value::as_str) != Some(proposal.id.as_str()) {
379 return Err(invalid(
380 "active proposal raw submission requires an exact string id".to_string(),
381 ));
382 }
383 let raw_actions = object.get("actions").ok_or_else(|| {
384 invalid("active proposal raw submission is missing its action array".to_string())
385 })?;
386 let actions: Vec<car_ir::Action> = serde_json::from_value(raw_actions.clone())
387 .map_err(|error| invalid(format!("raw submission actions are invalid: {error}")))?;
388 if actions != proposal.actions {
389 return Err(invalid(
390 "raw submission actions do not match the accepted normal-serde proposal".to_string(),
391 ));
392 }
393 if object
394 .get("source")
395 .is_some_and(|source| source.as_str() != Some(proposal.source.as_str()))
396 {
397 return Err(invalid(
398 "raw submission source does not match the accepted proposal".to_string(),
399 ));
400 }
401 if let Some(context) = object.get("context") {
402 let context: HashMap<String, Value> = serde_json::from_value(context.clone())
403 .map_err(|error| invalid(format!("raw submission context is invalid: {error}")))?;
404 if context != proposal.context {
405 return Err(invalid(
406 "raw submission context does not match the accepted proposal".to_string(),
407 ));
408 }
409 }
410 if let Some(timestamp) = object.get("timestamp") {
411 let timestamp: DateTime<Utc> = serde_json::from_value(timestamp.clone())
412 .map_err(|error| invalid(format!("raw submission timestamp is invalid: {error}")))?;
413 if timestamp != proposal.timestamp {
414 return Err(invalid(
415 "raw submission timestamp does not match the accepted proposal".to_string(),
416 ));
417 }
418 }
419 Ok(())
420}
421
422#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
426pub struct ProposalExecutionMarker {
427 pub run_id: String,
428 pub client_id: String,
429 #[serde(default, skip_serializing_if = "Option::is_none")]
430 pub requested_policy_session_id: Option<String>,
431 #[serde(default, skip_serializing_if = "Option::is_none")]
432 pub policy_session_id: Option<String>,
433 pub original_proposal_id: String,
434 pub original_submission: Value,
435 pub original_proposal: Value,
436 pub proposal_digest: String,
437}
438
439impl ProposalExecutionMarker {
440 pub fn validate(&self) -> std::io::Result<()> {
441 let invalid =
442 |message: String| std::io::Error::new(std::io::ErrorKind::InvalidData, message);
443 if self.run_id.is_empty()
444 || self.client_id.is_empty()
445 || self.original_proposal_id.is_empty()
446 {
447 return Err(invalid(
448 "proposal execution marker requires non-empty run/client/proposal identities"
449 .to_string(),
450 ));
451 }
452 if self.policy_session_id.is_some()
453 && self.policy_session_id != self.requested_policy_session_id
454 {
455 return Err(invalid(
456 "authenticated policy session is not the caller-requested policy session"
457 .to_string(),
458 ));
459 }
460 let proposal: ActionProposal = serde_json::from_value(self.original_proposal.clone())
461 .map_err(|error| invalid(format!("execution marker proposal is invalid: {error}")))?;
462 if proposal.id != self.original_proposal_id {
463 return Err(invalid(
464 "execution marker proposal id does not match original proposal identity"
465 .to_string(),
466 ));
467 }
468 let digest = proposal_digest(&proposal)
469 .map_err(|error| invalid(format!("execution marker proposal JCS failed: {error}")))?;
470 if self.proposal_digest != digest || !is_lowercase_sha256(&self.proposal_digest) {
471 return Err(invalid(
472 "execution marker digest does not bind its exact proposal preimage".to_string(),
473 ));
474 }
475 validate_original_submission(&self.original_submission, &proposal)
476 }
477}
478
479impl PendingProposalFinalization {
480 pub fn validate(&self) -> std::io::Result<()> {
481 let invalid =
482 |message: String| std::io::Error::new(std::io::ErrorKind::InvalidData, message);
483 if self.run_id.is_empty()
484 || self.client_id.is_empty()
485 || self.original_proposal_id.is_empty()
486 || self.final_proposal_id.is_empty()
487 {
488 return Err(invalid(
489 "proposal finalization requires non-empty run/client/proposal identities"
490 .to_string(),
491 ));
492 }
493 if self.policy_session_id.is_some()
494 && self.policy_session_id != self.requested_policy_session_id
495 {
496 return Err(invalid(
497 "authenticated pending policy session is not the requested policy session"
498 .to_string(),
499 ));
500 }
501 if self.original_proposal.id != self.original_proposal_id
502 || self.proposal_result.original_proposal_id != self.original_proposal_id
503 {
504 return Err(invalid(
505 "proposal finalization original proposal identities do not match".to_string(),
506 ));
507 }
508 if self.final_proposal.id != self.final_proposal_id
509 || self.proposal_result.proposal_id != self.final_proposal_id
510 || self.proposal_result.final_proposal.as_ref() != Some(&self.final_proposal)
511 {
512 return Err(invalid(
513 "active v3 proposal finalization final proposal preimage/id do not match"
514 .to_string(),
515 ));
516 }
517 validate_original_submission(&self.original_submission, &self.original_proposal)?;
518
519 let result_value = serde_json::to_value(&self.proposal_result)
520 .map_err(|error| invalid(format!("proposal result serialization failed: {error}")))?;
521 let canonical = car_inference::catalog_identity::canonical_json(&result_value)
522 .map_err(|error| invalid(format!("proposal result JCS failed: {error}")))?;
523 let result_digest = format!("{:x}", Sha256::digest(canonical.as_bytes()));
524 if self.result_digest != result_digest || !is_lowercase_sha256(&self.result_digest) {
525 return Err(invalid(
526 "proposal finalization result digest does not match typed result".to_string(),
527 ));
528 }
529
530 let lineage = &self.proposal_result.replan_lineage;
531 if lineage.is_empty() {
532 return Err(invalid(
533 "active v3 proposal result is missing generation-zero lineage".to_string(),
534 ));
535 }
536 let original_digest = proposal_digest(&self.original_proposal)
537 .map_err(|error| invalid(format!("original proposal JCS failed: {error}")))?;
538 if lineage[0].generation != 0
539 || lineage[0].proposal_id != self.original_proposal_id
540 || lineage[0].proposal_digest.as_deref() != Some(original_digest.as_str())
541 {
542 return Err(invalid(
543 "generation-zero lineage does not bind the exact original proposal preimage"
544 .to_string(),
545 ));
546 }
547 if lineage[0].status == ProposalLineageStatus::Rejected && lineage.len() != 1 {
548 return Err(invalid(
549 "a generation-zero rejection cannot be followed by another generation".to_string(),
550 ));
551 }
552
553 let mut accepted_by_generation = HashMap::new();
554 for accepted in &self.accepted_proposal_preimages {
555 if accepted_by_generation
556 .insert(accepted.generation, &accepted.proposal)
557 .is_some()
558 {
559 return Err(invalid(format!(
560 "duplicate accepted proposal preimage for generation {}",
561 accepted.generation
562 )));
563 }
564 }
565 for (index, entry) in lineage.iter().enumerate() {
566 if entry.generation != u32::try_from(index).unwrap_or(u32::MAX) {
567 return Err(invalid(
568 "proposal lineage generations are not contiguous from zero".to_string(),
569 ));
570 }
571 match entry.status {
572 ProposalLineageStatus::Accepted => {
573 if entry.rejection_reason.is_some() {
574 return Err(invalid(format!(
575 "accepted lineage generation {} cannot carry a rejection reason",
576 entry.generation
577 )));
578 }
579 let proposal =
580 accepted_by_generation
581 .get(&entry.generation)
582 .ok_or_else(|| {
583 invalid(format!(
584 "accepted lineage generation {} is missing its exact proposal preimage",
585 entry.generation
586 ))
587 })?;
588 let digest = proposal_digest(proposal).map_err(|error| {
589 invalid(format!(
590 "accepted proposal generation {} JCS failed: {error}",
591 entry.generation
592 ))
593 })?;
594 if proposal.id != entry.proposal_id
595 || entry.proposal_digest.as_deref() != Some(digest.as_str())
596 || !is_lowercase_sha256(&digest)
597 {
598 return Err(invalid(format!(
599 "accepted lineage generation {} does not bind its exact lowercase-JCS preimage",
600 entry.generation
601 )));
602 }
603 }
604 ProposalLineageStatus::Rejected => {
605 if entry
606 .rejection_reason
607 .as_deref()
608 .is_none_or(|reason| reason.trim().is_empty())
609 {
610 return Err(invalid(format!(
611 "rejected lineage generation {} is missing its exact rejection reason",
612 entry.generation
613 )));
614 }
615 if entry
616 .proposal_digest
617 .as_deref()
618 .is_some_and(|digest| !is_lowercase_sha256(digest))
619 {
620 return Err(invalid(format!(
621 "rejected lineage generation {} has an invalid proposal digest",
622 entry.generation
623 )));
624 }
625 if accepted_by_generation.contains_key(&entry.generation) {
626 return Err(invalid(format!(
627 "rejected lineage generation {} cannot carry an accepted preimage",
628 entry.generation
629 )));
630 }
631 }
632 }
633 }
634 if accepted_by_generation.len()
635 != lineage
636 .iter()
637 .filter(|entry| entry.status == ProposalLineageStatus::Accepted)
638 .count()
639 {
640 return Err(invalid(
641 "proposal finalization contains an unbound accepted proposal preimage".to_string(),
642 ));
643 }
644
645 if let Some(last_accepted) = lineage
646 .iter()
647 .rev()
648 .find(|entry| entry.status == ProposalLineageStatus::Accepted)
649 {
650 let final_preimage = accepted_by_generation
651 .get(&last_accepted.generation)
652 .expect("accepted lineage was validated above");
653 if *final_preimage != &self.final_proposal {
654 return Err(invalid(
655 "most recent accepted proposal is not the active v3 final proposal".to_string(),
656 ));
657 }
658 } else if lineage.len() != 1
659 || lineage[0].status != ProposalLineageStatus::Rejected
660 || self.final_proposal != self.original_proposal
661 {
662 return Err(invalid(
663 "a result without an accepted generation must be a generation-zero rejection"
664 .to_string(),
665 ));
666 }
667
668 let final_action_ids: std::collections::HashSet<_> = self
669 .final_proposal
670 .actions
671 .iter()
672 .map(|action| action.id.as_str())
673 .collect();
674 let result_action_ids: std::collections::HashSet<_> = self
675 .proposal_result
676 .results
677 .iter()
678 .map(|result| result.action_id.as_str())
679 .collect();
680 if final_action_ids.len() != self.final_proposal.actions.len()
681 || result_action_ids.len() != self.proposal_result.results.len()
682 || final_action_ids != result_action_ids
683 {
684 return Err(invalid(
685 "typed proposal results do not match the final proposal action identities"
686 .to_string(),
687 ));
688 }
689 Ok(())
690 }
691
692 fn validate_provenance(
693 &self,
694 started: &car_proto::RunStarted,
695 marker: &ProposalExecutionMarker,
696 ) -> std::io::Result<()> {
697 let invalid =
698 |message: String| std::io::Error::new(std::io::ErrorKind::InvalidData, message);
699 let started_client = started.client_id.as_deref().ok_or_else(|| {
700 invalid("durable RunStarted is missing its client identity".to_string())
701 })?;
702 if started.run_id != self.run_id || started_client != self.client_id {
703 return Err(invalid(
704 "pending run/client identity does not match durable RunStarted".to_string(),
705 ));
706 }
707 let original_proposal = serde_json::to_value(&self.original_proposal).map_err(|error| {
708 invalid(format!(
709 "pending original proposal serialization failed: {error}"
710 ))
711 })?;
712 let original_digest = proposal_digest(&self.original_proposal)
713 .map_err(|error| invalid(format!("pending original proposal JCS failed: {error}")))?;
714 if marker.run_id != self.run_id
715 || marker.client_id != self.client_id
716 || marker.requested_policy_session_id != self.requested_policy_session_id
717 || marker.policy_session_id != self.policy_session_id
718 || marker.original_proposal_id != self.original_proposal_id
719 || marker.original_submission != self.original_submission
720 || marker.original_proposal != original_proposal
721 || marker.proposal_digest != original_digest
722 {
723 return Err(invalid(
724 "pending finalization identity does not match durable execution marker".to_string(),
725 ));
726 }
727 Ok(())
728 }
729
730 pub fn event_data(&self) -> HashMap<String, Value> {
731 HashMap::from([
732 (
733 "original_submission_id".to_string(),
734 Value::from(self.original_proposal_id.clone()),
735 ),
736 (
737 "final_proposal_id".to_string(),
738 Value::from(self.final_proposal_id.clone()),
739 ),
740 (
741 "original_proposal".to_string(),
742 serde_json::to_value(&self.original_proposal).unwrap_or(Value::Null),
743 ),
744 (
745 "final_proposal".to_string(),
746 serde_json::to_value(&self.final_proposal).unwrap_or(Value::Null),
747 ),
748 (
749 "original_submission".to_string(),
750 self.original_submission.clone(),
751 ),
752 (
753 "replan_lineage".to_string(),
754 serde_json::to_value(&self.proposal_result.replan_lineage).unwrap_or(Value::Null),
755 ),
756 (
757 "all_succeeded".to_string(),
758 Value::from(self.proposal_result.all_succeeded()),
759 ),
760 (
761 "action_count".to_string(),
762 Value::from(self.proposal_result.results.len()),
763 ),
764 (
765 "proposal_result".to_string(),
766 serde_json::to_value(&self.proposal_result).unwrap_or(Value::Null),
767 ),
768 (
769 "result_digest".to_string(),
770 Value::from(self.result_digest.clone()),
771 ),
772 ])
773 }
774}
775
776#[derive(Debug, Clone, Copy)]
806pub struct RetentionConfig {
807 max_per_agent: usize,
808 max_age_days: i64,
809}
810
811impl RetentionConfig {
812 pub fn new(max_per_agent: usize, max_age_days: i64) -> Self {
815 Self {
816 max_per_agent,
817 max_age_days,
818 }
819 }
820
821 pub fn count_cap(&self) -> Option<usize> {
823 (self.max_per_agent > 0).then_some(self.max_per_agent)
824 }
825
826 pub fn age_cap_days(&self) -> Option<i64> {
829 self.age_cutoff(Utc::now()).map(|_| self.max_age_days)
830 }
831
832 fn age_cutoff(&self, now: DateTime<Utc>) -> Option<DateTime<Utc>> {
840 if self.max_age_days <= 0 {
841 return None;
842 }
843 chrono::Duration::try_days(self.max_age_days).and_then(|d| now.checked_sub_signed(d))
844 }
845}
846
847impl Default for RetentionConfig {
848 fn default() -> Self {
849 Self::new(DEFAULT_MAX_RUNS_PER_AGENT, DEFAULT_MAX_AGE_DAYS)
850 }
851}
852
853#[derive(Debug, Clone, Default, Deserialize)]
856struct RunsConfigFile {
857 #[serde(default)]
858 runs: RunsSection,
859}
860
861#[derive(Debug, Clone, Default, Deserialize)]
862struct RunsSection {
863 #[serde(default)]
864 max_per_agent: Option<usize>,
865 #[serde(default)]
866 max_age_days: Option<i64>,
867}
868
869impl RetentionConfig {
870 pub fn from_car_dir(car_dir: &Path) -> Self {
879 let mut cfg = Self::default();
880 let path = car_dir.join("config.toml");
881 let Ok(text) = std::fs::read_to_string(&path) else {
882 return cfg;
883 };
884 let Ok(parsed) = toml::from_str::<RunsConfigFile>(&text) else {
885 return cfg;
886 };
887 if let Some(n) = parsed.runs.max_per_agent {
888 cfg.max_per_agent = n;
889 }
890 if let Some(d) = parsed.runs.max_age_days {
891 cfg.max_age_days = d;
892 }
893 if cfg.max_age_days < 0 || (cfg.max_age_days > 0 && cfg.age_cutoff(Utc::now()).is_none()) {
899 tracing::warn!(
900 max_age_days = cfg.max_age_days,
901 path = %path.display(),
902 "[runs] max_age_days is not a usable number of days; treating the age cap as disabled"
903 );
904 }
905 cfg
906 }
907}
908
909#[derive(Debug, Clone)]
916pub struct RunStore {
917 root: PathBuf,
919 retention: RetentionConfig,
920 failures: RunStoreFailureInjector,
921 summary_read_gate: Option<RunStoreSummaryReadGate>,
922 summary_write_gate: Option<RunStoreSummaryWriteGate>,
923 append_gate: Option<RunStoreAppendGate>,
924 lookup_gate: Option<RunStoreLookupGate>,
925 private_path_failures: Option<car_secrets::PrivatePathDurabilityFailureInjector>,
926 append_locks: Arc<Mutex<HashMap<PathBuf, Weak<Mutex<()>>>>>,
927 trace_corruptions: Arc<Mutex<HashMap<(String, String), RunTraceCorruption>>>,
928}
929
930#[derive(Debug, Default)]
931struct SummaryReadGateState {
932 armed: bool,
933 entered: bool,
934}
935
936#[derive(Debug, Clone, Default)]
939pub struct RunStoreSummaryReadGate {
940 state: Arc<(Mutex<SummaryReadGateState>, Condvar)>,
941}
942
943pub type RunStoreAppendGate = RunStoreSummaryReadGate;
944pub type RunStoreLookupGate = RunStoreSummaryReadGate;
945#[doc(hidden)]
946pub type RunStoreSummaryWriteGate = RunStoreSummaryReadGate;
947
948impl RunStoreSummaryReadGate {
949 pub fn block_next(&self) {
950 let (lock, _) = &*self.state;
951 let mut state = lock.lock().expect("run-store gate mutex poisoned");
952 state.armed = true;
953 state.entered = false;
954 }
955
956 pub fn wait_until_entered(&self, timeout: std::time::Duration) -> bool {
957 let (lock, ready) = &*self.state;
958 let state = lock.lock().expect("run-store gate mutex poisoned");
959 let (state, _) = ready
960 .wait_timeout_while(state, timeout, |state| !state.entered)
961 .expect("run-store gate mutex poisoned while waiting");
962 state.entered
963 }
964
965 pub fn release(&self) {
966 let (lock, ready) = &*self.state;
967 let mut state = lock.lock().expect("run-store gate mutex poisoned");
968 state.armed = false;
969 ready.notify_all();
970 }
971
972 fn wait_if_armed(&self) {
973 let (lock, ready) = &*self.state;
974 let mut state = lock.lock().expect("run-store gate mutex poisoned");
975 if !state.armed || state.entered {
976 return;
977 }
978 state.entered = true;
979 ready.notify_all();
980 while state.armed {
981 state = ready
982 .wait(state)
983 .expect("run-store gate mutex poisoned while blocked");
984 }
985 }
986}
987
988#[derive(Debug, Clone)]
989pub struct ProposalTraceEnsure {
990 pub records: Vec<RunRecord>,
991 pub appended: bool,
992}
993
994#[derive(Debug, Clone, Copy, PartialEq, Eq)]
996pub enum RunStoreFailurePoint {
997 MarkerWrite,
998 Write,
999 Flush,
1000 Fsync,
1001 PendingUnlink,
1002 MarkerUnlink,
1003 DirectoryFsync,
1004 ResponseSerialization,
1005 SummaryWrite,
1006 CorruptionMarkerWrite,
1007 CorruptionSummaryInvalidate,
1008}
1009
1010#[derive(Debug, Clone, Default)]
1011pub struct RunStoreFailureInjector {
1012 failures: Arc<Mutex<VecDeque<RunStoreFailurePoint>>>,
1013}
1014
1015fn injected_storage_full() -> std::io::Error {
1016 std::io::Error::new(
1017 std::io::ErrorKind::StorageFull,
1018 "injected run-store storage full",
1019 )
1020}
1021
1022impl RunStoreFailureInjector {
1023 pub fn fail_next(&self, point: RunStoreFailurePoint) {
1024 self.failures
1025 .lock()
1026 .expect("run-store failure injector mutex poisoned")
1027 .push_back(point);
1028 }
1029
1030 fn take(&self, point: RunStoreFailurePoint) -> bool {
1031 let mut failures = self
1032 .failures
1033 .lock()
1034 .expect("run-store failure injector mutex poisoned");
1035 if failures.front() == Some(&point) {
1036 failures.pop_front();
1037 true
1038 } else {
1039 false
1040 }
1041 }
1042}
1043
1044impl RunStore {
1045 pub fn retention(&self) -> RetentionConfig {
1051 self.retention
1052 }
1053
1054 pub fn new(runs_root: PathBuf, retention: RetentionConfig) -> Self {
1055 Self {
1056 root: runs_root,
1057 retention,
1058 failures: RunStoreFailureInjector::default(),
1059 summary_read_gate: None,
1060 summary_write_gate: None,
1061 append_gate: None,
1062 lookup_gate: None,
1063 private_path_failures: None,
1064 append_locks: Arc::new(Mutex::new(HashMap::new())),
1065 trace_corruptions: Arc::new(Mutex::new(HashMap::new())),
1066 }
1067 }
1068
1069 pub fn with_failure_injector(mut self, failures: RunStoreFailureInjector) -> Self {
1070 self.failures = failures;
1071 self
1072 }
1073
1074 pub fn with_summary_read_gate(mut self, gate: RunStoreSummaryReadGate) -> Self {
1075 self.summary_read_gate = Some(gate);
1076 self
1077 }
1078
1079 #[doc(hidden)]
1080 pub fn with_summary_write_gate(mut self, gate: RunStoreSummaryWriteGate) -> Self {
1081 self.summary_write_gate = Some(gate);
1082 self
1083 }
1084
1085 pub fn with_append_gate(mut self, gate: RunStoreAppendGate) -> Self {
1086 self.append_gate = Some(gate);
1087 self
1088 }
1089
1090 pub fn with_lookup_gate(mut self, gate: RunStoreLookupGate) -> Self {
1091 self.lookup_gate = Some(gate);
1092 self
1093 }
1094
1095 pub fn with_private_path_failure_injector(
1097 mut self,
1098 failures: car_secrets::PrivatePathDurabilityFailureInjector,
1099 ) -> Self {
1100 self.private_path_failures = Some(failures);
1101 self
1102 }
1103
1104 fn ensure_private_dir(&self, path: &Path) -> std::io::Result<()> {
1105 match self.private_path_failures.as_ref() {
1106 Some(failures) => car_secrets::ensure_private_dir_with_failure_injector(path, failures),
1107 None => car_secrets::ensure_private_dir(path),
1108 }
1109 }
1110
1111 fn create_private_file(&self, path: &Path) -> std::io::Result<File> {
1112 match self.private_path_failures.as_ref() {
1113 Some(failures) => {
1114 car_secrets::create_private_file_with_failure_injector(path, failures)
1115 }
1116 None => car_secrets::create_private_file(path),
1117 }
1118 }
1119
1120 fn open_private_append(&self, path: &Path) -> std::io::Result<File> {
1121 match self.private_path_failures.as_ref() {
1122 Some(failures) => {
1123 car_secrets::open_private_append_with_failure_injector(path, failures)
1124 }
1125 None => car_secrets::open_private_append(path),
1126 }
1127 }
1128
1129 pub fn from_journal_dir(journal_dir: &Path) -> Self {
1135 let car_dir = journal_dir
1136 .parent()
1137 .map(Path::to_path_buf)
1138 .unwrap_or_else(|| PathBuf::from("."));
1139 let root = car_dir.join("runs");
1140 let retention = RetentionConfig::from_car_dir(&car_dir);
1141 Self::new(root, retention)
1142 }
1143
1144 pub fn root(&self) -> &Path {
1146 &self.root
1147 }
1148
1149 fn proposal_outbox_root(&self) -> PathBuf {
1150 self.root
1151 .parent()
1152 .unwrap_or_else(|| Path::new("."))
1153 .join("proposal-finalization")
1154 }
1155
1156 fn proposal_outbox_path(&self, run_id: &str) -> PathBuf {
1157 let key = format!("{:x}", Sha256::digest(run_id.as_bytes()));
1158 self.proposal_outbox_root().join(format!("{key}.json"))
1159 }
1160
1161 fn proposal_execution_root(&self) -> PathBuf {
1162 self.root
1163 .parent()
1164 .unwrap_or_else(|| Path::new("."))
1165 .join("proposal-execution")
1166 }
1167
1168 fn proposal_execution_path(&self, run_id: &str) -> PathBuf {
1169 let key = format!("{:x}", Sha256::digest(run_id.as_bytes()));
1170 self.proposal_execution_root().join(format!("{key}.json"))
1171 }
1172
1173 fn proposal_retry_rollback_root(&self) -> PathBuf {
1174 self.root
1175 .parent()
1176 .unwrap_or_else(|| Path::new("."))
1177 .join("proposal-retry-rollbacks")
1178 }
1179
1180 fn proposal_retry_rollback_path(&self, run_id: &str) -> PathBuf {
1181 let key = format!("{:x}", Sha256::digest(run_id.as_bytes()));
1182 self.proposal_retry_rollback_root()
1183 .join(format!("{key}.json"))
1184 }
1185
1186 fn proposal_id_claim_root(&self, run_id: &str) -> PathBuf {
1187 let run_key = format!("{:x}", Sha256::digest(run_id.as_bytes()));
1188 self.root
1189 .parent()
1190 .unwrap_or_else(|| Path::new("."))
1191 .join("proposal-id-claims")
1192 .join(run_key)
1193 }
1194
1195 fn proposal_id_claim_path(&self, run_id: &str, proposal_id: &str) -> PathBuf {
1196 let proposal_key = format!("{:x}", Sha256::digest(proposal_id.as_bytes()));
1197 self.proposal_id_claim_root(run_id)
1198 .join(format!("{proposal_key}.json"))
1199 }
1200
1201 fn completed_response_root(&self) -> PathBuf {
1202 self.root
1203 .parent()
1204 .unwrap_or_else(|| Path::new("."))
1205 .join("proposal-completed")
1206 }
1207
1208 fn completed_response_run_root(&self, run_id: &str) -> PathBuf {
1209 let run_key = format!("{:x}", Sha256::digest(run_id.as_bytes()));
1210 self.completed_response_root().join(run_key)
1211 }
1212
1213 fn completed_response_index_root(&self) -> PathBuf {
1214 self.root
1215 .parent()
1216 .unwrap_or_else(|| Path::new("."))
1217 .join("proposal-completed-index")
1218 }
1219
1220 fn completed_response_index_migration_path(&self) -> PathBuf {
1221 self.completed_response_index_root()
1222 .join("owner-index-migration.json")
1223 }
1224
1225 fn completed_response_owner_key(
1226 requested_policy_session_id: Option<&str>,
1227 original_submission: &Value,
1228 ) -> std::io::Result<String> {
1229 let key_preimage = serde_json::to_vec(&(requested_policy_session_id, original_submission))
1230 .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
1231 Ok(format!("{:x}", Sha256::digest(key_preimage)))
1232 }
1233
1234 fn completed_response_owner_path(
1235 &self,
1236 requested_policy_session_id: Option<&str>,
1237 original_submission: &Value,
1238 ) -> std::io::Result<PathBuf> {
1239 Ok(self.completed_response_index_root().join(format!(
1240 "{}.json",
1241 Self::completed_response_owner_key(requested_policy_session_id, original_submission,)?
1242 )))
1243 }
1244
1245 fn completed_response_key(
1246 client_id: &str,
1247 requested_policy_session_id: Option<&str>,
1248 original_submission: &Value,
1249 ) -> std::io::Result<String> {
1250 let key_preimage =
1251 serde_json::to_vec(&(client_id, requested_policy_session_id, original_submission))
1252 .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
1253 Ok(format!("{:x}", Sha256::digest(key_preimage)))
1254 }
1255
1256 fn completed_response_path(
1257 &self,
1258 run_id: &str,
1259 client_id: &str,
1260 requested_policy_session_id: Option<&str>,
1261 original_submission: &Value,
1262 ) -> std::io::Result<PathBuf> {
1263 Ok(self.completed_response_run_root(run_id).join(format!(
1264 "{}.json",
1265 Self::completed_response_key(
1266 client_id,
1267 requested_policy_session_id,
1268 original_submission,
1269 )?
1270 )))
1271 }
1272
1273 pub fn run_started(&self, run_id: &str) -> std::io::Result<Option<car_proto::RunStarted>> {
1278 if let Some(gate) = &self.lookup_gate {
1279 gate.wait_if_armed();
1280 }
1281 match self.open_root_for_read() {
1282 Ok(()) => {}
1283 Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
1284 Err(error) => return Err(error),
1285 }
1286 let file_name = format!("{}.jsonl", sanitize(run_id));
1287 let mut found = None;
1288 for agent in std::fs::read_dir(&self.root)? {
1289 let agent = agent?;
1290 let file_type = agent.file_type()?;
1291 if !file_type.is_dir() || file_type.is_symlink() {
1292 continue;
1293 }
1294 car_secrets::ensure_private_dir(&agent.path())?;
1295 let path = agent.path().join(&file_name);
1296 match car_secrets::open_private_read(&path) {
1297 Ok(file) if found.is_none() => found = Some((path, file)),
1298 Ok(_) => {
1299 return Err(std::io::Error::new(
1300 std::io::ErrorKind::InvalidData,
1301 format!("durable run `{run_id}` exists under multiple agents"),
1302 ))
1303 }
1304 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
1305 Err(error) => return Err(error),
1306 }
1307 }
1308 let Some((path, file)) = found else {
1309 return Ok(None);
1310 };
1311 let file_len = file.metadata()?.len();
1312 let records = load_private_records(&path, &file)?;
1313 if records.is_empty() && file_len == 0 {
1314 return Ok(None);
1315 }
1316 let mut started_rows = records.iter().filter_map(|record| match record {
1317 RunRecord::Started(started) => Some(started),
1318 _ => None,
1319 });
1320 let started = started_rows.next().ok_or_else(|| {
1321 std::io::Error::new(
1322 std::io::ErrorKind::InvalidData,
1323 format!("durable run record for `{run_id}` has no RunStarted"),
1324 )
1325 })?;
1326 if started_rows.next().is_some()
1327 || started.run_id != run_id
1328 || started.client_id.as_deref().is_none_or(str::is_empty)
1329 {
1330 return Err(std::io::Error::new(
1331 std::io::ErrorKind::InvalidData,
1332 format!("durable RunStarted identity for `{run_id}` is ambiguous or invalid"),
1333 ));
1334 }
1335 Ok(Some(started.clone()))
1336 }
1337
1338 fn durable_run_started(&self, run_id: &str) -> std::io::Result<car_proto::RunStarted> {
1339 self.run_started(run_id)?.ok_or_else(|| {
1340 std::io::Error::new(
1341 std::io::ErrorKind::NotFound,
1342 format!("durable RunStarted for run `{run_id}` is absent"),
1343 )
1344 })
1345 }
1346
1347 pub fn rollback_empty_run_start(&self, started: &car_proto::RunStarted) -> std::io::Result<()> {
1352 let dir = match self.open_agent_dir_for_read(&started.agent_id) {
1353 Ok(dir) => dir,
1354 Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()),
1355 Err(error) => return Err(error),
1356 };
1357 let path = self.run_path(&started.agent_id, &started.run_id);
1358 let append_lock = self.append_lock(&path);
1359 let _guard = append_lock
1360 .lock()
1361 .map_err(|_| std::io::Error::other("run append lock poisoned"))?;
1362 let file = match car_secrets::open_private_read(&path) {
1363 Ok(file) => file,
1364 Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()),
1365 Err(error) => return Err(error),
1366 };
1367 car_secrets::revalidate_private_path(&path, &file)?;
1368 if file.metadata()?.len() != 0 {
1369 return Err(std::io::Error::new(
1370 std::io::ErrorKind::AlreadyExists,
1371 format!(
1372 "run `{}` acquired durable bytes while empty-start rollback was pending",
1373 started.run_id
1374 ),
1375 ));
1376 }
1377 drop(file);
1378 std::fs::remove_file(&path)?;
1379 sync_directory(dir)
1380 }
1381
1382 pub fn pending_provenance(
1383 &self,
1384 pending: &PendingProposalFinalization,
1385 ) -> std::io::Result<(car_proto::RunStarted, ProposalExecutionMarker)> {
1386 pending.validate()?;
1387 let started = self.durable_run_started(&pending.run_id)?;
1388 let marker = self.execution_marker(&pending.run_id)?.ok_or_else(|| {
1389 std::io::Error::new(
1390 std::io::ErrorKind::InvalidData,
1391 "pending finalization is missing its durable execution marker",
1392 )
1393 })?;
1394 pending.validate_provenance(&started, &marker)?;
1395 Ok((started, marker))
1396 }
1397
1398 pub fn write_execution_marker(&self, marker: &ProposalExecutionMarker) -> std::io::Result<()> {
1400 marker.validate()?;
1401 if self.failures.take(RunStoreFailurePoint::MarkerWrite) {
1402 return Err(std::io::Error::other(
1403 "injected proposal execution marker write failure",
1404 ));
1405 }
1406 let root = self.proposal_execution_root();
1407 self.ensure_private_dir(&root)?;
1408 ensure_backup_excluded(&root)?;
1409 let path = self.proposal_execution_path(&marker.run_id);
1410 match car_secrets::open_private_read(&path) {
1411 Ok(file) => {
1412 let existing: ProposalExecutionMarker = serde_json::from_reader(file)
1413 .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
1414 if existing == *marker {
1415 return Ok(());
1416 }
1417 return Err(std::io::Error::new(
1418 std::io::ErrorKind::AlreadyExists,
1419 "run already has a different execution-in-progress marker",
1420 ));
1421 }
1422 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
1423 Err(error) => return Err(error),
1424 }
1425 let temp = root.join(format!(
1426 ".{}.{}.tmp",
1427 path.file_stem()
1428 .and_then(|value| value.to_str())
1429 .unwrap_or("proposal"),
1430 uuid::Uuid::new_v4().simple()
1431 ));
1432 let write_result = (|| {
1433 let mut file = self.create_private_file(&temp)?;
1434 serde_json::to_writer(&mut file, marker)
1435 .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
1436 file.flush()?;
1437 file.sync_all()?;
1438 car_secrets::revalidate_private_path(&temp, &file)?;
1439 drop(file);
1440 car_secrets::atomic_replace_private_file(&temp, &path)?;
1441 sync_directory(&root)
1442 })();
1443 if write_result.is_err() {
1444 let _ = std::fs::remove_file(&temp);
1445 }
1446 write_result
1447 }
1448
1449 pub fn execution_marker(
1450 &self,
1451 run_id: &str,
1452 ) -> std::io::Result<Option<ProposalExecutionMarker>> {
1453 let path = self.proposal_execution_path(run_id);
1454 let file = match car_secrets::open_private_read(&path) {
1455 Ok(file) => file,
1456 Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
1457 Err(error) => return Err(error),
1458 };
1459 let marker: ProposalExecutionMarker = serde_json::from_reader(file)
1460 .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
1461 if marker.run_id != run_id {
1462 return Err(std::io::Error::new(
1463 std::io::ErrorKind::InvalidData,
1464 format!(
1465 "proposal execution marker run_id `{}` does not match requested run `{run_id}`",
1466 marker.run_id
1467 ),
1468 ));
1469 }
1470 marker.validate()?;
1471 Ok(Some(marker))
1472 }
1473
1474 pub fn write_proposal_retry_rollback(
1478 &self,
1479 rollback: &ProposalRetryRollback,
1480 ) -> std::io::Result<()> {
1481 rollback.validate()?;
1482 let root = self.proposal_retry_rollback_root();
1483 self.ensure_private_dir(&root)?;
1484 ensure_backup_excluded(&root)?;
1485 let path = self.proposal_retry_rollback_path(&rollback.run_id);
1486 match car_secrets::open_private_read(&path) {
1487 Ok(file) => {
1488 let existing: ProposalRetryRollback = serde_json::from_reader(file)
1489 .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
1490 existing.validate()?;
1491 if existing == *rollback {
1492 return Ok(());
1493 }
1494 return Err(std::io::Error::new(
1495 std::io::ErrorKind::AlreadyExists,
1496 "run already has a different proposal retry rollback intent",
1497 ));
1498 }
1499 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
1500 Err(error) => return Err(error),
1501 }
1502 self.write_private_json_atomic(&root, &path, rollback)
1503 }
1504
1505 pub fn proposal_retry_rollback(
1506 &self,
1507 run_id: &str,
1508 ) -> std::io::Result<Option<ProposalRetryRollback>> {
1509 let path = self.proposal_retry_rollback_path(run_id);
1510 let file = match car_secrets::open_private_read(&path) {
1511 Ok(file) => file,
1512 Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
1513 Err(error) => return Err(error),
1514 };
1515 let rollback: ProposalRetryRollback = serde_json::from_reader(file)
1516 .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
1517 rollback.validate()?;
1518 if rollback.run_id != run_id || path != self.proposal_retry_rollback_path(&rollback.run_id)
1519 {
1520 return Err(std::io::Error::new(
1521 std::io::ErrorKind::InvalidData,
1522 "proposal retry rollback path does not match its run identity",
1523 ));
1524 }
1525 Ok(Some(rollback))
1526 }
1527
1528 pub fn clear_proposal_retry_rollback(
1529 &self,
1530 expected: &ProposalRetryRollback,
1531 ) -> std::io::Result<()> {
1532 let Some(existing) = self.proposal_retry_rollback(&expected.run_id)? else {
1533 return Ok(());
1534 };
1535 if existing != *expected {
1536 return Err(std::io::Error::new(
1537 std::io::ErrorKind::InvalidInput,
1538 "proposal retry rollback does not match the exact authority",
1539 ));
1540 }
1541 let root = self.proposal_retry_rollback_root();
1542 std::fs::remove_file(self.proposal_retry_rollback_path(&expected.run_id))?;
1543 sync_directory(root)
1544 }
1545
1546 pub fn reconcile_proposal_retry_rollbacks(&self) -> std::io::Result<usize> {
1550 let root = self.proposal_retry_rollback_root();
1551 let entries = match std::fs::read_dir(&root) {
1552 Ok(entries) => entries,
1553 Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(0),
1554 Err(error) => return Err(error),
1555 };
1556 let mut reconciled = 0usize;
1557 for entry in entries {
1558 let entry = entry?;
1559 if entry.path().extension().and_then(|value| value.to_str()) != Some("json") {
1560 continue;
1561 }
1562 let file = car_secrets::open_private_read(&entry.path())?;
1563 let rollback: ProposalRetryRollback = serde_json::from_reader(file)
1564 .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
1565 rollback.validate()?;
1566 if entry.path() != self.proposal_retry_rollback_path(&rollback.run_id) {
1567 return Err(std::io::Error::new(
1568 std::io::ErrorKind::InvalidData,
1569 "proposal retry rollback filename does not match its run identity",
1570 ));
1571 }
1572 if self.execution_marker(&rollback.run_id)?.is_none()
1573 && self.completed_proposal_retry_owner(
1574 rollback.requested_policy_session_id.as_deref(),
1575 &rollback.original_submission,
1576 )? == Some((rollback.run_id.clone(), rollback.client_id.clone()))
1577 {
1578 self.release_proposal_retry_owner(
1579 &rollback.run_id,
1580 &rollback.client_id,
1581 rollback.requested_policy_session_id.as_deref(),
1582 &rollback.original_submission,
1583 )?;
1584 }
1585 self.clear_proposal_retry_rollback(&rollback)?;
1586 reconciled = reconciled.saturating_add(1);
1587 }
1588 Ok(reconciled)
1589 }
1590
1591 pub fn claim_proposal_id(
1595 &self,
1596 run_id: &str,
1597 client_id: &str,
1598 proposal_id: &str,
1599 original_submission: &Value,
1600 ) -> std::io::Result<ProposalIdClaimOutcome> {
1601 if run_id.is_empty() || client_id.is_empty() || proposal_id.is_empty() {
1602 return Err(std::io::Error::new(
1603 std::io::ErrorKind::InvalidInput,
1604 "proposal id claim requires non-empty run/client/proposal identities",
1605 ));
1606 }
1607 let expected = ProposalIdClaim {
1608 run_id: run_id.to_string(),
1609 client_id: client_id.to_string(),
1610 proposal_id: proposal_id.to_string(),
1611 original_submission: original_submission.clone(),
1612 };
1613 let root = self.proposal_id_claim_root(run_id);
1614 self.ensure_private_dir(&root)?;
1615 ensure_backup_excluded(&root)?;
1616 let path = self.proposal_id_claim_path(run_id, proposal_id);
1617 let lock = self.append_lock(&path);
1618 let _guard = lock
1619 .lock()
1620 .map_err(|_| std::io::Error::other("proposal id claim lock poisoned"))?;
1621 match car_secrets::open_private_read(&path) {
1622 Ok(file) => {
1623 let existing: ProposalIdClaim = serde_json::from_reader(file)
1624 .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
1625 if existing == expected {
1626 return Ok(ProposalIdClaimOutcome::ExistingExact);
1627 }
1628 return Err(std::io::Error::new(
1629 std::io::ErrorKind::AlreadyExists,
1630 "proposal id is already bound to a different submission in this run",
1631 ));
1632 }
1633 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
1634 Err(error) => return Err(error),
1635 }
1636 self.write_private_json_atomic(&root, &path, &expected)?;
1637 Ok(ProposalIdClaimOutcome::Acquired)
1638 }
1639
1640 fn write_private_json_atomic<T: Serialize>(
1641 &self,
1642 root: &Path,
1643 path: &Path,
1644 value: &T,
1645 ) -> std::io::Result<()> {
1646 let temp = root.join(format!(
1647 ".{}.{}.tmp",
1648 path.file_stem()
1649 .and_then(|value| value.to_str())
1650 .unwrap_or("record"),
1651 uuid::Uuid::new_v4().simple()
1652 ));
1653 let write_result = (|| {
1654 let mut file = self.create_private_file(&temp)?;
1655 serde_json::to_writer(&mut file, value)
1656 .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
1657 file.flush()?;
1658 file.sync_all()?;
1659 car_secrets::revalidate_private_path(&temp, &file)?;
1660 drop(file);
1661 car_secrets::atomic_replace_private_file(&temp, path)?;
1662 sync_directory(root)
1663 })();
1664 if write_result.is_err() {
1665 let _ = std::fs::remove_file(&temp);
1666 }
1667 write_result
1668 }
1669
1670 pub fn clear_execution_marker(
1671 &self,
1672 expected: &ProposalExecutionMarker,
1673 ) -> std::io::Result<()> {
1674 let Some(existing) = self.execution_marker(&expected.run_id)? else {
1675 return Ok(());
1676 };
1677 if existing != *expected {
1678 return Err(std::io::Error::new(
1679 std::io::ErrorKind::InvalidInput,
1680 "proposal execution marker does not match the full expected identity tuple",
1681 ));
1682 }
1683 let root = self.proposal_execution_root();
1684 std::fs::remove_file(self.proposal_execution_path(&expected.run_id))?;
1685 sync_directory(root)
1686 }
1687
1688 pub fn write_pending_proposal(
1692 &self,
1693 pending: &PendingProposalFinalization,
1694 ) -> std::io::Result<()> {
1695 self.pending_provenance(pending)?;
1696 if pending.run_id.is_empty()
1697 || pending.client_id.is_empty()
1698 || pending.original_proposal_id.is_empty()
1699 || pending.final_proposal_id.is_empty()
1700 || pending.result_digest.len() != 64
1701 {
1702 return Err(std::io::Error::new(
1703 std::io::ErrorKind::InvalidInput,
1704 "proposal finalization requires run/client/proposal identities and a SHA-256 digest",
1705 ));
1706 }
1707 let root = self.proposal_outbox_root();
1708 self.ensure_private_dir(&root)?;
1709 ensure_backup_excluded(&root)?;
1710 let path = self.proposal_outbox_path(&pending.run_id);
1711 match car_secrets::open_private_read(&path) {
1712 Ok(file) => {
1713 let existing: PendingProposalFinalization = serde_json::from_reader(file)
1714 .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
1715 if existing == *pending {
1716 return Ok(());
1717 }
1718 return Err(std::io::Error::new(
1719 std::io::ErrorKind::AlreadyExists,
1720 "run already has a different pending proposal finalization",
1721 ));
1722 }
1723 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
1724 Err(error) => return Err(error),
1725 }
1726
1727 let temp = root.join(format!(
1728 ".{}.{}.tmp",
1729 path.file_stem()
1730 .and_then(|value| value.to_str())
1731 .unwrap_or("proposal"),
1732 uuid::Uuid::new_v4().simple()
1733 ));
1734 let write_result = (|| {
1735 let mut file = self.create_private_file(&temp)?;
1736 serde_json::to_writer(&mut file, pending)
1737 .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
1738 file.flush()?;
1739 file.sync_all()?;
1740 car_secrets::revalidate_private_path(&temp, &file)?;
1741 drop(file);
1742 car_secrets::atomic_replace_private_file(&temp, &path)?;
1743 sync_directory(&root)
1744 })();
1745 if write_result.is_err() {
1746 let _ = std::fs::remove_file(&temp);
1747 }
1748 write_result
1749 }
1750
1751 pub fn pending_proposal(
1752 &self,
1753 run_id: &str,
1754 ) -> std::io::Result<Option<PendingProposalFinalization>> {
1755 let path = self.proposal_outbox_path(run_id);
1756 let file = match car_secrets::open_private_read(&path) {
1757 Ok(file) => file,
1758 Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
1759 Err(error) => return Err(error),
1760 };
1761 let pending: PendingProposalFinalization = serde_json::from_reader(file)
1762 .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
1763 if pending.run_id != run_id {
1764 return Err(std::io::Error::new(
1765 std::io::ErrorKind::InvalidData,
1766 format!(
1767 "proposal finalization run_id `{}` does not match requested run `{run_id}`",
1768 pending.run_id
1769 ),
1770 ));
1771 }
1772 self.pending_provenance(&pending)?;
1773 Ok(Some(pending))
1774 }
1775
1776 pub fn all_pending_proposals(&self) -> std::io::Result<Vec<PendingProposalFinalization>> {
1777 let root = self.proposal_outbox_root();
1778 let entries = match std::fs::read_dir(root) {
1779 Ok(entries) => entries,
1780 Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
1781 Err(error) => return Err(error),
1782 };
1783 let mut pending = Vec::new();
1784 for entry in entries {
1785 let entry = entry?;
1786 if entry.path().extension().and_then(|ext| ext.to_str()) != Some("json") {
1787 continue;
1788 }
1789 let file = car_secrets::open_private_read(&entry.path())?;
1790 let row: PendingProposalFinalization = serde_json::from_reader(file)
1791 .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
1792 if entry.path() != self.proposal_outbox_path(&row.run_id) {
1793 return Err(std::io::Error::new(
1794 std::io::ErrorKind::InvalidData,
1795 format!(
1796 "proposal finalization filename does not match internal run_id `{}`",
1797 row.run_id
1798 ),
1799 ));
1800 }
1801 self.pending_provenance(&row)?;
1802 pending.push(row);
1803 }
1804 Ok(pending)
1805 }
1806
1807 pub fn clear_pending_proposal(
1811 &self,
1812 expected: &PendingProposalFinalization,
1813 ) -> std::io::Result<()> {
1814 self.pending_provenance(expected)?;
1815 let existing = self.pending_proposal(&expected.run_id)?.ok_or_else(|| {
1816 std::io::Error::new(
1817 std::io::ErrorKind::NotFound,
1818 "proposal finalization outbox row is absent",
1819 )
1820 })?;
1821 if existing != *expected {
1822 return Err(std::io::Error::new(
1823 std::io::ErrorKind::InvalidInput,
1824 "proposal finalization outbox does not match the full expected transaction",
1825 ));
1826 }
1827 let root = self.proposal_outbox_root();
1828 std::fs::remove_file(self.proposal_outbox_path(&expected.run_id))?;
1829 sync_directory(root)
1830 }
1831
1832 fn validate_completed_response(
1836 &self,
1837 receipt: &CompletedProposalResponse,
1838 ) -> std::io::Result<()> {
1839 receipt.validate()?;
1840 Ok(())
1841 }
1842
1843 fn read_completed_proposal_owner(
1844 &self,
1845 requested_policy_session_id: Option<&str>,
1846 original_submission: &Value,
1847 ) -> std::io::Result<Option<CompletedProposalOwnership>> {
1848 let path =
1849 self.completed_response_owner_path(requested_policy_session_id, original_submission)?;
1850 let file = match car_secrets::open_private_read(&path) {
1851 Ok(file) => file,
1852 Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
1853 Err(error) => return Err(error),
1854 };
1855 let owner: CompletedProposalOwnership = serde_json::from_reader(file)
1856 .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
1857 if owner.requested_policy_session_id.as_deref() != requested_policy_session_id
1858 || owner.original_submission != *original_submission
1859 || path
1860 != self.completed_response_owner_path(
1861 owner.requested_policy_session_id.as_deref(),
1862 &owner.original_submission,
1863 )?
1864 || owner.run_id.is_empty()
1865 || owner.client_id.is_empty()
1866 {
1867 return Err(std::io::Error::new(
1868 std::io::ErrorKind::InvalidData,
1869 "completed proposal ownership index does not match its retry tuple",
1870 ));
1871 }
1872 Ok(Some(owner))
1873 }
1874
1875 pub fn reserve_proposal_retry_owner(
1880 &self,
1881 run_id: &str,
1882 client_id: &str,
1883 requested_policy_session_id: Option<&str>,
1884 original_submission: &Value,
1885 ) -> std::io::Result<ProposalRetryReservation> {
1886 if run_id.is_empty() || client_id.is_empty() || requested_policy_session_id == Some("") {
1887 return Err(std::io::Error::new(
1888 std::io::ErrorKind::InvalidInput,
1889 "proposal retry reservation requires non-empty run/client/policy identities",
1890 ));
1891 }
1892 let expected = CompletedProposalOwnership::new(
1893 run_id,
1894 client_id,
1895 requested_policy_session_id,
1896 original_submission,
1897 );
1898 let root = self.completed_response_index_root();
1899 self.ensure_private_dir(&root)?;
1900 ensure_backup_excluded(&root)?;
1901 sync_directory(root.parent().unwrap_or_else(|| Path::new(".")))?;
1902 let path =
1903 self.completed_response_owner_path(requested_policy_session_id, original_submission)?;
1904 let lock = self.append_lock(&path);
1905 let _guard = lock
1906 .lock()
1907 .map_err(|_| std::io::Error::other("proposal retry reservation lock poisoned"))?;
1908 if let Some(existing) =
1909 self.read_completed_proposal_owner(requested_policy_session_id, original_submission)?
1910 {
1911 return Ok(ProposalRetryReservation::Existing {
1912 run_id: existing.run_id,
1913 client_id: existing.client_id,
1914 });
1915 }
1916
1917 let mut file = match self.create_private_file(&path) {
1918 Ok(file) => file,
1919 Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {
1920 let existing = self
1921 .read_completed_proposal_owner(
1922 requested_policy_session_id,
1923 original_submission,
1924 )?
1925 .ok_or_else(|| {
1926 std::io::Error::new(
1927 std::io::ErrorKind::InvalidData,
1928 "proposal retry reservation appeared without a readable owner",
1929 )
1930 })?;
1931 return Ok(ProposalRetryReservation::Existing {
1932 run_id: existing.run_id,
1933 client_id: existing.client_id,
1934 });
1935 }
1936 Err(error) => return Err(error),
1937 };
1938 serde_json::to_writer(&mut file, &expected)
1939 .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
1940 file.flush()?;
1941 file.sync_all()?;
1942 car_secrets::revalidate_private_path(&path, &file)?;
1943 sync_directory(&root)?;
1944 Ok(ProposalRetryReservation::Acquired)
1945 }
1946
1947 pub fn release_proposal_retry_owner(
1953 &self,
1954 run_id: &str,
1955 client_id: &str,
1956 requested_policy_session_id: Option<&str>,
1957 original_submission: &Value,
1958 ) -> std::io::Result<()> {
1959 let path =
1960 self.completed_response_owner_path(requested_policy_session_id, original_submission)?;
1961 let lock = self.append_lock(&path);
1962 let _guard = lock
1963 .lock()
1964 .map_err(|_| std::io::Error::other("proposal retry release lock poisoned"))?;
1965 let Some(existing) =
1966 self.read_completed_proposal_owner(requested_policy_session_id, original_submission)?
1967 else {
1968 return Ok(());
1969 };
1970 if existing.run_id != run_id || existing.client_id != client_id {
1971 return Err(std::io::Error::new(
1972 std::io::ErrorKind::InvalidInput,
1973 "proposal retry owner does not match the exact rollback authority",
1974 ));
1975 }
1976 let root = self.completed_response_index_root();
1977 std::fs::remove_file(path)?;
1978 sync_directory(root)
1979 }
1980
1981 fn claim_completed_proposal_owner(
1985 &self,
1986 pending: &PendingProposalFinalization,
1987 ) -> std::io::Result<()> {
1988 match self.reserve_proposal_retry_owner(
1989 &pending.run_id,
1990 &pending.client_id,
1991 pending.requested_policy_session_id.as_deref(),
1992 &pending.original_submission,
1993 )? {
1994 ProposalRetryReservation::Acquired => Ok(()),
1995 ProposalRetryReservation::Existing { run_id, client_id }
1996 if run_id == pending.run_id && client_id == pending.client_id =>
1997 {
1998 Ok(())
1999 }
2000 ProposalRetryReservation::Existing { .. } => Err(std::io::Error::new(
2001 std::io::ErrorKind::AlreadyExists,
2002 "proposal retry tuple already belongs to a different completed response",
2003 )),
2004 }
2005 }
2006
2007 pub fn write_completed_proposal(
2012 &self,
2013 pending: &PendingProposalFinalization,
2014 ) -> std::io::Result<CompletedProposalResponse> {
2015 self.pending_provenance(pending)?;
2016 let receipt = CompletedProposalResponse {
2017 finalization: pending.clone(),
2018 };
2019 self.validate_completed_response(&receipt)?;
2020 self.claim_completed_proposal_owner(pending)?;
2021 let root = self.completed_response_root();
2022 self.ensure_private_dir(&root)?;
2023 ensure_backup_excluded(&root)?;
2024 sync_directory(root.parent().unwrap_or_else(|| Path::new(".")))?;
2025 let run_root = self.completed_response_run_root(&pending.run_id);
2026 self.ensure_private_dir(&run_root)?;
2027 sync_directory(&root)?;
2028 let path = self.completed_response_path(
2029 &pending.run_id,
2030 &pending.client_id,
2031 pending.requested_policy_session_id.as_deref(),
2032 &pending.original_submission,
2033 )?;
2034 match car_secrets::open_private_read(&path) {
2035 Ok(file) => {
2036 let existing: CompletedProposalResponse = serde_json::from_reader(file)
2037 .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
2038 self.validate_completed_response(&existing)?;
2039 if existing == receipt {
2040 return Ok(existing);
2041 }
2042 return Err(std::io::Error::new(
2043 std::io::ErrorKind::AlreadyExists,
2044 "exact proposal retry key already has a different completed response",
2045 ));
2046 }
2047 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
2048 Err(error) => return Err(error),
2049 }
2050
2051 let temp = run_root.join(format!(".response.{}.tmp", uuid::Uuid::new_v4().simple()));
2052 let write_result = (|| {
2053 let mut file = self.create_private_file(&temp)?;
2054 serde_json::to_writer(&mut file, &receipt)
2055 .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
2056 file.flush()?;
2057 file.sync_all()?;
2058 car_secrets::revalidate_private_path(&temp, &file)?;
2059 drop(file);
2060 car_secrets::atomic_replace_private_file(&temp, &path)?;
2061 sync_directory(&run_root)
2062 })();
2063 if write_result.is_err() {
2064 let _ = std::fs::remove_file(&temp);
2065 }
2066 write_result.map(|()| receipt)
2067 }
2068
2069 pub fn completed_proposal(
2073 &self,
2074 run_id: &str,
2075 client_id: &str,
2076 requested_policy_session_id: Option<&str>,
2077 original_submission: &Value,
2078 ) -> std::io::Result<Option<CompletedProposalResponse>> {
2079 let path = self.completed_response_path(
2080 run_id,
2081 client_id,
2082 requested_policy_session_id,
2083 original_submission,
2084 )?;
2085 let file = match car_secrets::open_private_read(&path) {
2086 Ok(file) => file,
2087 Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
2088 Err(error) => return Err(error),
2089 };
2090 let receipt: CompletedProposalResponse = serde_json::from_reader(file)
2091 .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
2092 self.validate_completed_response(&receipt)?;
2093 let pending = &receipt.finalization;
2094 if pending.run_id != run_id
2095 || pending.client_id != client_id
2096 || pending.requested_policy_session_id.as_deref() != requested_policy_session_id
2097 || pending.original_submission != *original_submission
2098 || path
2099 != self.completed_response_path(
2100 &pending.run_id,
2101 &pending.client_id,
2102 pending.requested_policy_session_id.as_deref(),
2103 &pending.original_submission,
2104 )?
2105 {
2106 return Err(std::io::Error::new(
2107 std::io::ErrorKind::InvalidData,
2108 "completed proposal response does not match its exact retry key",
2109 ));
2110 }
2111 Ok(Some(receipt))
2112 }
2113
2114 pub fn completed_proposal_for_resumed_owner(
2121 &self,
2122 run_id: &str,
2123 durable_client_id: &str,
2124 original_submission: &Value,
2125 ) -> std::io::Result<Option<CompletedProposalResponse>> {
2126 let run_root = self.completed_response_run_root(run_id);
2127 let entries = match std::fs::read_dir(&run_root) {
2128 Ok(entries) => entries,
2129 Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
2130 Err(error) => return Err(error),
2131 };
2132 car_secrets::ensure_private_dir(&run_root)?;
2133
2134 let mut receipt_paths = Vec::new();
2135 for entry in entries {
2136 let entry = entry?;
2137 if entry.path().extension().and_then(|ext| ext.to_str()) != Some("json") {
2138 continue;
2139 }
2140 if receipt_paths.len() >= MAX_RESUMED_PROPOSAL_RECEIPTS_PER_RUN {
2141 return Err(std::io::Error::new(
2142 std::io::ErrorKind::InvalidData,
2143 "resumed proposal recovery exceeded the per-run receipt count limit",
2144 ));
2145 }
2146 receipt_paths.push(entry.path());
2147 }
2148
2149 let mut matching = None;
2150 let mut scanned_bytes = 0_u64;
2151 for path in receipt_paths {
2152 let file = car_secrets::open_private_read(&path)?;
2153 scanned_bytes = scanned_bytes
2154 .checked_add(file.metadata()?.len())
2155 .filter(|total| *total <= MAX_RESUMED_PROPOSAL_RECEIPT_BYTES_PER_RUN)
2156 .ok_or_else(|| {
2157 std::io::Error::new(
2158 std::io::ErrorKind::InvalidData,
2159 "resumed proposal recovery exceeded the per-run receipt byte limit",
2160 )
2161 })?;
2162 let receipt: CompletedProposalResponse = serde_json::from_reader(file)
2163 .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
2164 self.validate_completed_response(&receipt)?;
2165 let pending = &receipt.finalization;
2166 if pending.run_id != run_id
2167 || path
2168 != self.completed_response_path(
2169 &pending.run_id,
2170 &pending.client_id,
2171 pending.requested_policy_session_id.as_deref(),
2172 &pending.original_submission,
2173 )?
2174 {
2175 return Err(std::io::Error::new(
2176 std::io::ErrorKind::InvalidData,
2177 "completed proposal response path does not match its typed identity",
2178 ));
2179 }
2180 if pending.client_id != durable_client_id
2181 || pending.original_submission != *original_submission
2182 {
2183 continue;
2184 }
2185 if matching.replace(receipt).is_some() {
2186 return Err(std::io::Error::new(
2187 std::io::ErrorKind::InvalidData,
2188 "resumed proposal recovery is ambiguous for the exact durable owner and submission",
2189 ));
2190 }
2191 }
2192 Ok(matching)
2193 }
2194
2195 pub fn all_completed_proposals(&self) -> std::io::Result<Vec<CompletedProposalResponse>> {
2196 let root = self.completed_response_root();
2197 let run_dirs = match std::fs::read_dir(&root) {
2198 Ok(entries) => entries,
2199 Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
2200 Err(error) => return Err(error),
2201 };
2202 let mut receipts = Vec::new();
2203 for run_dir in run_dirs {
2204 let run_dir = run_dir?;
2205 if !run_dir.file_type()?.is_dir() || run_dir.file_type()?.is_symlink() {
2206 continue;
2207 }
2208 car_secrets::ensure_private_dir(&run_dir.path())?;
2209 for entry in std::fs::read_dir(run_dir.path())? {
2210 let entry = entry?;
2211 if entry.path().extension().and_then(|ext| ext.to_str()) != Some("json") {
2212 continue;
2213 }
2214 let file = car_secrets::open_private_read(&entry.path())?;
2215 let receipt: CompletedProposalResponse = serde_json::from_reader(file)
2216 .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
2217 self.validate_completed_response(&receipt)?;
2218 let pending = &receipt.finalization;
2219 if run_dir.path() != self.completed_response_run_root(&pending.run_id)
2220 || entry.path()
2221 != self.completed_response_path(
2222 &pending.run_id,
2223 &pending.client_id,
2224 pending.requested_policy_session_id.as_deref(),
2225 &pending.original_submission,
2226 )?
2227 {
2228 return Err(std::io::Error::new(
2229 std::io::ErrorKind::InvalidData,
2230 "completed proposal response path does not match its typed identity",
2231 ));
2232 }
2233 self.claim_completed_proposal_owner(pending)?;
2234 receipts.push(receipt);
2235 }
2236 }
2237 Ok(receipts)
2238 }
2239
2240 pub fn reconcile_completed_proposal_migration(&self) -> std::io::Result<()> {
2246 let checkpoint_path = self.completed_response_index_migration_path();
2247 match car_secrets::open_private_read(&checkpoint_path) {
2248 Ok(file) => {
2249 let checkpoint: CompletedProposalOwnerIndexMigration =
2250 serde_json::from_reader(file).map_err(|error| {
2251 std::io::Error::new(std::io::ErrorKind::InvalidData, error)
2252 })?;
2253 if checkpoint.version > COMPLETED_PROPOSAL_OWNER_INDEX_MIGRATION_VERSION {
2254 return Err(std::io::Error::new(
2255 std::io::ErrorKind::InvalidData,
2256 format!(
2257 "unsupported completed-proposal owner-index migration version {}",
2258 checkpoint.version
2259 ),
2260 ));
2261 }
2262 if checkpoint.version == COMPLETED_PROPOSAL_OWNER_INDEX_MIGRATION_VERSION {
2263 return Ok(());
2264 }
2265 }
2269 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
2270 Err(error) => return Err(error),
2271 }
2272
2273 const MAX_LEGACY_RECEIPT_BYTES: u64 = 16 * 1024 * 1024;
2277 let receipts_root = self.completed_response_root();
2278 match std::fs::read_dir(&receipts_root) {
2279 Ok(run_dirs) => {
2280 for run_dir in run_dirs {
2281 let run_dir = run_dir?;
2282 if !run_dir.file_type()?.is_dir() || run_dir.file_type()?.is_symlink() {
2283 continue;
2284 }
2285 car_secrets::ensure_private_dir(&run_dir.path())?;
2286 for entry in std::fs::read_dir(run_dir.path())? {
2287 let entry = entry?;
2288 if entry.path().extension().and_then(|ext| ext.to_str()) != Some("json") {
2289 continue;
2290 }
2291 let file = car_secrets::open_private_read(&entry.path())?;
2292 if file.metadata()?.len() > MAX_LEGACY_RECEIPT_BYTES {
2293 return Err(std::io::Error::new(
2294 std::io::ErrorKind::InvalidData,
2295 "completed proposal receipt exceeds migration byte limit",
2296 ));
2297 }
2298 let receipt: CompletedProposalResponse = serde_json::from_reader(file)
2299 .map_err(|error| {
2300 std::io::Error::new(std::io::ErrorKind::InvalidData, error)
2301 })?;
2302 self.validate_completed_response(&receipt)?;
2303 let pending = &receipt.finalization;
2304 if run_dir.path() != self.completed_response_run_root(&pending.run_id)
2305 || entry.path()
2306 != self.completed_response_path(
2307 &pending.run_id,
2308 &pending.client_id,
2309 pending.requested_policy_session_id.as_deref(),
2310 &pending.original_submission,
2311 )?
2312 {
2313 return Err(std::io::Error::new(
2314 std::io::ErrorKind::InvalidData,
2315 "completed proposal response path does not match its typed identity",
2316 ));
2317 }
2318 self.claim_completed_proposal_owner(pending)?;
2319 if self.run_started(&pending.run_id)?.is_some() {
2320 self.ensure_proposal_turns(pending)?;
2321 }
2322 self.cleanup_completed_proposal_guards(&receipt)?;
2326 }
2327 }
2328 }
2329 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
2330 Err(error) => return Err(error),
2331 }
2332 let root = self.completed_response_index_root();
2333 self.ensure_private_dir(&root)?;
2334 ensure_backup_excluded(&root)?;
2335 sync_directory(root.parent().unwrap_or_else(|| Path::new(".")))?;
2336 let temp = root.join(format!(
2337 ".owner-index-migration.{}.tmp",
2338 uuid::Uuid::new_v4().simple()
2339 ));
2340 let write_result = (|| {
2341 let mut file = self.create_private_file(&temp)?;
2342 serde_json::to_writer(
2343 &mut file,
2344 &CompletedProposalOwnerIndexMigration {
2345 version: COMPLETED_PROPOSAL_OWNER_INDEX_MIGRATION_VERSION,
2346 },
2347 )
2348 .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
2349 file.flush()?;
2350 file.sync_all()?;
2351 car_secrets::revalidate_private_path(&temp, &file)?;
2352 drop(file);
2353 car_secrets::atomic_replace_private_file(&temp, &checkpoint_path)?;
2354 sync_directory(&root)
2355 })();
2356 if write_result.is_err() {
2357 let _ = std::fs::remove_file(&temp);
2358 }
2359 write_result
2360 }
2361
2362 pub fn completed_proposal_retry_owner(
2363 &self,
2364 requested_policy_session_id: Option<&str>,
2365 original_submission: &Value,
2366 ) -> std::io::Result<Option<(String, String)>> {
2367 Ok(self
2368 .read_completed_proposal_owner(requested_policy_session_id, original_submission)?
2369 .map(|owner| (owner.run_id, owner.client_id)))
2370 }
2371
2372 pub fn cleanup_completed_proposal_guards(
2376 &self,
2377 receipt: &CompletedProposalResponse,
2378 ) -> std::io::Result<()> {
2379 self.validate_completed_response(receipt)?;
2380 let pending = &receipt.finalization;
2381 let expected_marker = receipt.execution_marker()?;
2387 if let Some(existing) = self.execution_marker(&pending.run_id)? {
2388 if existing != expected_marker {
2389 return Err(std::io::Error::new(
2390 std::io::ErrorKind::InvalidData,
2391 "completed response does not authorize this execution marker cleanup",
2392 ));
2393 }
2394 if self.failures.take(RunStoreFailurePoint::MarkerUnlink) {
2395 return Err(std::io::Error::other(
2396 "injected execution marker unlink failure",
2397 ));
2398 }
2399 std::fs::remove_file(self.proposal_execution_path(&pending.run_id))?;
2400 self.sync_cleanup_directory(&self.proposal_execution_root())?;
2401 }
2402
2403 let pending_path = self.proposal_outbox_path(&pending.run_id);
2404 match car_secrets::open_private_read(&pending_path) {
2405 Ok(file) => {
2406 let existing: PendingProposalFinalization = serde_json::from_reader(file)
2407 .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
2408 existing.validate()?;
2409 if existing != *pending {
2410 return Err(std::io::Error::new(
2411 std::io::ErrorKind::InvalidData,
2412 "completed response does not authorize this pending finalization cleanup",
2413 ));
2414 }
2415 if self.failures.take(RunStoreFailurePoint::PendingUnlink) {
2416 return Err(std::io::Error::other(
2417 "injected pending proposal unlink failure",
2418 ));
2419 }
2420 std::fs::remove_file(&pending_path)?;
2421 self.sync_cleanup_directory(&self.proposal_outbox_root())?;
2422 }
2423 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
2424 Err(error) => return Err(error),
2425 }
2426 Ok(())
2427 }
2428
2429 fn sync_cleanup_directory(&self, root: &Path) -> std::io::Result<()> {
2430 if self.failures.take(RunStoreFailurePoint::DirectoryFsync) {
2431 return Err(std::io::Error::other(
2432 "injected proposal cleanup directory fsync failure",
2433 ));
2434 }
2435 sync_directory(root)
2436 }
2437
2438 pub fn completed_proposal_response_value(
2439 &self,
2440 receipt: &CompletedProposalResponse,
2441 ) -> std::io::Result<Value> {
2442 self.validate_completed_response(receipt)?;
2443 if self
2444 .failures
2445 .take(RunStoreFailurePoint::ResponseSerialization)
2446 {
2447 return Err(std::io::Error::other(
2448 "injected completed proposal response serialization failure",
2449 ));
2450 }
2451 let result = receipt.proposal_result();
2452 let mut value = serde_json::to_value(result)
2453 .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
2454
2455 let wire_results = value
2463 .get_mut("results")
2464 .and_then(Value::as_array_mut)
2465 .ok_or_else(|| {
2466 std::io::Error::new(
2467 std::io::ErrorKind::InvalidData,
2468 "completed proposal response is missing its results array",
2469 )
2470 })?;
2471 if wire_results.len() != result.results.len() {
2472 return Err(std::io::Error::new(
2473 std::io::ErrorKind::InvalidData,
2474 "completed proposal response result count changed during serialization",
2475 ));
2476 }
2477 for (wire, typed) in wire_results.iter_mut().zip(&result.results) {
2478 if typed.rolled_back {
2479 wire.as_object_mut()
2480 .ok_or_else(|| {
2481 std::io::Error::new(
2482 std::io::ErrorKind::InvalidData,
2483 "completed proposal response contains a non-object action result",
2484 )
2485 })?
2486 .insert("rolled_back".to_string(), Value::Bool(true));
2487 }
2488 }
2489 Ok(value)
2490 }
2491
2492 pub fn prepare_storage(&self) -> std::io::Result<()> {
2496 self.ensure_root()
2497 }
2498
2499 fn run_path(&self, agent_id: &str, run_id: &str) -> PathBuf {
2501 self.root
2502 .join(sanitize(agent_id))
2503 .join(format!("{}.jsonl", sanitize(run_id)))
2504 }
2505
2506 fn run_summary_index_path(agent_path: &Path) -> PathBuf {
2507 agent_path.join(RUN_SUMMARY_INDEX_FILE)
2508 }
2509
2510 fn run_summary_sidecar_root(agent_path: &Path) -> PathBuf {
2511 agent_path.join(RUN_SUMMARY_SIDECAR_DIR)
2512 }
2513
2514 fn run_summary_key(run_id: &str) -> [u8; RUN_SUMMARY_KEY_BYTES] {
2515 Sha256::digest(run_id.as_bytes()).into()
2516 }
2517
2518 fn run_summary_sidecar_path(agent_path: &Path, key: &[u8; 32]) -> PathBuf {
2519 let name = key
2520 .iter()
2521 .map(|byte| format!("{byte:02x}"))
2522 .collect::<String>();
2523 Self::run_summary_sidecar_root(agent_path).join(format!("{name}.json"))
2524 }
2525
2526 fn run_trace_corruption_sidecar_root(agent_path: &Path) -> PathBuf {
2527 agent_path.join(RUN_TRACE_CORRUPTION_SIDECAR_DIR)
2528 }
2529
2530 fn run_trace_corruption_sidecar_path(agent_path: &Path, key: &[u8; 32]) -> PathBuf {
2531 let name = key
2532 .iter()
2533 .map(|byte| format!("{byte:02x}"))
2534 .collect::<String>();
2535 Self::run_trace_corruption_sidecar_root(agent_path).join(format!("{name}.json"))
2536 }
2537
2538 fn read_run_trace_corruption_marker(
2539 &self,
2540 agent_path: &Path,
2541 agent_id: &str,
2542 run_id: &str,
2543 ) -> std::io::Result<Option<RunTraceCorruption>> {
2544 let key = Self::run_summary_key(run_id);
2545 let path = Self::run_trace_corruption_sidecar_path(agent_path, &key);
2546 let file = match car_secrets::open_private_read(&path) {
2547 Ok(file) => file,
2548 Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
2549 Err(error) => return Err(error),
2550 };
2551 let marker: RunTraceCorruptionMarker = serde_json::from_reader(file)
2552 .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
2553 if marker.run_id != run_id || marker.agent_id != agent_id {
2554 return Err(std::io::Error::new(
2555 std::io::ErrorKind::InvalidData,
2556 "run trace corruption marker identity does not match its path",
2557 ));
2558 }
2559 Ok(Some(marker.corruption))
2560 }
2561
2562 fn write_run_trace_corruption_marker(
2563 &self,
2564 agent_path: &Path,
2565 agent_id: &str,
2566 run_id: &str,
2567 corruption: &RunTraceCorruption,
2568 ) -> std::io::Result<()> {
2569 if let Some(existing) =
2570 self.read_run_trace_corruption_marker(agent_path, agent_id, run_id)?
2571 {
2572 if existing == *corruption {
2573 return Ok(());
2574 }
2575 return Err(std::io::Error::new(
2576 std::io::ErrorKind::InvalidData,
2577 "run trace corruption marker conflicts with an existing marker",
2578 ));
2579 }
2580 if self
2581 .failures
2582 .take(RunStoreFailurePoint::CorruptionMarkerWrite)
2583 {
2584 return Err(injected_storage_full());
2585 }
2586 let root = Self::run_trace_corruption_sidecar_root(agent_path);
2587 self.ensure_private_dir(&root)?;
2588 let path =
2589 Self::run_trace_corruption_sidecar_path(agent_path, &Self::run_summary_key(run_id));
2590 self.write_private_json_atomic(
2591 &root,
2592 &path,
2593 &RunTraceCorruptionMarker {
2594 run_id: run_id.to_string(),
2595 agent_id: agent_id.to_string(),
2596 corruption: corruption.clone(),
2597 },
2598 )
2599 }
2600
2601 fn persist_run_trace_corruption_marker(
2602 &self,
2603 agent_path: &Path,
2604 agent_id: &str,
2605 run_id: &str,
2606 corruption: &RunTraceCorruption,
2607 ) -> std::io::Result<()> {
2608 let error = match self
2609 .write_run_trace_corruption_marker(agent_path, agent_id, run_id, corruption)
2610 {
2611 Ok(()) => return Ok(()),
2612 Err(error) => error,
2613 };
2614 if self
2615 .read_run_trace_corruption_marker(agent_path, agent_id, run_id)
2616 .ok()
2617 .flatten()
2618 .is_some()
2619 {
2620 return Err(error);
2621 }
2622
2623 let sidecar_root = Self::run_summary_sidecar_root(agent_path);
2627 let sidecar = Self::run_summary_sidecar_path(agent_path, &Self::run_summary_key(run_id));
2628 if self
2629 .failures
2630 .take(RunStoreFailurePoint::CorruptionSummaryInvalidate)
2631 {
2632 return Err(std::io::Error::new(
2633 error.kind(),
2634 format!("{error}; stale run summary invalidation also failed: permission denied"),
2635 ));
2636 }
2637 match std::fs::remove_file(&sidecar) {
2638 Ok(()) => sync_directory(&sidecar_root)?,
2639 Err(remove_error) if remove_error.kind() == std::io::ErrorKind::NotFound => {}
2640 Err(remove_error) => {
2641 return Err(std::io::Error::new(
2642 error.kind(),
2643 format!("{error}; stale run summary invalidation also failed: {remove_error}"),
2644 ));
2645 }
2646 }
2647 Err(error)
2648 }
2649
2650 fn remember_run_trace_corruption(
2651 &self,
2652 agent_id: &str,
2653 run_id: &str,
2654 corruption: &RunTraceCorruption,
2655 ) {
2656 self.trace_corruptions
2657 .lock()
2658 .expect("run trace corruption registry poisoned")
2659 .insert(
2660 (agent_id.to_string(), run_id.to_string()),
2661 corruption.clone(),
2662 );
2663 }
2664
2665 fn known_run_trace_corruption(
2666 &self,
2667 agent_id: &str,
2668 run_id: &str,
2669 ) -> Option<RunTraceCorruption> {
2670 self.trace_corruptions
2671 .lock()
2672 .expect("run trace corruption registry poisoned")
2673 .get(&(agent_id.to_string(), run_id.to_string()))
2674 .cloned()
2675 }
2676
2677 fn publish_strict_read_corruption(
2682 &self,
2683 agent_path: &Path,
2684 agent_id: &str,
2685 run_id: &str,
2686 error: std::io::Error,
2687 ) -> std::io::Error {
2688 let Some(corruption) = trace_corruption_from_error(&error) else {
2689 return error;
2690 };
2691 self.remember_run_trace_corruption(agent_id, run_id, &corruption);
2692 match self.persist_run_trace_corruption_marker(agent_path, agent_id, run_id, &corruption) {
2693 Ok(()) => error,
2694 Err(source) => summary_refresh_error(source, corruption),
2695 }
2696 }
2697
2698 fn apply_run_trace_corruption_marker(
2699 &self,
2700 agent_path: &Path,
2701 summary: &mut RunSummary,
2702 ) -> std::io::Result<()> {
2703 let corruption = match self.known_run_trace_corruption(&summary.agent_id, &summary.run_id) {
2704 Some(corruption) => Some(corruption),
2705 None => self.read_run_trace_corruption_marker(
2706 agent_path,
2707 &summary.agent_id,
2708 &summary.run_id,
2709 )?,
2710 };
2711 if let Some(corruption) = corruption {
2712 summary.status = RunStatus::Incomplete;
2713 summary.trace_corruption = Some(corruption);
2714 }
2715 Ok(())
2716 }
2717
2718 fn read_summary_sidecar(
2719 &self,
2720 agent_path: &Path,
2721 key: &[u8; 32],
2722 ) -> std::io::Result<RunSummary> {
2723 let path = Self::run_summary_sidecar_path(agent_path, key);
2724 let file = car_secrets::open_private_read(&path)?;
2725 let summary: RunSummary = serde_json::from_reader(file)
2726 .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
2727 if Self::run_summary_key(&summary.run_id) != *key {
2728 return Err(std::io::Error::new(
2729 std::io::ErrorKind::InvalidData,
2730 "run summary sidecar identity does not match its index key",
2731 ));
2732 }
2733 Ok(summary)
2734 }
2735
2736 fn write_summary_sidecar(
2737 &self,
2738 agent_path: &Path,
2739 summary: &RunSummary,
2740 ) -> std::io::Result<()> {
2741 if let Some(gate) = &self.summary_write_gate {
2742 gate.wait_if_armed();
2743 }
2744 if self.failures.take(RunStoreFailurePoint::SummaryWrite) {
2745 return Err(injected_storage_full());
2746 }
2747 let root = Self::run_summary_sidecar_root(agent_path);
2748 self.ensure_private_dir(&root)?;
2749 let path =
2750 Self::run_summary_sidecar_path(agent_path, &Self::run_summary_key(&summary.run_id));
2751 self.write_private_json_atomic(&root, &path, summary)
2752 }
2753
2754 fn write_summary_index(
2755 &self,
2756 agent_path: &Path,
2757 summaries: &[RunSummary],
2758 ) -> std::io::Result<()> {
2759 let mut ordered = summaries.to_vec();
2760 let mut next_sequence = ordered.iter().map(|row| row.sequence).max().unwrap_or(0);
2761 let mut missing = ordered
2762 .iter_mut()
2763 .filter(|row| row.sequence == 0)
2764 .collect::<Vec<_>>();
2765 missing.sort_by(|left, right| {
2766 left.started_at
2767 .cmp(&right.started_at)
2768 .then_with(|| right.run_id.cmp(&left.run_id))
2769 });
2770 for summary in missing {
2771 next_sequence = next_sequence.checked_add(1).ok_or_else(|| {
2772 std::io::Error::new(
2773 std::io::ErrorKind::InvalidData,
2774 "run summary sequence exhausted",
2775 )
2776 })?;
2777 summary.sequence = next_sequence;
2778 }
2779 ordered.sort_by(|left, right| right.sequence.cmp(&left.sequence));
2780 if ordered
2781 .windows(2)
2782 .any(|pair| pair[0].sequence == pair[1].sequence)
2783 {
2784 return Err(std::io::Error::new(
2785 std::io::ErrorKind::InvalidData,
2786 "run summary index contains duplicate sequences",
2787 ));
2788 }
2789 let sidecar_root = Self::run_summary_sidecar_root(agent_path);
2790 self.ensure_private_dir(&sidecar_root)?;
2791 let mut retained = std::collections::HashSet::new();
2792 let mut bytes = Vec::with_capacity(ordered.len() * RUN_SUMMARY_INDEX_RECORD_BYTES);
2793 for summary in &ordered {
2794 let key = Self::run_summary_key(&summary.run_id);
2795 retained.insert(key);
2796 self.write_summary_sidecar(agent_path, summary)?;
2797 bytes.extend_from_slice(&summary.sequence.to_be_bytes());
2798 bytes.extend_from_slice(&key);
2799 }
2800 if let Ok(entries) = std::fs::read_dir(&sidecar_root) {
2801 for entry in entries.flatten() {
2802 let path = entry.path();
2803 let Some(stem) = path.file_stem().and_then(|value| value.to_str()) else {
2804 continue;
2805 };
2806 let is_retained = retained.iter().any(|key| {
2807 key.iter()
2808 .map(|byte| format!("{byte:02x}"))
2809 .collect::<String>()
2810 == stem
2811 });
2812 if !is_retained && path.extension().and_then(|value| value.to_str()) == Some("json")
2813 {
2814 let _ = std::fs::remove_file(path);
2815 }
2816 }
2817 }
2818 let path = Self::run_summary_index_path(agent_path);
2819 let temp = agent_path.join(format!(
2820 ".run-summary-index-v2.{}.tmp",
2821 uuid::Uuid::new_v4().simple()
2822 ));
2823 let result = (|| {
2824 let mut file = self.create_private_file(&temp)?;
2825 file.write_all(&bytes)?;
2826 file.flush()?;
2827 file.sync_all()?;
2828 car_secrets::revalidate_private_path(&temp, &file)?;
2829 drop(file);
2830 car_secrets::atomic_replace_private_file(&temp, &path)?;
2831 sync_directory(agent_path)
2832 })();
2833 if result.is_err() {
2834 let _ = std::fs::remove_file(temp);
2835 }
2836 result
2837 }
2838
2839 fn refresh_run_summary(
2840 &self,
2841 agent_id: &str,
2842 run_id: &str,
2843 add_to_order: bool,
2844 ) -> std::io::Result<()> {
2845 let agent_path = self.open_agent_dir_for_read(agent_id)?;
2846 let index_path = Self::run_summary_index_path(&agent_path);
2852 let index_lock = self.append_lock(&index_path);
2853 let _guard = index_lock
2854 .lock()
2855 .map_err(|_| std::io::Error::other("run summary index lock poisoned"))?;
2856 let path = self.run_path(agent_id, run_id);
2857 let scan = match summarize_file_checked(&path) {
2858 Ok(scan) => scan,
2859 Err(error) => {
2860 let Some(corruption) = trace_corruption_from_error(&error) else {
2861 return Err(error);
2862 };
2863 self.remember_run_trace_corruption(agent_id, run_id, &corruption);
2864 self.persist_run_trace_corruption_marker(
2865 &agent_path,
2866 agent_id,
2867 run_id,
2868 &corruption,
2869 )
2870 .map_err(|source| summary_refresh_error(source, corruption))?;
2871 return Err(error);
2872 }
2873 };
2874 if let Some(corruption) = &scan.corruption {
2875 self.remember_run_trace_corruption(agent_id, run_id, corruption);
2876 self.persist_run_trace_corruption_marker(&agent_path, agent_id, run_id, corruption)
2877 .map_err(|source| summary_refresh_error(source, corruption.clone()))?;
2878 }
2879 let mut summary = scan.summary.ok_or_else(|| {
2880 let error = std::io::Error::new(
2881 std::io::ErrorKind::InvalidData,
2882 "durable trace cannot be summarized for the run index",
2883 );
2884 match scan.corruption.clone() {
2885 Some(corruption) => summary_refresh_error(error, corruption),
2886 None => error,
2887 }
2888 })?;
2889 self.apply_run_trace_corruption_marker(&agent_path, &mut summary)?;
2890 let corruption = summary.trace_corruption.clone();
2891 let update = (|| {
2892 if !add_to_order {
2893 let key = Self::run_summary_key(run_id);
2894 summary.sequence = self.read_summary_sidecar(&agent_path, &key)?.sequence;
2895 self.write_summary_sidecar(&agent_path, &summary)?;
2896 return Ok(());
2897 }
2898 let mut summaries = Vec::new();
2899 if let Ok(mut index) = car_secrets::open_private_read(&index_path) {
2900 let len = index.metadata()?.len() as usize;
2901 if !len.is_multiple_of(RUN_SUMMARY_INDEX_RECORD_BYTES) {
2902 return Err(std::io::Error::new(
2903 std::io::ErrorKind::InvalidData,
2904 "run summary index has a partial record",
2905 ));
2906 }
2907 let mut record = [0u8; RUN_SUMMARY_INDEX_RECORD_BYTES];
2908 while index.read_exact(&mut record).is_ok() {
2909 let sequence = u64::from_be_bytes(record[..8].try_into().expect("eight bytes"));
2910 let key: [u8; RUN_SUMMARY_KEY_BYTES] =
2911 record[8..].try_into().expect("summary key bytes");
2912 let existing = self.read_summary_sidecar(&agent_path, &key)?;
2913 if existing.sequence != sequence {
2914 return Err(std::io::Error::new(
2915 std::io::ErrorKind::InvalidData,
2916 "run summary sidecar sequence does not match its index record",
2917 ));
2918 }
2919 if existing.run_id != run_id {
2920 summaries.push(existing);
2921 } else {
2922 summary.sequence = sequence;
2923 }
2924 }
2925 }
2926 summaries.push(summary.clone());
2927 self.write_summary_index(&agent_path, &summaries)
2928 })();
2929 preserve_summary_corruption(update, &corruption)?;
2930 reject_corrupt_summary(&summary)
2931 }
2932
2933 fn ensure_root(&self) -> std::io::Result<()> {
2938 self.ensure_private_dir(&self.root)?;
2939 ensure_backup_excluded(&self.root)?;
2940 Ok(())
2941 }
2942
2943 fn open_root_for_read(&self) -> std::io::Result<()> {
2946 std::fs::symlink_metadata(&self.root)?;
2947 self.ensure_root()
2948 }
2949
2950 fn ensure_agent_dir(&self, agent_id: &str) -> std::io::Result<PathBuf> {
2952 self.ensure_root()?;
2953 let dir = self.root.join(sanitize(agent_id));
2954 self.ensure_private_dir(&dir)?;
2955 Ok(dir)
2956 }
2957
2958 fn open_agent_dir_for_read(&self, agent_id: &str) -> std::io::Result<PathBuf> {
2961 self.open_root_for_read()?;
2962 let dir = self.root.join(sanitize(agent_id));
2963 std::fs::symlink_metadata(&dir)?;
2964 self.ensure_private_dir(&dir)?;
2965 Ok(dir)
2966 }
2967
2968 pub fn append_records(
2976 &self,
2977 agent_id: &str,
2978 run_id: &str,
2979 records: &[RunRecord],
2980 ) -> std::io::Result<()> {
2981 if records.is_empty() {
2982 return Ok(());
2983 }
2984 self.ensure_agent_dir(agent_id)?;
2985 let path = self.run_path(agent_id, run_id);
2986 let append_lock = self.append_lock(&path);
2987 let _guard = append_lock
2988 .lock()
2989 .map_err(|_| std::io::Error::other("run append lock poisoned"))?;
2990 let existed_before_open = path.exists();
2991 let mut file = self.open_private_append(&path)?;
2992 if self.failures.take(RunStoreFailurePoint::Write) {
2993 return Err(injected_storage_full());
2994 }
2995 append_jsonl_batch_to_path(&path, &mut file, records)?;
2996 self.durable_receipt(&path, &mut file, existed_before_open)
2997 }
2998
2999 pub fn write_started(&self, started: &car_proto::RunStarted) -> std::io::Result<()> {
3001 if started.run_id.is_empty()
3002 || started.agent_id.is_empty()
3003 || started.client_id.as_deref().is_none_or(str::is_empty)
3004 {
3005 return Err(std::io::Error::new(
3006 std::io::ErrorKind::InvalidInput,
3007 "active RunStarted requires client_id",
3008 ));
3009 }
3010 self.ensure_boundary_record(
3011 &started.agent_id,
3012 &started.run_id,
3013 RunRecord::Started(started.clone()),
3014 )?;
3015 self.refresh_run_summary(&started.agent_id, &started.run_id, true)
3016 }
3017
3018 pub fn append_turns(
3020 &self,
3021 agent_id: &str,
3022 run_id: &str,
3023 turns: &[RunRecord],
3024 ) -> std::io::Result<()> {
3025 if let Some(gate) = &self.append_gate {
3026 gate.wait_if_armed();
3027 }
3028 self.append_records(agent_id, run_id, turns)?;
3029 if let Err(error) = self.refresh_run_summary(agent_id, run_id, false) {
3030 if is_trace_corruption_error(&error) {
3031 return Err(error);
3032 }
3033 tracing::error!(%agent_id, %run_id, %error, "run summary index update deferred until startup repair");
3034 }
3035 Ok(())
3036 }
3037
3038 pub fn run_trace_corruption_for(
3042 &self,
3043 agent_id: &str,
3044 run_id: &str,
3045 ) -> std::io::Result<Option<RunTraceCorruption>> {
3046 if let Some(gate) = &self.summary_read_gate {
3047 gate.wait_if_armed();
3048 }
3049 if let Some(corruption) = self.known_run_trace_corruption(agent_id, run_id) {
3050 return Ok(Some(corruption));
3051 }
3052 let agent_path = match self.open_agent_dir_for_read(agent_id) {
3053 Ok(path) => path,
3054 Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
3055 Err(error) => return Err(error),
3056 };
3057 if let Some(corruption) =
3058 self.read_run_trace_corruption_marker(&agent_path, agent_id, run_id)?
3059 {
3060 return Ok(Some(corruption));
3061 }
3062 let key = Self::run_summary_key(run_id);
3063 match self.read_summary_sidecar(&agent_path, &key) {
3064 Ok(summary) => {
3065 if summary.run_id != run_id || summary.agent_id != agent_id {
3066 return Err(std::io::Error::new(
3067 std::io::ErrorKind::InvalidData,
3068 "run summary sidecar owner does not match requested run",
3069 ));
3070 }
3071 Ok(summary.trace_corruption)
3072 }
3073 Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
3074 Err(error) => Err(error),
3075 }
3076 }
3077
3078 pub fn ensure_proposal_turns(
3083 &self,
3084 pending: &PendingProposalFinalization,
3085 ) -> std::io::Result<ProposalTraceEnsure> {
3086 if let Some(gate) = &self.append_gate {
3087 gate.wait_if_armed();
3088 }
3089 pending.validate()?;
3090 let started = self.durable_run_started(&pending.run_id)?;
3091 match self.execution_marker(&pending.run_id)? {
3092 Some(marker) => pending.validate_provenance(&started, &marker)?,
3093 None => {
3094 let completed = self.completed_proposal(
3095 &pending.run_id,
3096 &pending.client_id,
3097 pending.requested_policy_session_id.as_deref(),
3098 &pending.original_submission,
3099 )?;
3100 if completed.as_ref().map(|receipt| &receipt.finalization) != Some(pending) {
3101 return Err(std::io::Error::new(
3102 std::io::ErrorKind::InvalidData,
3103 "proposal trace has neither its execution marker nor an exact completed receipt",
3104 ));
3105 }
3106 }
3107 }
3108 let mut expected = crate::run_trace::record_turns(
3109 &pending.final_proposal,
3110 &pending.proposal_result.results,
3111 0,
3112 );
3113 for record in &mut expected {
3114 let RunRecord::Turn(turn) = record else {
3115 return Err(std::io::Error::new(
3116 std::io::ErrorKind::InvalidData,
3117 "proposal trace generator returned a non-turn record",
3118 ));
3119 };
3120 if !crate::handler::enforce_proposal_turn_byte_cap(turn) {
3121 return Err(std::io::Error::new(
3122 std::io::ErrorKind::InvalidData,
3123 "proposal turn exceeds the durable byte limit without changing authenticated parameters",
3124 ));
3125 }
3126 }
3127 if expected.is_empty() {
3128 return Ok(ProposalTraceEnsure {
3129 records: expected,
3130 appended: false,
3131 });
3132 }
3133
3134 let path = self.run_path(&started.agent_id, &pending.run_id);
3135 let append_lock = self.append_lock(&path);
3136 let _guard = append_lock
3137 .lock()
3138 .map_err(|_| std::io::Error::other("proposal trace append lock poisoned"))?;
3139 let file = car_secrets::open_private_read(&path)?;
3140 let mut turn_count = 0usize;
3141 let mut matching = Vec::new();
3142 let scan = scan_records(&file, |record| {
3143 if let RunRecord::Turn(turn) = record {
3144 turn_count = turn_count.saturating_add(1);
3145 if turn.proposal_id.as_deref() == Some(pending.final_proposal_id.as_str()) {
3146 matching.push(RunRecord::Turn(turn));
3147 }
3148 }
3149 });
3150 if let Err(error) = scan {
3151 let error = scan_error_to_io(error);
3152 let _ = self.refresh_run_summary(&started.agent_id, &pending.run_id, false);
3153 return Err(error);
3154 }
3155 car_secrets::revalidate_private_path(&path, &file)?;
3156
3157 let to_append = if !matching.is_empty() {
3158 let start = matching
3159 .first()
3160 .and_then(|record| match record {
3161 RunRecord::Turn(turn) => Some(turn.index),
3162 _ => None,
3163 })
3164 .unwrap_or(0);
3165 for (offset, record) in expected.iter_mut().enumerate() {
3166 if let RunRecord::Turn(turn) = record {
3167 turn.index = start + offset;
3168 }
3169 }
3170 let exact_prefix = matching.len() <= expected.len()
3171 && matching.iter().zip(&expected).all(|(left, right)| {
3172 serde_json::to_value(left).ok() == serde_json::to_value(right).ok()
3173 });
3174 if !exact_prefix || turn_count != start.saturating_add(matching.len()) {
3175 return Err(std::io::Error::new(
3176 std::io::ErrorKind::InvalidData,
3177 "durable proposal trace conflicts with final proposal identity or has later rows",
3178 ));
3179 }
3180 if matching.len() == expected.len() {
3181 return Ok(ProposalTraceEnsure {
3182 records: matching,
3183 appended: false,
3184 });
3185 }
3186 expected[matching.len()..].to_vec()
3187 } else {
3188 for (offset, record) in expected.iter_mut().enumerate() {
3189 if let RunRecord::Turn(turn) = record {
3190 turn.index = turn_count + offset;
3191 }
3192 }
3193 expected.clone()
3194 };
3195
3196 if turn_count.saturating_add(to_append.len()) > crate::session::RECORD_TURNS_RUN_CEILING {
3197 return Err(std::io::Error::new(
3198 std::io::ErrorKind::InvalidData,
3199 "proposal trace would exceed the durable run turn ceiling",
3200 ));
3201 }
3202
3203 let existed_before_open = path.exists();
3204 let mut append = self.open_private_append(&path)?;
3205 if self.failures.take(RunStoreFailurePoint::Write) {
3206 return Err(injected_storage_full());
3207 }
3208 append_jsonl_batch_to_path(&path, &mut append, &to_append)?;
3209 self.durable_receipt(&path, &mut append, existed_before_open)?;
3210 if let Err(error) = self.refresh_run_summary(&started.agent_id, &pending.run_id, false) {
3211 if is_trace_corruption_error(&error) {
3212 return Err(error);
3213 }
3214 tracing::error!(run_id = %pending.run_id, %error, "proposal trace summary update deferred until startup repair");
3215 }
3216 Ok(ProposalTraceEnsure {
3217 records: expected,
3218 appended: true,
3219 })
3220 }
3221
3222 pub fn write_ended(&self, ended: &car_proto::RunEnded) -> std::io::Result<()> {
3225 if ended.run_id.is_empty()
3226 || ended.agent_id.is_empty()
3227 || ended.client_id.as_deref().is_none_or(str::is_empty)
3228 || ended.completion_digest.as_deref().is_none_or(str::is_empty)
3229 {
3230 return Err(std::io::Error::new(
3231 std::io::ErrorKind::InvalidInput,
3232 "active RunEnded requires client_id and completion_digest",
3233 ));
3234 }
3235 let expected_digest = crate::session::run_completion_digest(&ended.termination)
3236 .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
3237 if ended.completion_digest.as_deref() != Some(expected_digest.as_str()) {
3238 return Err(std::io::Error::new(
3239 std::io::ErrorKind::InvalidData,
3240 "RunEnded completion_digest does not match JCS RunTermination",
3241 ));
3242 }
3243 self.ensure_boundary_record(
3244 &ended.agent_id,
3245 &ended.run_id,
3246 RunRecord::Ended(ended.clone()),
3247 )?;
3248 self.refresh_run_summary(&ended.agent_id, &ended.run_id, false)
3249 }
3250
3251 pub fn write_cancellation_requested(
3255 &self,
3256 agent_id: &str,
3257 requested: &car_proto::RunCancellationRequested,
3258 ) -> std::io::Result<()> {
3259 self.ensure_boundary_record(
3260 agent_id,
3261 &requested.run_id,
3262 RunRecord::CancellationRequested(requested.clone()),
3263 )?;
3264 self.refresh_run_summary(agent_id, &requested.run_id, false)
3265 }
3266
3267 pub fn write_cancellation_result(
3270 &self,
3271 agent_id: &str,
3272 result: &car_proto::RunCancelResponse,
3273 ) -> std::io::Result<()> {
3274 self.ensure_boundary_record(
3275 agent_id,
3276 &result.run_id,
3277 RunRecord::CancellationResult(result.clone()),
3278 )?;
3279 self.refresh_run_summary(agent_id, &result.run_id, false)
3280 }
3281
3282 fn append_lock(&self, path: &Path) -> Arc<Mutex<()>> {
3283 let mut locks = self
3284 .append_locks
3285 .lock()
3286 .expect("run append-lock registry poisoned");
3287 if let Some(lock) = locks.get(path).and_then(Weak::upgrade) {
3288 return lock;
3289 }
3290 locks.retain(|_, lock| lock.strong_count() > 0);
3291 let lock = Arc::new(Mutex::new(()));
3292 locks.insert(path.to_path_buf(), Arc::downgrade(&lock));
3293 lock
3294 }
3295
3296 fn ensure_boundary_record(
3297 &self,
3298 agent_id: &str,
3299 run_id: &str,
3300 record: RunRecord,
3301 ) -> std::io::Result<()> {
3302 self.ensure_agent_dir(agent_id)?;
3303 let path = self.run_path(agent_id, run_id);
3304 let append_lock = self.append_lock(&path);
3305 let _guard = append_lock
3306 .lock()
3307 .map_err(|_| std::io::Error::other("run append lock poisoned"))?;
3308 let existed_before_open = path.exists();
3309 let mut file = self.open_private_append(&path)?;
3310 car_secrets::revalidate_private_path(&path, &file)?;
3311 let existing = match load_records(&file) {
3312 Ok(records) => records,
3313 Err(error) => {
3314 let _ = self.refresh_run_summary(agent_id, run_id, false);
3317 return Err(error);
3318 }
3319 };
3320
3321 let exact_exists = match &record {
3322 RunRecord::Started(wanted) => {
3323 if let Some(found) = existing.iter().find_map(|row| match row {
3324 RunRecord::Started(started) => Some(started),
3325 _ => None,
3326 }) {
3327 if found != wanted {
3328 return Err(std::io::Error::new(
3329 std::io::ErrorKind::AlreadyExists,
3330 "run_id is already reserved by a different RunStarted owner/preimage",
3331 ));
3332 }
3333 true
3334 } else {
3335 if !existing.is_empty() {
3336 return Err(std::io::Error::new(
3337 std::io::ErrorKind::InvalidData,
3338 "non-empty run trace is missing its RunStarted boundary",
3339 ));
3340 }
3341 false
3342 }
3343 }
3344 RunRecord::Ended(wanted) => {
3345 let started = existing.iter().find_map(|row| match row {
3346 RunRecord::Started(started) => Some(started),
3347 _ => None,
3348 });
3349 let Some(started) = started else {
3350 return Err(std::io::Error::new(
3351 std::io::ErrorKind::InvalidData,
3352 "RunEnded cannot precede RunStarted",
3353 ));
3354 };
3355 if started.run_id != wanted.run_id
3356 || started.agent_id != wanted.agent_id
3357 || started.client_id != wanted.client_id
3358 {
3359 return Err(std::io::Error::new(
3360 std::io::ErrorKind::PermissionDenied,
3361 "RunEnded owner does not match durable RunStarted",
3362 ));
3363 }
3364 if let Some(found) = existing.iter().find_map(|row| match row {
3365 RunRecord::Ended(ended) => Some(ended),
3366 _ => None,
3367 }) {
3368 if !same_run_ended(found, wanted) {
3369 return Err(std::io::Error::new(
3370 std::io::ErrorKind::AlreadyExists,
3371 "run already has a different durable terminal",
3372 ));
3373 }
3374 true
3375 } else {
3376 false
3377 }
3378 }
3379 RunRecord::Turn(_) => false,
3380 RunRecord::CancellationRequested(wanted) => {
3381 let started = existing.iter().find_map(|row| match row {
3382 RunRecord::Started(started) => Some(started),
3383 _ => None,
3384 });
3385 let Some(started) = started else {
3386 return Err(std::io::Error::new(
3387 std::io::ErrorKind::InvalidData,
3388 "cancellation request cannot precede RunStarted",
3389 ));
3390 };
3391 if started.run_id != wanted.run_id || started.agent_id != agent_id {
3392 return Err(std::io::Error::new(
3393 std::io::ErrorKind::PermissionDenied,
3394 "cancellation request owner does not match durable RunStarted",
3395 ));
3396 }
3397 if existing
3398 .iter()
3399 .any(|row| matches!(row, RunRecord::Ended(_)))
3400 {
3401 return Err(std::io::Error::new(
3402 std::io::ErrorKind::AlreadyExists,
3403 "run is already terminal",
3404 ));
3405 }
3406 if let Some(found) = existing.iter().find_map(|row| match row {
3407 RunRecord::CancellationRequested(row) => Some(row),
3408 _ => None,
3409 }) {
3410 if found != wanted {
3411 return Err(std::io::Error::new(
3412 std::io::ErrorKind::AlreadyExists,
3413 "run already has a different cancellation request",
3414 ));
3415 }
3416 true
3417 } else {
3418 false
3419 }
3420 }
3421 RunRecord::CancellationResult(wanted) => {
3422 if wanted.status != car_proto::RunCancellationStatus::TerminationUnconfirmed
3423 || wanted.terminal_digest.is_some()
3424 {
3425 return Err(std::io::Error::new(
3426 std::io::ErrorKind::InvalidInput,
3427 "only body-free termination_unconfirmed receipts are nonterminal records",
3428 ));
3429 }
3430 let request = existing.iter().find_map(|row| match row {
3431 RunRecord::CancellationRequested(row) => Some(row),
3432 _ => None,
3433 });
3434 let Some(request) = request else {
3435 return Err(std::io::Error::new(
3436 std::io::ErrorKind::InvalidData,
3437 "cancellation result cannot precede its request",
3438 ));
3439 };
3440 if request.run_id != wanted.run_id
3441 || request.idempotency_key != wanted.idempotency_key
3442 || request.reason_digest != wanted.reason_digest
3443 || request.principal != wanted.principal
3444 || request.action_id != wanted.action_id
3445 || request.request_id != wanted.request_id
3446 {
3447 return Err(std::io::Error::new(
3448 std::io::ErrorKind::InvalidData,
3449 "cancellation result conflicts with its durable request",
3450 ));
3451 }
3452 if existing
3453 .iter()
3454 .any(|row| matches!(row, RunRecord::Ended(_)))
3455 {
3456 return Err(std::io::Error::new(
3457 std::io::ErrorKind::AlreadyExists,
3458 "run is already terminal",
3459 ));
3460 }
3461 if let Some(found) = existing.iter().find_map(|row| match row {
3462 RunRecord::CancellationResult(row) => Some(row),
3463 _ => None,
3464 }) {
3465 if found != wanted {
3466 return Err(std::io::Error::new(
3467 std::io::ErrorKind::AlreadyExists,
3468 "run already has a different cancellation result",
3469 ));
3470 }
3471 true
3472 } else {
3473 false
3474 }
3475 }
3476 };
3477
3478 if !exact_exists {
3479 if self.failures.take(RunStoreFailurePoint::Write) {
3480 return Err(injected_storage_full());
3481 }
3482 append_jsonl_batch_to_path(&path, &mut file, &[record])?;
3483 }
3484 self.durable_receipt(&path, &mut file, existed_before_open)
3485 }
3486
3487 fn durable_receipt(
3488 &self,
3489 path: &Path,
3490 file: &mut File,
3491 existed_before_open: bool,
3492 ) -> std::io::Result<()> {
3493 if self.failures.take(RunStoreFailurePoint::Flush) {
3494 return Err(std::io::Error::other("injected run-store flush failure"));
3495 }
3496 file.flush()?;
3497 if self.failures.take(RunStoreFailurePoint::Fsync) {
3498 return Err(std::io::Error::other("injected run-store fsync failure"));
3499 }
3500 file.sync_all()?;
3501 if !existed_before_open {
3502 if let Some(parent) = path.parent() {
3503 sync_directory(parent)?;
3504 }
3505 }
3506 car_secrets::revalidate_private_path(path, file)
3507 }
3508
3509 pub fn get_run_trace(&self, run_id: &str) -> Option<Vec<RunRecord>> {
3516 self.get_run_trace_checked(run_id).ok().flatten()
3517 }
3518
3519 pub fn get_run_trace_checked(&self, run_id: &str) -> std::io::Result<Option<Vec<RunRecord>>> {
3523 let Some((path, file)) = self.resolve_run_file(run_id) else {
3524 return Ok(None);
3525 };
3526 let Some(agent_path) = path.parent() else {
3527 return Err(std::io::Error::new(
3528 std::io::ErrorKind::InvalidData,
3529 "run trace path has no owning agent directory",
3530 ));
3531 };
3532 let Some(agent_id) = agent_path.file_name().and_then(|value| value.to_str()) else {
3533 return Err(std::io::Error::new(
3534 std::io::ErrorKind::InvalidData,
3535 "run trace path has no valid owning agent id",
3536 ));
3537 };
3538 load_private_records(&path, &file)
3539 .map(Some)
3540 .map_err(|error| {
3541 self.publish_strict_read_corruption(agent_path, agent_id, run_id, error)
3542 })
3543 }
3544
3545 pub fn get_run_trace_for(&self, agent_id: &str, run_id: &str) -> Option<Vec<RunRecord>> {
3549 self.get_run_trace_for_checked(agent_id, run_id)
3550 .ok()
3551 .flatten()
3552 }
3553
3554 pub fn get_run_trace_for_checked(
3555 &self,
3556 agent_id: &str,
3557 run_id: &str,
3558 ) -> std::io::Result<Option<Vec<RunRecord>>> {
3559 let agent_path = match self.open_agent_dir_for_read(agent_id) {
3560 Ok(path) => path,
3561 Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
3562 Err(error) => return Err(error),
3563 };
3564 let path = self.run_path(agent_id, run_id);
3565 let file = match car_secrets::open_private_read(&path) {
3566 Ok(file) => file,
3567 Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
3568 Err(error) => return Err(error),
3569 };
3570 load_private_records(&path, &file)
3571 .map(Some)
3572 .map_err(|error| {
3573 self.publish_strict_read_corruption(&agent_path, agent_id, run_id, error)
3574 })
3575 }
3576
3577 pub fn get_run_trace_page_for(
3581 &self,
3582 agent_id: &str,
3583 run_id: &str,
3584 cursor: usize,
3585 limit: usize,
3586 ) -> std::io::Result<Option<(Vec<RunRecord>, Option<usize>)>> {
3587 let agent_path = match self.open_agent_dir_for_read(agent_id) {
3588 Ok(path) => path,
3589 Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
3590 Err(error) => return Err(error),
3591 };
3592 let path = self.run_path(agent_id, run_id);
3593 let file = match car_secrets::open_private_read(&path) {
3594 Ok(file) => file,
3595 Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
3596 Err(error) => return Err(error),
3597 };
3598 let mut records = Vec::with_capacity(limit.saturating_add(1));
3599 let mut valid_index = 0usize;
3600 if let Err(error) = scan_records(&file, |record| {
3601 if valid_index >= cursor && records.len() <= limit {
3602 records.push(record);
3603 }
3604 valid_index = valid_index.saturating_add(1);
3605 }) {
3606 let error = scan_error_to_io(error);
3607 return Err(self.publish_strict_read_corruption(&agent_path, agent_id, run_id, error));
3608 }
3609 car_secrets::revalidate_private_path(&path, &file)?;
3610 let has_more = records.len() > limit;
3611 records.truncate(limit);
3612 let next_cursor = has_more.then_some(cursor.saturating_add(records.len()));
3613 Ok(Some((records, next_cursor)))
3614 }
3615
3616 pub fn get_run_turn_page_for(
3620 &self,
3621 agent_id: &str,
3622 run_id: &str,
3623 cursor: usize,
3624 limit: usize,
3625 ) -> std::io::Result<Option<(Vec<RunRecord>, Option<usize>, usize, RunStatus)>> {
3626 let agent_path = match self.open_agent_dir_for_read(agent_id) {
3627 Ok(path) => path,
3628 Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
3629 Err(error) => return Err(error),
3630 };
3631 let key = Self::run_summary_key(run_id);
3632 let mut summary = match self.read_summary_sidecar(&agent_path, &key) {
3633 Ok(summary) if summary.run_id == run_id && summary.agent_id == agent_id => summary,
3634 Ok(_) => {
3635 return Err(std::io::Error::new(
3636 std::io::ErrorKind::InvalidData,
3637 "run summary identity does not match the requested trace",
3638 ))
3639 }
3640 Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
3641 Err(error) => return Err(error),
3642 };
3643 self.apply_run_trace_corruption_marker(&agent_path, &mut summary)?;
3644 reject_corrupt_summary(&summary)?;
3645 if cursor > summary.turn_count {
3646 return Err(std::io::Error::new(
3647 std::io::ErrorKind::InvalidInput,
3648 format!(
3649 "runs.subscribe cursor {cursor} exceeds live_cursor {}",
3650 summary.turn_count
3651 ),
3652 ));
3653 }
3654 let path = self.run_path(agent_id, run_id);
3655 let file = car_secrets::open_private_read(&path)?;
3656 let mut turns = Vec::with_capacity(limit.saturating_add(1));
3657 let mut turn_index = 0usize;
3658 if let Err(error) = scan_records(&file, |record| {
3659 if !matches!(record, RunRecord::Turn(_)) {
3660 return;
3661 }
3662 if turn_index >= cursor && turns.len() <= limit {
3663 turns.push(record);
3664 }
3665 turn_index = turn_index.saturating_add(1);
3666 }) {
3667 let error = scan_error_to_io(error);
3668 return Err(self.publish_strict_read_corruption(&agent_path, agent_id, run_id, error));
3669 }
3670 car_secrets::revalidate_private_path(&path, &file)?;
3671 if turn_index < summary.turn_count && turns.len() <= limit {
3672 return Err(std::io::Error::new(
3673 std::io::ErrorKind::InvalidData,
3674 "run trace ended before its durable summary turn count",
3675 ));
3676 }
3677 let has_more = turns.len() > limit;
3678 turns.truncate(limit);
3679 let next_cursor = has_more.then_some(cursor.saturating_add(turns.len()));
3680 Ok(Some((
3681 turns,
3682 next_cursor,
3683 summary.turn_count,
3684 summary.status,
3685 )))
3686 }
3687
3688 pub fn list_runs_page(
3692 &self,
3693 agent_id: &str,
3694 cursor: usize,
3695 limit: usize,
3696 ) -> std::io::Result<(Vec<RunSummary>, Option<usize>)> {
3697 let agent_path = match self.open_agent_dir_for_read(agent_id) {
3698 Ok(path) => path,
3699 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
3700 return Ok((Vec::new(), None))
3701 }
3702 Err(error) => return Err(error),
3703 };
3704 let index_path = Self::run_summary_index_path(&agent_path);
3705 let mut index = car_secrets::open_private_read(&index_path)?;
3706 let byte_len = index.metadata()?.len() as usize;
3707 if !byte_len.is_multiple_of(RUN_SUMMARY_INDEX_RECORD_BYTES) {
3708 return Err(std::io::Error::new(
3709 std::io::ErrorKind::InvalidData,
3710 "run summary index has a partial record",
3711 ));
3712 }
3713 let total = byte_len / RUN_SUMMARY_INDEX_RECORD_BYTES;
3714 if total == 0 {
3715 return Ok((Vec::new(), None));
3716 }
3717
3718 let mut start = 0usize;
3723 if cursor != 0 {
3724 let cursor = u64::try_from(cursor).map_err(|_| {
3725 std::io::Error::new(std::io::ErrorKind::InvalidInput, "cursor overflow")
3726 })?;
3727 let mut low = 0usize;
3728 let mut high = total;
3729 while low < high {
3730 let mid = low + (high - low) / 2;
3731 index.seek(SeekFrom::Start(
3732 (mid * RUN_SUMMARY_INDEX_RECORD_BYTES) as u64,
3733 ))?;
3734 let mut sequence = [0u8; 8];
3735 index.read_exact(&mut sequence)?;
3736 if u64::from_be_bytes(sequence) >= cursor {
3737 low = mid + 1;
3738 } else {
3739 high = mid;
3740 }
3741 }
3742 start = low;
3743 }
3744 if start >= total {
3745 return Ok((Vec::new(), None));
3746 }
3747 index.seek(SeekFrom::Start(
3748 (start * RUN_SUMMARY_INDEX_RECORD_BYTES) as u64,
3749 ))?;
3750 let count = limit.saturating_add(1).min(total - start);
3751 let mut summaries = Vec::with_capacity(count);
3752 for _ in 0..count {
3753 let mut record = [0u8; RUN_SUMMARY_INDEX_RECORD_BYTES];
3754 index.read_exact(&mut record)?;
3755 let sequence = u64::from_be_bytes(record[..8].try_into().expect("eight bytes"));
3756 let key: [u8; RUN_SUMMARY_KEY_BYTES] =
3757 record[8..].try_into().expect("summary key bytes");
3758 let mut summary = self.read_summary_sidecar(&agent_path, &key)?;
3759 if summary.agent_id != agent_id || summary.sequence != sequence {
3760 return Err(std::io::Error::new(
3761 std::io::ErrorKind::InvalidData,
3762 "run summary index does not match its durable sidecar",
3763 ));
3764 }
3765 self.apply_run_trace_corruption_marker(&agent_path, &mut summary)?;
3766 summaries.push(summary);
3767 }
3768 let has_more = summaries.len() > limit;
3769 summaries.truncate(limit);
3770 let next_cursor = if has_more {
3771 summaries
3772 .last()
3773 .map(|summary| {
3774 usize::try_from(summary.sequence).map_err(|_| {
3775 std::io::Error::new(
3776 std::io::ErrorKind::InvalidData,
3777 "run summary sequence cannot be represented by the wire cursor",
3778 )
3779 })
3780 })
3781 .transpose()?
3782 } else {
3783 None
3784 };
3785 Ok((summaries, next_cursor))
3786 }
3787
3788 pub fn list_runs(&self, agent_id: &str) -> Vec<RunSummary> {
3793 let Ok(dir) = self.open_agent_dir_for_read(agent_id) else {
3794 return Vec::new();
3795 };
3796 let mut out = Vec::new();
3797 let Ok(entries) = std::fs::read_dir(&dir) else {
3798 return out;
3799 };
3800 for entry in entries {
3801 let Ok(entry) = entry else {
3802 return Vec::new();
3803 };
3804 let path = entry.path();
3805 if path.extension().and_then(|e| e.to_str()) != Some("jsonl") {
3806 continue;
3807 }
3808 if let Some(mut summary) = summarize_file(&path) {
3809 if self
3810 .apply_run_trace_corruption_marker(&dir, &mut summary)
3811 .is_err()
3812 {
3813 return Vec::new();
3814 }
3815 out.push(summary);
3816 }
3817 }
3818 out.sort_by(|a, b| {
3820 b.started_at
3821 .cmp(&a.started_at)
3822 .then_with(|| a.run_id.cmp(&b.run_id))
3823 });
3824 out
3825 }
3826
3827 pub fn visit_run_boundaries<F>(&self, mut visitor: F)
3832 where
3833 F: FnMut(
3834 car_proto::RunStarted,
3835 Option<car_proto::RunEnded>,
3836 Option<car_proto::RunCancellationRequested>,
3837 Option<car_proto::RunCancelResponse>,
3838 ),
3839 {
3840 if self.open_root_for_read().is_err() {
3841 return;
3842 }
3843 let Ok(agent_dirs) = std::fs::read_dir(&self.root) else {
3844 return;
3845 };
3846 for agent in agent_dirs.flatten() {
3847 let Ok(file_type) = agent.file_type() else {
3848 continue;
3849 };
3850 if !file_type.is_dir() || file_type.is_symlink() {
3851 continue;
3852 }
3853 let Ok(entries) = std::fs::read_dir(agent.path()) else {
3854 continue;
3855 };
3856 for entry in entries.flatten() {
3857 let path = entry.path();
3858 if path.extension().and_then(|ext| ext.to_str()) != Some("jsonl") {
3859 continue;
3860 }
3861 let Ok(file) = car_secrets::open_private_read(&path) else {
3862 continue;
3863 };
3864 if let Some((started, ended, requested, result)) =
3865 load_private_run_boundaries(&path, &file)
3866 {
3867 visitor(started, ended, requested, result);
3868 }
3869 }
3870 }
3871 }
3872
3873 fn resolve_run_file(&self, run_id: &str) -> Option<(PathBuf, File)> {
3877 self.open_root_for_read().ok()?;
3878 let file_name = format!("{}.jsonl", sanitize(run_id));
3879 let agent_dirs = std::fs::read_dir(&self.root).ok()?;
3880 for agent in agent_dirs {
3881 let agent = agent.ok()?;
3882 let file_type = agent.file_type().ok()?;
3883 if !file_type.is_dir() || file_type.is_symlink() {
3884 continue;
3885 }
3886 car_secrets::ensure_private_dir(&agent.path()).ok()?;
3887 let candidate = agent.path().join(&file_name);
3888 match car_secrets::open_private_read(&candidate) {
3889 Ok(file) => return Some((candidate, file)),
3890 Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue,
3891 Err(_) => return None,
3892 }
3893 }
3894 None
3895 }
3896
3897 pub fn agent_for_run(&self, run_id: &str) -> Option<String> {
3902 let (path, _file) = self.resolve_run_file(run_id)?;
3903 path.parent()
3904 .and_then(Path::file_name)
3905 .and_then(|s| s.to_str())
3906 .map(str::to_string)
3907 }
3908
3909 pub fn gc(&self) -> usize {
3915 let mut removed = 0;
3916 if self.open_root_for_read().is_err() {
3917 return 0;
3918 }
3919 let Ok(agent_dirs) = std::fs::read_dir(&self.root) else {
3920 return 0;
3921 };
3922 let cutoff = self.retention.age_cutoff(Utc::now());
3923 for agent in agent_dirs {
3924 let Ok(agent) = agent else {
3925 return removed;
3926 };
3927 let Ok(file_type) = agent.file_type() else {
3928 return removed;
3929 };
3930 if !file_type.is_dir() || file_type.is_symlink() {
3931 continue;
3932 }
3933 let agent_path = agent.path();
3934 if car_secrets::ensure_private_dir(&agent_path).is_err() {
3935 return removed;
3936 }
3937 removed += self.gc_agent_dir(&agent_path, cutoff);
3938 }
3939 removed
3940 }
3941
3942 pub fn adopt_orphans(&self) -> usize {
3958 let mut adopted = 0;
3959 if self.open_root_for_read().is_err() {
3960 return 0;
3961 }
3962 let Ok(agent_dirs) = std::fs::read_dir(&self.root) else {
3963 return 0;
3964 };
3965 let now = Utc::now();
3966 for agent in agent_dirs {
3967 let Ok(agent) = agent else {
3968 return adopted;
3969 };
3970 let Ok(file_type) = agent.file_type() else {
3971 return adopted;
3972 };
3973 if !file_type.is_dir() || file_type.is_symlink() {
3974 continue;
3975 }
3976 let agent_path = agent.path();
3977 if car_secrets::ensure_private_dir(&agent_path).is_err() {
3978 return adopted;
3979 }
3980 let Ok(entries) = std::fs::read_dir(&agent_path) else {
3981 continue;
3982 };
3983 for entry in entries {
3984 let Ok(entry) = entry else {
3985 return adopted;
3986 };
3987 let path = entry.path();
3988 if path.extension().and_then(|e| e.to_str()) != Some("jsonl") {
3989 continue;
3990 }
3991 let Some(summary) = summarize_file(&path) else {
3992 continue;
3993 };
3994 if summary.status != RunStatus::InProgress {
3995 continue;
3996 }
3997 let pending_state = self.pending_proposal(&summary.run_id);
4003 let execution_state = self.execution_marker(&summary.run_id);
4004 if !matches!(pending_state, Ok(None)) || !matches!(execution_state, Ok(None)) {
4005 continue;
4006 }
4007 let client_id = self
4009 .get_run_trace_for(&summary.agent_id, &summary.run_id)
4010 .and_then(|records| {
4011 records.into_iter().find_map(|record| match record {
4012 RunRecord::Started(started) => started.client_id,
4013 _ => None,
4014 })
4015 });
4016 let termination = RunTermination::Incomplete;
4017 let completion_digest = crate::session::run_completion_digest(&termination).ok();
4018 let incomplete = RunRecord::Ended(car_proto::RunEnded {
4019 run_id: summary.run_id.clone(),
4020 client_id,
4021 agent_id: summary.agent_id.clone(),
4022 termination,
4023 completion_digest,
4024 ended_at: now,
4025 });
4026 if self
4027 .append_records(&summary.agent_id, &summary.run_id, &[incomplete])
4028 .is_ok()
4029 {
4030 adopted += 1;
4031 }
4032 }
4033 }
4034 adopted
4035 }
4036
4037 fn gc_agent_dir(&self, agent_path: &Path, age_cutoff: Option<DateTime<Utc>>) -> usize {
4039 let mut runs: Vec<(PathBuf, RunSummary)> = Vec::new();
4043 let Ok(entries) = std::fs::read_dir(agent_path) else {
4044 return 0;
4045 };
4046 for entry in entries {
4047 let Ok(entry) = entry else {
4048 return 0;
4049 };
4050 let path = entry.path();
4051 if path.extension().and_then(|e| e.to_str()) != Some("jsonl") {
4052 continue;
4053 }
4054 let file_run_id = path
4055 .file_stem()
4056 .and_then(|value| value.to_str())
4057 .unwrap_or_default();
4058 let key = Self::run_summary_key(file_run_id);
4059 let existing = self.read_summary_sidecar(agent_path, &key).ok();
4060 let scan = summarize_file_checked(&path);
4061 let mut summary = match scan {
4062 Ok(scan) => match scan.summary {
4063 Some(summary) => {
4064 if let Some(corruption) = &scan.corruption {
4065 self.remember_run_trace_corruption(
4066 &summary.agent_id,
4067 file_run_id,
4068 corruption,
4069 );
4070 }
4071 summary
4072 }
4073 None => {
4074 let Some(mut existing) = existing.clone() else {
4075 continue;
4076 };
4077 let corruption = scan
4078 .corruption
4079 .or_else(|| {
4080 self.known_run_trace_corruption(&existing.agent_id, file_run_id)
4081 })
4082 .or_else(|| {
4083 self.read_run_trace_corruption_marker(
4084 agent_path,
4085 &existing.agent_id,
4086 file_run_id,
4087 )
4088 .ok()
4089 .flatten()
4090 });
4091 let Some(corruption) = corruption else {
4092 continue;
4093 };
4094 self.remember_run_trace_corruption(
4095 &existing.agent_id,
4096 file_run_id,
4097 &corruption,
4098 );
4099 existing.status = RunStatus::Incomplete;
4100 existing.trace_corruption = Some(corruption);
4101 existing
4102 }
4103 },
4104 Err(error) => {
4105 let Some(corruption) = trace_corruption_from_error(&error) else {
4106 continue;
4107 };
4108 let Some(mut existing) = existing.clone() else {
4109 continue;
4110 };
4111 self.remember_run_trace_corruption(
4112 &existing.agent_id,
4113 file_run_id,
4114 &corruption,
4115 );
4116 existing.status = RunStatus::Incomplete;
4117 existing.trace_corruption = Some(corruption);
4118 existing
4119 }
4120 };
4121 if let Some(existing) = existing {
4122 summary.sequence = existing.sequence;
4123 }
4124 if self
4125 .apply_run_trace_corruption_marker(agent_path, &mut summary)
4126 .is_err()
4127 {
4128 continue;
4129 }
4130 runs.push((path, summary));
4131 }
4132 runs.sort_by(|a, b| b.1.started_at.cmp(&a.1.started_at));
4134
4135 let mut removed = 0;
4136 let mut completed_rank = 0usize;
4140 for (path, summary) in runs.iter() {
4141 if summary.trace_corruption.is_some() {
4144 continue;
4145 }
4146 if matches!(
4149 summary.status,
4150 RunStatus::InProgress | RunStatus::CancellationPending
4151 ) {
4152 continue;
4153 }
4154 let over_count = self
4155 .retention
4156 .count_cap()
4157 .is_some_and(|cap| completed_rank >= cap);
4158 completed_rank += 1;
4159 let term_time = summary.ended_at.unwrap_or(summary.started_at);
4165 let too_old = age_cutoff.is_some_and(|cut| term_time < cut);
4166 if (over_count || too_old) && std::fs::remove_file(path).is_ok() {
4167 removed += 1;
4168 }
4169 }
4170 let retained: Vec<RunSummary> = runs
4171 .into_iter()
4172 .filter_map(|(path, summary)| path.exists().then_some(summary))
4173 .collect();
4174 if let Err(error) = self.write_summary_index(agent_path, &retained) {
4175 tracing::error!(path = %agent_path.display(), %error, "run summary index startup repair failed");
4176 }
4177 removed
4178 }
4179}
4180
4181fn same_run_ended(left: &car_proto::RunEnded, right: &car_proto::RunEnded) -> bool {
4182 serde_json::to_value(left).ok() == serde_json::to_value(right).ok()
4183}
4184
4185fn append_jsonl_batch_on_descriptor(file: &mut File, records: &[RunRecord]) -> std::io::Result<()> {
4190 car_secrets::revalidate_private_file(file)?;
4191 let mut buf = Vec::new();
4197 if last_byte_is_not_newline(file)? {
4198 buf.push(b'\n');
4199 }
4200 for rec in records {
4201 let line = serde_json::to_string(rec)
4202 .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
4203 buf.extend_from_slice(line.as_bytes());
4204 buf.push(b'\n');
4205 }
4206 file.write_all(&buf)?;
4207 car_secrets::revalidate_private_file(file)
4208}
4209
4210fn append_jsonl_batch_to_path(
4215 path: &Path,
4216 file: &mut File,
4217 records: &[RunRecord],
4218) -> std::io::Result<()> {
4219 car_secrets::revalidate_private_path(path, file)?;
4220 append_jsonl_batch_on_descriptor(file, records)?;
4221 car_secrets::revalidate_private_path(path, file)
4222}
4223
4224fn last_byte_is_not_newline(file: &mut File) -> std::io::Result<bool> {
4229 let len = file.seek(SeekFrom::End(0))?;
4230 if len == 0 {
4231 return Ok(false);
4232 }
4233 file.seek(SeekFrom::End(-1))?;
4234 let mut buf = [0u8; 1];
4235 file.read_exact(&mut buf)?;
4236 Ok(buf[0] != b'\n')
4237}
4238
4239#[derive(Debug)]
4240enum RecordScanError {
4241 Io(std::io::Error),
4242 Malformed { line: usize, detail: String },
4243}
4244
4245#[derive(Debug)]
4246struct StrictRunTraceCorruptionError {
4247 corruption: RunTraceCorruption,
4248 detail: String,
4249}
4250
4251impl std::fmt::Display for StrictRunTraceCorruptionError {
4252 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
4253 write!(
4254 formatter,
4255 "malformed run trace record at line {}: {}",
4256 self.corruption.line, self.detail
4257 )
4258 }
4259}
4260
4261impl std::error::Error for StrictRunTraceCorruptionError {}
4262
4263fn scan_error_to_io(error: RecordScanError) -> std::io::Error {
4264 match error {
4265 RecordScanError::Io(error) => error,
4266 RecordScanError::Malformed { line, detail } => std::io::Error::new(
4267 std::io::ErrorKind::InvalidData,
4268 StrictRunTraceCorruptionError {
4269 corruption: RunTraceCorruption {
4270 kind: RunTraceCorruptionKind::MalformedRecord,
4271 line,
4272 },
4273 detail,
4274 },
4275 ),
4276 }
4277}
4278
4279fn scan_records<F>(file: &File, mut visit: F) -> Result<(), RecordScanError>
4283where
4284 F: FnMut(RunRecord),
4285{
4286 let cloned = file.try_clone().map_err(RecordScanError::Io)?;
4287 let mut reader = std::io::BufReader::new(cloned);
4288 let mut bytes = Vec::new();
4289 let mut line = 0usize;
4290 loop {
4291 bytes.clear();
4292 let read = reader
4293 .read_until(b'\n', &mut bytes)
4294 .map_err(RecordScanError::Io)?;
4295 if read == 0 {
4296 break;
4297 }
4298 line = line.saturating_add(1);
4299 let terminated = bytes.last() == Some(&b'\n');
4300 if terminated {
4301 bytes.pop();
4302 if bytes.last() == Some(&b'\r') {
4303 bytes.pop();
4304 }
4305 }
4306 if bytes.iter().all(u8::is_ascii_whitespace) {
4307 continue;
4308 }
4309 match serde_json::from_slice::<RunRecord>(&bytes) {
4310 Ok(record) => visit(record),
4311 Err(_) if !terminated => break,
4312 Err(error) => {
4313 return Err(RecordScanError::Malformed {
4314 line,
4315 detail: error.to_string(),
4316 })
4317 }
4318 }
4319 }
4320 Ok(())
4321}
4322
4323fn load_records(file: &File) -> std::io::Result<Vec<RunRecord>> {
4324 let mut records = Vec::new();
4325 scan_records(file, |record| records.push(record)).map_err(scan_error_to_io)?;
4326 Ok(records)
4327}
4328
4329fn load_private_records(path: &Path, file: &File) -> std::io::Result<Vec<RunRecord>> {
4330 let records = load_records(file)?;
4331 car_secrets::revalidate_private_path(path, file)?;
4332 Ok(records)
4333}
4334
4335fn load_private_run_boundaries(
4339 path: &Path,
4340 file: &File,
4341) -> Option<(
4342 car_proto::RunStarted,
4343 Option<car_proto::RunEnded>,
4344 Option<car_proto::RunCancellationRequested>,
4345 Option<car_proto::RunCancelResponse>,
4346)> {
4347 let mut started = None;
4348 let mut ended = None;
4349 let mut requested = None;
4350 let mut result = None;
4351 scan_records(file, |record| match record {
4352 RunRecord::Started(row) if started.is_none() => started = Some(row),
4353 RunRecord::Ended(row) if ended.is_none() => ended = Some(row),
4354 RunRecord::CancellationRequested(row) if requested.is_none() => requested = Some(row),
4355 RunRecord::CancellationResult(row) if result.is_none() => result = Some(row),
4356 _ => {}
4357 })
4358 .ok()?;
4359 car_secrets::revalidate_private_path(path, file).ok()?;
4360 started.map(|started| (started, ended, requested, result))
4361}
4362
4363fn reject_corrupt_summary(summary: &RunSummary) -> std::io::Result<()> {
4364 if let Some(corruption) = &summary.trace_corruption {
4365 return Err(std::io::Error::new(
4366 std::io::ErrorKind::InvalidData,
4367 format!("malformed run trace record at line {}", corruption.line),
4368 ));
4369 }
4370 Ok(())
4371}
4372
4373#[derive(Debug)]
4374struct RunSummaryRefreshError {
4375 source: std::io::Error,
4376 corruption: RunTraceCorruption,
4377}
4378
4379impl std::fmt::Display for RunSummaryRefreshError {
4380 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
4381 write!(
4382 formatter,
4383 "malformed run trace record at line {}; summary persistence also failed: {}",
4384 self.corruption.line, self.source
4385 )
4386 }
4387}
4388
4389impl std::error::Error for RunSummaryRefreshError {
4390 fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
4391 Some(&self.source)
4392 }
4393}
4394
4395fn summary_refresh_error(source: std::io::Error, corruption: RunTraceCorruption) -> std::io::Error {
4396 std::io::Error::new(source.kind(), RunSummaryRefreshError { source, corruption })
4397}
4398
4399fn preserve_summary_corruption<T>(
4400 result: std::io::Result<T>,
4401 corruption: &Option<RunTraceCorruption>,
4402) -> std::io::Result<T> {
4403 match corruption {
4404 Some(corruption) => {
4405 result.map_err(|source| summary_refresh_error(source, corruption.clone()))
4406 }
4407 None => result,
4408 }
4409}
4410
4411pub(crate) fn trace_corruption_from_error(error: &std::io::Error) -> Option<RunTraceCorruption> {
4412 let source = error.get_ref()?;
4413 if let Some(failure) = source.downcast_ref::<RunSummaryRefreshError>() {
4414 return Some(failure.corruption.clone());
4415 }
4416 source
4417 .downcast_ref::<StrictRunTraceCorruptionError>()
4418 .map(|failure| failure.corruption.clone())
4419}
4420
4421pub(crate) fn is_trace_corruption_error(error: &std::io::Error) -> bool {
4422 trace_corruption_from_error(error).is_some()
4423 || (error.kind() == std::io::ErrorKind::InvalidData
4424 && error.to_string().contains("malformed run trace record"))
4425}
4426
4427fn summary_from_records(
4428 started: Option<car_proto::RunStarted>,
4429 ended: Option<car_proto::RunEnded>,
4430 turn_count: usize,
4431 cancellation_pending: bool,
4432 trace_corruption: Option<RunTraceCorruption>,
4433) -> Option<RunSummary> {
4434 let started = started?;
4435 let (status, ended_at) = if trace_corruption.is_some() {
4436 (RunStatus::Incomplete, None)
4437 } else {
4438 match &ended {
4439 Some(e) => {
4440 let status = match &e.termination {
4441 RunTermination::Outcome { .. } => RunStatus::Completed,
4442 RunTermination::Incomplete => RunStatus::Incomplete,
4443 RunTermination::Cancelled { .. } => RunStatus::Cancelled,
4444 };
4445 (status, Some(e.ended_at))
4446 }
4447 None if cancellation_pending => (RunStatus::CancellationPending, None),
4448 None => (RunStatus::InProgress, None),
4449 }
4450 };
4451 Some(RunSummary {
4452 run_id: started.run_id,
4453 agent_id: started.agent_id,
4454 intent: started.intent,
4455 started_at: started.started_at,
4456 ended_at,
4457 status,
4458 turn_count,
4459 sequence: 0,
4460 trace_corruption,
4461 })
4462}
4463
4464struct RunSummaryScan {
4469 summary: Option<RunSummary>,
4470 corruption: Option<RunTraceCorruption>,
4471}
4472
4473fn summarize_file_checked(path: &Path) -> std::io::Result<RunSummaryScan> {
4474 let file = car_secrets::open_private_read(path)?;
4475 let mut started: Option<car_proto::RunStarted> = None;
4476 let mut ended: Option<car_proto::RunEnded> = None;
4477 let mut turn_count = 0usize;
4478 let mut cancellation_pending = false;
4479 let scan = scan_records(&file, |record| match record {
4480 RunRecord::Started(s) => started = Some(s),
4481 RunRecord::Ended(e) => ended = Some(e),
4482 RunRecord::Turn(_) => turn_count += 1,
4483 RunRecord::CancellationRequested(_) | RunRecord::CancellationResult(_) => {
4484 cancellation_pending = true
4485 }
4486 });
4487 let trace_corruption = match scan {
4488 Ok(()) => None,
4489 Err(RecordScanError::Malformed { line, .. }) => Some(RunTraceCorruption {
4490 kind: RunTraceCorruptionKind::MalformedRecord,
4491 line,
4492 }),
4493 Err(RecordScanError::Io(error)) => return Err(error),
4494 };
4495 preserve_summary_corruption(
4496 car_secrets::revalidate_private_path(path, &file),
4497 &trace_corruption,
4498 )?;
4499 let summary = summary_from_records(
4500 started,
4501 ended,
4502 turn_count,
4503 cancellation_pending,
4504 trace_corruption.clone(),
4505 );
4506 Ok(RunSummaryScan {
4507 summary,
4508 corruption: trace_corruption,
4509 })
4510}
4511
4512fn summarize_file(path: &Path) -> Option<RunSummary> {
4513 summarize_file_checked(path).ok()?.summary
4514}
4515
4516fn sanitize(id: &str) -> String {
4522 let cleaned: String = id
4523 .chars()
4524 .map(|c| {
4525 if c.is_control() || matches!(c, '<' | '>' | ':' | '"' | '/' | '\\' | '|' | '?' | '*') {
4526 '_'
4527 } else {
4528 c
4529 }
4530 })
4531 .collect();
4532 let trimmed = cleaned.trim_matches(['.', ' ']);
4533 if trimmed.is_empty() {
4534 "_".to_string()
4535 } else {
4536 trimmed.to_string()
4537 }
4538}
4539
4540fn ensure_backup_excluded(dir: &Path) -> std::io::Result<()> {
4545 let marker = dir.join(".nobackup");
4546 match car_secrets::create_private_file(&marker) {
4547 Ok(mut file) => {
4548 file.write_all(b"car run traces - excluded from backup\n")?;
4549 car_secrets::revalidate_private_file(&file)?;
4550 car_secrets::revalidate_private_path(&marker, &file)?;
4551 }
4552 Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {
4553 let file = car_secrets::open_private_read(&marker)?;
4554 car_secrets::revalidate_private_path(&marker, &file)?;
4555 }
4556 Err(error) => return Err(error),
4557 }
4558 #[cfg(target_os = "macos")]
4559 set_macos_backup_excluded(dir);
4560 car_secrets::ensure_private_dir(dir)?;
4561 Ok(())
4562}
4563
4564#[cfg(target_os = "macos")]
4569fn set_macos_backup_excluded(dir: &Path) {
4570 let _ = std::process::Command::new("xattr")
4573 .args(["-w", "com.apple.metadata:com_apple_backup_excludeItem", "1"])
4574 .arg(dir)
4575 .output();
4576}
4577
4578#[cfg(test)]
4579mod tests {
4580 use super::*;
4581 use car_ir::{AgentOutcome, CostSummary, OutcomeMetrics, OutcomeStatus, ProposalLineageEntry};
4582 use car_proto::{RunEnded, RunStarted, RunTurn, VerifierVerdict};
4583 use serde_json::json;
4584
4585 #[test]
4586 fn run_path_segments_replace_windows_reserved_characters() {
4587 assert_eq!(sanitize("name:bulldozer-agent"), "name_bulldozer-agent");
4588 assert_eq!(sanitize("<>:\"/\\|?*\0"), "__________");
4589 assert_eq!(sanitize(".. "), "_");
4590 }
4591
4592 fn store(root: PathBuf) -> RunStore {
4593 RunStore::new(root, RetentionConfig::default())
4594 }
4595
4596 fn started(run_id: &str, agent_id: &str, when: DateTime<Utc>) -> RunStarted {
4597 RunStarted {
4598 run_id: run_id.to_string(),
4599 client_id: Some("test-client".to_string()),
4600 agent_id: agent_id.to_string(),
4601 intent: "do the thing".to_string(),
4602 outcome_description: None,
4603 started_at: when,
4604 }
4605 }
4606
4607 fn turn(index: usize, prompt: &str) -> RunRecord {
4608 RunRecord::Turn(RunTurn {
4609 index,
4610 proposal_id: None,
4611 action_id: None,
4612 action_status: None,
4613 action_duration_ms: None,
4614 action_completed_at: None,
4615 depends_on: None,
4616 state_dependencies: None,
4617 prompt: Some(prompt.to_string()),
4618 tool: Some("drive_cli".to_string()),
4619 parameters: json!({ "prompt": prompt }),
4620 output: Some(json!({ "exit_code": 0 })),
4621 cli_outcome: None,
4622 verifier_verdict: VerifierVerdict::NotRun,
4623 policy_rejected: None,
4624 })
4625 }
4626
4627 fn ended(run_id: &str, agent_id: &str, status: OutcomeStatus) -> RunRecord {
4628 let outcome = AgentOutcome {
4629 status,
4630 summary: "done".to_string(),
4631 evidence: vec![],
4632 metrics: OutcomeMetrics::default(),
4633 timestamp: Utc::now(),
4634 };
4635 let termination = RunTermination::Outcome { status, outcome };
4636 let completion_digest = crate::session::run_completion_digest(&termination).unwrap();
4637 RunRecord::Ended(RunEnded {
4638 run_id: run_id.to_string(),
4639 client_id: Some("test-client".to_string()),
4640 agent_id: agent_id.to_string(),
4641 termination,
4642 completion_digest: Some(completion_digest),
4643 ended_at: Utc::now(),
4644 })
4645 }
4646
4647 fn valid_pending(run_id: &str) -> PendingProposalFinalization {
4648 let proposal = ActionProposal {
4649 id: format!("proposal-{run_id}"),
4650 source: "run-store-test".to_string(),
4651 actions: Vec::new(),
4652 timestamp: Utc::now(),
4653 context: HashMap::new(),
4654 };
4655 let digest = proposal_digest(&proposal).unwrap();
4656 let proposal_result = ProposalResult {
4657 proposal_id: proposal.id.clone(),
4658 original_proposal_id: proposal.id.clone(),
4659 final_proposal: Some(proposal.clone()),
4660 replan_lineage: vec![ProposalLineageEntry {
4661 generation: 0,
4662 proposal_id: proposal.id.clone(),
4663 proposal_digest: Some(digest.clone()),
4664 status: ProposalLineageStatus::Accepted,
4665 rejection_reason: None,
4666 }],
4667 accepted_proposal_preimages: vec![car_ir::AcceptedProposalPreimage {
4668 generation: 0,
4669 proposal_digest: digest,
4670 proposal: proposal.clone(),
4671 }],
4672 results: Vec::new(),
4673 cost: CostSummary::default(),
4674 };
4675 let result_value = serde_json::to_value(&proposal_result).unwrap();
4676 let canonical = car_inference::catalog_identity::canonical_json(&result_value).unwrap();
4677 PendingProposalFinalization {
4678 run_id: run_id.to_string(),
4679 client_id: "test-client".to_string(),
4680 requested_policy_session_id: None,
4681 policy_session_id: None,
4682 original_proposal_id: proposal.id.clone(),
4683 final_proposal_id: proposal.id.clone(),
4684 original_submission: json!({
4685 "id": proposal.id.clone(),
4686 "source": "run-store-test",
4687 "actions": []
4688 }),
4689 original_proposal: proposal.clone(),
4690 final_proposal: proposal.clone(),
4691 accepted_proposal_preimages: vec![AcceptedProposalPreimage {
4692 generation: 0,
4693 proposal,
4694 }],
4695 proposal_result,
4696 result_digest: format!("{:x}", Sha256::digest(canonical.as_bytes())),
4697 }
4698 }
4699
4700 fn refresh_pending_result_digest(pending: &mut PendingProposalFinalization) {
4701 let value = serde_json::to_value(&pending.proposal_result).unwrap();
4702 let canonical = car_inference::catalog_identity::canonical_json(&value).unwrap();
4703 pending.result_digest = format!("{:x}", Sha256::digest(canonical.as_bytes()));
4704 }
4705
4706 fn marker_for_pending(pending: &PendingProposalFinalization) -> ProposalExecutionMarker {
4707 ProposalExecutionMarker {
4708 run_id: pending.run_id.clone(),
4709 client_id: pending.client_id.clone(),
4710 requested_policy_session_id: pending.requested_policy_session_id.clone(),
4711 policy_session_id: pending.policy_session_id.clone(),
4712 original_proposal_id: pending.original_proposal_id.clone(),
4713 original_submission: pending.original_submission.clone(),
4714 original_proposal: serde_json::to_value(&pending.original_proposal).unwrap(),
4715 proposal_digest: proposal_digest(&pending.original_proposal).unwrap(),
4716 }
4717 }
4718
4719 fn rebind_pending_original(pending: &mut PendingProposalFinalization) {
4720 pending.original_submission["source"] = json!("pending-self-claim");
4721 pending.original_proposal.source = "pending-self-claim".to_string();
4722 pending.final_proposal = pending.original_proposal.clone();
4723 pending.accepted_proposal_preimages[0].proposal = pending.original_proposal.clone();
4724 pending.proposal_result.final_proposal = Some(pending.original_proposal.clone());
4725 pending.proposal_result.replan_lineage[0].proposal_digest =
4726 Some(proposal_digest(&pending.original_proposal).unwrap());
4727 refresh_pending_result_digest(pending);
4728 }
4729
4730 fn write_pending_provenance(store: &RunStore, pending: &PendingProposalFinalization) {
4731 store
4732 .write_started(&started(&pending.run_id, "agent-a", Utc::now()))
4733 .unwrap();
4734 store
4735 .write_execution_marker(&marker_for_pending(pending))
4736 .unwrap();
4737 }
4738
4739 fn write_completed_receipt_fixture(
4740 store: &RunStore,
4741 pending: PendingProposalFinalization,
4742 exact_bytes: Option<usize>,
4743 invalid_tail: bool,
4744 ) -> CompletedProposalResponse {
4745 let receipt = CompletedProposalResponse {
4746 finalization: pending,
4747 };
4748 receipt.validate().unwrap();
4749 let pending = &receipt.finalization;
4750 let run_root = store.completed_response_run_root(&pending.run_id);
4751 store.ensure_private_dir(&run_root).unwrap();
4752 let path = store
4753 .completed_response_path(
4754 &pending.run_id,
4755 &pending.client_id,
4756 pending.requested_policy_session_id.as_deref(),
4757 &pending.original_submission,
4758 )
4759 .unwrap();
4760 let mut bytes = serde_json::to_vec(&receipt).unwrap();
4761 if let Some(exact_bytes) = exact_bytes {
4762 assert!(bytes.len() <= exact_bytes);
4763 bytes.resize(exact_bytes, b' ');
4764 }
4765 if invalid_tail {
4766 bytes.push(b'!');
4767 }
4768 let mut file = store.create_private_file(&path).unwrap();
4769 file.write_all(&bytes).unwrap();
4770 file.sync_all().unwrap();
4771 receipt
4772 }
4773
4774 #[test]
4775 fn first_use_agent_receipt_parent_sync_failure_is_not_acknowledged_and_retry_is_exact() {
4776 let tmp = tempfile::TempDir::new().unwrap();
4777 let failures = car_secrets::PrivatePathDurabilityFailureInjector::default();
4778 let store =
4779 store(tmp.path().join("runs")).with_private_path_failure_injector(failures.clone());
4780 store.prepare_storage().unwrap();
4781 failures.fail_next(car_secrets::PrivatePathDurabilityFailurePoint::ParentDirectorySync);
4782 let boundary = started("private-first-use", "new-agent", Utc::now());
4783
4784 let error = store.write_started(&boundary).unwrap_err();
4785 assert!(error
4786 .to_string()
4787 .contains("injected private-path parent directory sync failure"));
4788 assert!(
4789 store.get_run_trace(&boundary.run_id).is_none(),
4790 "an unacknowledged agent-directory entry must not contain a run boundary"
4791 );
4792
4793 store.write_started(&boundary).unwrap();
4794 let trace = store.get_run_trace(&boundary.run_id).unwrap();
4795 assert_eq!(
4796 trace
4797 .iter()
4798 .filter(|row| matches!(row, RunRecord::Started(_)))
4799 .count(),
4800 1,
4801 "retry must persist exactly one RunStarted boundary"
4802 );
4803 }
4804
4805 #[test]
4806 fn proposal_sidecar_parent_sync_failures_are_not_acknowledged_and_retry_exactly() {
4807 let tmp = tempfile::TempDir::new().unwrap();
4808 let failures = car_secrets::PrivatePathDurabilityFailureInjector::default();
4809 let store =
4810 store(tmp.path().join("runs")).with_private_path_failure_injector(failures.clone());
4811 let pending = valid_pending("private-sidecar-first-use");
4812 store
4813 .write_started(&started(&pending.run_id, "agent-a", Utc::now()))
4814 .unwrap();
4815 let marker = marker_for_pending(&pending);
4816
4817 failures.fail_next(car_secrets::PrivatePathDurabilityFailurePoint::ParentDirectorySync);
4818 assert!(store.write_execution_marker(&marker).is_err());
4819 assert_eq!(store.execution_marker(&pending.run_id).unwrap(), None);
4820 store.write_execution_marker(&marker).unwrap();
4821 assert_eq!(
4822 store.execution_marker(&pending.run_id).unwrap(),
4823 Some(marker)
4824 );
4825
4826 failures.fail_next(car_secrets::PrivatePathDurabilityFailurePoint::ParentDirectorySync);
4827 assert!(store.write_pending_proposal(&pending).is_err());
4828 assert_eq!(store.pending_proposal(&pending.run_id).unwrap(), None);
4829 store.write_pending_proposal(&pending).unwrap();
4830 assert_eq!(
4831 store.pending_proposal(&pending.run_id).unwrap(),
4832 Some(pending)
4833 );
4834 }
4835
4836 #[test]
4840 fn completed_run_readable_after_restart() {
4841 let tmp = tempfile::TempDir::new().unwrap();
4842 let root = tmp.path().join("runs");
4843 let s1 = store(root.clone());
4844 s1.write_started(&started("run-1", "agent-a", Utc::now()))
4845 .unwrap();
4846 s1.append_turns("agent-a", "run-1", &[turn(0, "first")])
4847 .unwrap();
4848 s1.append_records(
4849 "agent-a",
4850 "run-1",
4851 &[ended("run-1", "agent-a", OutcomeStatus::Success)],
4852 )
4853 .unwrap();
4854
4855 let s2 = store(root);
4857 let trace = s2
4858 .get_run_trace("run-1")
4859 .expect("trace readable after restart");
4860 assert!(matches!(trace.first(), Some(RunRecord::Started(_))));
4861 assert!(matches!(trace.last(), Some(RunRecord::Ended(_))));
4862 let turns = trace
4863 .iter()
4864 .filter(|r| matches!(r, RunRecord::Turn(_)))
4865 .count();
4866 assert_eq!(turns, 1);
4867 }
4868
4869 #[test]
4872 fn runs_isolated_per_agent_and_run() {
4873 let tmp = tempfile::TempDir::new().unwrap();
4874 let s = store(tmp.path().join("runs"));
4875 s.write_started(&started("run-1", "agent-a", Utc::now()))
4876 .unwrap();
4877 s.append_turns("agent-a", "run-1", &[turn(0, "a-first")])
4878 .unwrap();
4879 s.write_started(&started("run-2", "agent-a", Utc::now()))
4880 .unwrap();
4881 s.append_turns("agent-a", "run-2", &[turn(0, "a-second")])
4882 .unwrap();
4883 s.write_started(&started("run-3", "agent-b", Utc::now()))
4884 .unwrap();
4885 s.append_turns("agent-b", "run-3", &[turn(0, "b-first")])
4886 .unwrap();
4887
4888 let t1 = s.get_run_trace("run-1").unwrap();
4890 let t2 = s.get_run_trace("run-2").unwrap();
4891 let t3 = s.get_run_trace("run-3").unwrap();
4892 assert_eq!(turn_prompt(&t1), "a-first");
4893 assert_eq!(turn_prompt(&t2), "a-second");
4894 assert_eq!(turn_prompt(&t3), "b-first");
4895 assert_eq!(s.agent_for_run("run-1").as_deref(), Some("agent-a"));
4897 assert_eq!(s.agent_for_run("run-3").as_deref(), Some("agent-b"));
4898 assert_eq!(s.list_runs("agent-a").len(), 2);
4900 assert_eq!(s.list_runs("agent-b").len(), 1);
4901 }
4902
4903 #[test]
4904 fn summary_index_pages_in_stable_newest_first_order() {
4905 let tmp = tempfile::TempDir::new().unwrap();
4906 let s = store(tmp.path().join("runs"));
4907 let now = Utc::now();
4908 s.write_started(&started(
4909 "run-old",
4910 "agent-page",
4911 now - chrono::Duration::seconds(2),
4912 ))
4913 .unwrap();
4914 s.append_turns("agent-page", "run-old", &[turn(0, "old")])
4915 .unwrap();
4916 s.write_started(&started("run-new", "agent-page", now))
4917 .unwrap();
4918 s.append_turns(
4919 "agent-page",
4920 "run-new",
4921 &[turn(0, "new-0"), turn(1, "new-1")],
4922 )
4923 .unwrap();
4924
4925 let (first, next) = s.list_runs_page("agent-page", 0, 1).unwrap();
4926 assert_eq!(first.len(), 1);
4927 assert_eq!(first[0].run_id, "run-new");
4928 assert_eq!(first[0].turn_count, 2);
4929 let next = next.expect("older run remains");
4930
4931 s.write_started(&started(
4934 "run-newest",
4935 "agent-page",
4936 now + chrono::Duration::seconds(1),
4937 ))
4938 .unwrap();
4939
4940 let (second, next) = s.list_runs_page("agent-page", next, 1).unwrap();
4941 assert_eq!(second.len(), 1);
4942 assert_eq!(second[0].run_id, "run-old");
4943 assert_eq!(second[0].turn_count, 1);
4944 assert_eq!(next, None);
4945
4946 let (past_end, next) = s.list_runs_page("agent-page", 1, 1).unwrap();
4947 assert!(past_end.is_empty());
4948 assert_eq!(next, None);
4949 }
4950
4951 #[test]
4952 fn newline_terminated_malformed_record_quarantines_completed_trace() {
4953 let tmp = tempfile::TempDir::new().unwrap();
4954 let root = tmp.path().join("runs");
4955 let s = store(root.clone());
4956 s.write_started(&started("corrupt", "agent-a", Utc::now()))
4957 .unwrap();
4958 s.append_turns("agent-a", "corrupt", &[turn(0, "before")])
4959 .unwrap();
4960
4961 let path = root.join("agent-a").join("corrupt.jsonl");
4962 let mut file = car_secrets::open_private_append(&path).unwrap();
4963 file.write_all(b"{malformed-middle}\n").unwrap();
4964 file.flush().unwrap();
4965 file.sync_all().unwrap();
4966 drop(file);
4967
4968 let RunRecord::Ended(terminal) = ended("corrupt", "agent-a", OutcomeStatus::Success) else {
4969 unreachable!()
4970 };
4971 let error = s
4972 .write_ended(&terminal)
4973 .expect_err("a durable malformed row must prevent completion");
4974 assert!(error.to_string().contains("line 3"), "{error}");
4975
4976 let (summaries, _) = s.list_runs_page("agent-a", 0, 1).unwrap();
4977 assert_eq!(summaries[0].status, RunStatus::Incomplete);
4978 assert_eq!(
4979 serde_json::to_value(&summaries[0]).unwrap()["trace_corruption"],
4980 json!({"kind":"malformed_record","line":3})
4981 );
4982 assert!(s
4983 .get_run_turn_page_for("agent-a", "corrupt", 0, 10)
4984 .unwrap_err()
4985 .to_string()
4986 .contains("line 3"));
4987 }
4988
4989 fn turn_prompt(trace: &[RunRecord]) -> String {
4990 trace
4991 .iter()
4992 .find_map(|r| match r {
4993 RunRecord::Turn(t) => t.prompt.clone(),
4994 _ => None,
4995 })
4996 .unwrap_or_default()
4997 }
4998
4999 #[cfg(unix)]
5001 #[test]
5002 fn perms_are_0600_files_0700_dirs() {
5003 use std::os::unix::fs::PermissionsExt;
5004 let tmp = tempfile::TempDir::new().unwrap();
5005 let root = tmp.path().join("runs");
5006 let s = store(root.clone());
5007 s.write_started(&started("run-1", "agent-a", Utc::now()))
5008 .unwrap();
5009
5010 let file = root.join("agent-a").join("run-1.jsonl");
5011 let fmode = std::fs::metadata(&file).unwrap().permissions().mode() & 0o777;
5012 assert_eq!(fmode, 0o600, "run file must be 0600, got {:o}", fmode);
5013
5014 let root_mode = std::fs::metadata(&root).unwrap().permissions().mode() & 0o777;
5015 assert_eq!(
5016 root_mode, 0o700,
5017 "runs/ dir must be 0700, got {:o}",
5018 root_mode
5019 );
5020 let agent_mode = std::fs::metadata(root.join("agent-a"))
5021 .unwrap()
5022 .permissions()
5023 .mode()
5024 & 0o777;
5025 assert_eq!(
5026 agent_mode, 0o700,
5027 "agent dir must be 0700, got {:o}",
5028 agent_mode
5029 );
5030
5031 let marker = root.join(".nobackup");
5033 assert!(marker.exists(), ".nobackup marker written");
5034 let marker_mode = std::fs::metadata(marker).unwrap().permissions().mode() & 0o777;
5035 assert_eq!(
5036 marker_mode, 0o600,
5037 ".nobackup marker must be 0600, got {:o}",
5038 marker_mode
5039 );
5040 }
5041
5042 #[cfg(unix)]
5043 #[test]
5044 fn replay_hardens_owned_legacy_tree_without_changing_content() {
5045 use std::os::unix::fs::PermissionsExt;
5046
5047 let tmp = tempfile::TempDir::new().unwrap();
5048 let root = tmp.path().join("runs");
5049 let agent = root.join("agent-a");
5050 std::fs::create_dir_all(&agent).unwrap();
5051 let path = agent.join("run-1.jsonl");
5052 let line =
5053 serde_json::to_string(&RunRecord::Started(started("run-1", "agent-a", Utc::now())))
5054 .unwrap();
5055 let original = format!("{line}\n");
5056 std::fs::write(&path, original.as_bytes()).unwrap();
5057 std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o755)).unwrap();
5058 std::fs::set_permissions(&agent, std::fs::Permissions::from_mode(0o755)).unwrap();
5059 std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644)).unwrap();
5060
5061 let trace = store(root.clone())
5062 .get_run_trace_for("agent-a", "run-1")
5063 .expect("legacy run remains replayable");
5064 assert_eq!(trace.len(), 1);
5065 assert_eq!(std::fs::read_to_string(&path).unwrap(), original);
5066 assert_eq!(
5067 std::fs::metadata(root).unwrap().permissions().mode() & 0o777,
5068 0o700
5069 );
5070 assert_eq!(
5071 std::fs::metadata(agent).unwrap().permissions().mode() & 0o777,
5072 0o700
5073 );
5074 assert_eq!(
5075 std::fs::metadata(path).unwrap().permissions().mode() & 0o777,
5076 0o600
5077 );
5078 }
5079
5080 #[cfg(unix)]
5081 #[test]
5082 fn run_file_symlinks_and_hardlinks_are_rejected_without_touching_victim() {
5083 use std::os::unix::fs::symlink;
5084
5085 let tmp = tempfile::TempDir::new().unwrap();
5086 let root = tmp.path().join("runs");
5087 let agent = root.join("agent-a");
5088 std::fs::create_dir_all(&agent).unwrap();
5089 let victim = tmp.path().join("victim.jsonl");
5090 std::fs::write(&victim, b"victim\n").unwrap();
5091
5092 let symlink_path = agent.join("symlink.jsonl");
5093 symlink(&victim, &symlink_path).unwrap();
5094 let s = store(root.clone());
5095 assert!(s.get_run_trace_for("agent-a", "symlink").is_none());
5096 assert!(s
5097 .append_records(
5098 "agent-a",
5099 "symlink",
5100 &[RunRecord::Started(started(
5101 "symlink",
5102 "agent-a",
5103 Utc::now()
5104 ))],
5105 )
5106 .is_err());
5107 assert_eq!(std::fs::read(&victim).unwrap(), b"victim\n");
5108
5109 std::fs::remove_file(&symlink_path).unwrap();
5110 let hardlink_path = agent.join("hardlink.jsonl");
5111 std::fs::hard_link(&victim, &hardlink_path).unwrap();
5112 assert!(s.get_run_trace_for("agent-a", "hardlink").is_none());
5113 assert!(s
5114 .append_records(
5115 "agent-a",
5116 "hardlink",
5117 &[RunRecord::Started(started(
5118 "hardlink",
5119 "agent-a",
5120 Utc::now()
5121 ))],
5122 )
5123 .is_err());
5124 assert_eq!(std::fs::read(&victim).unwrap(), b"victim\n");
5125 }
5126
5127 #[cfg(unix)]
5128 #[test]
5129 fn symlink_agent_directory_and_special_run_file_are_rejected() {
5130 use std::os::unix::fs::symlink;
5131 use std::os::unix::net::UnixListener;
5132
5133 let tmp = tempfile::TempDir::new().unwrap();
5134 let root = tmp.path().join("runs");
5135 std::fs::create_dir(&root).unwrap();
5136 let victim_dir = tmp.path().join("victim-agent");
5137 std::fs::create_dir(&victim_dir).unwrap();
5138 symlink(&victim_dir, root.join("agent-link")).unwrap();
5139
5140 let s = store(root.clone());
5141 assert!(s.list_runs("agent-link").is_empty());
5142 assert!(s
5143 .append_records(
5144 "agent-link",
5145 "run-1",
5146 &[RunRecord::Started(started(
5147 "run-1",
5148 "agent-link",
5149 Utc::now()
5150 ))],
5151 )
5152 .is_err());
5153 assert!(std::fs::read_dir(&victim_dir).unwrap().next().is_none());
5154
5155 let agent = root.join("agent-a");
5156 std::fs::create_dir(&agent).unwrap();
5157 let socket_path = agent.join("socket.jsonl");
5158 let _listener = UnixListener::bind(&socket_path).unwrap();
5159 assert!(s.get_run_trace_for("agent-a", "socket").is_none());
5160 assert!(s
5161 .append_records(
5162 "agent-a",
5163 "socket",
5164 &[RunRecord::Started(started("socket", "agent-a", Utc::now()))],
5165 )
5166 .is_err());
5167 }
5168
5169 #[cfg(unix)]
5170 #[test]
5171 fn append_uses_the_validated_descriptor_after_path_substitution() {
5172 use std::os::unix::fs::symlink;
5173
5174 let tmp = tempfile::TempDir::new().unwrap();
5175 let path = tmp.path().join("run.jsonl");
5176 std::fs::write(&path, b"torn").unwrap();
5177 let mut file = car_secrets::open_private_append(&path).unwrap();
5178 let victim = tmp.path().join("victim");
5179 std::fs::write(&victim, b"victim\n").unwrap();
5180
5181 std::fs::remove_file(&path).unwrap();
5182 symlink(&victim, &path).unwrap();
5183 assert!(append_jsonl_batch_to_path(&path, &mut file, &[turn(0, "safe")]).is_err());
5184
5185 assert_eq!(std::fs::read(&victim).unwrap(), b"victim\n");
5186 }
5187
5188 #[cfg(unix)]
5189 #[test]
5190 fn replay_rejects_path_substitution_after_open() {
5191 use std::os::unix::fs::symlink;
5192
5193 let tmp = tempfile::TempDir::new().unwrap();
5194 let path = tmp.path().join("run.jsonl");
5195 let line =
5196 serde_json::to_string(&RunRecord::Started(started("run-1", "agent-a", Utc::now())))
5197 .unwrap();
5198 std::fs::write(&path, format!("{line}\n")).unwrap();
5199 let file = car_secrets::open_private_read(&path).unwrap();
5200 let victim = tmp.path().join("victim");
5201 std::fs::write(&victim, b"victim\n").unwrap();
5202
5203 std::fs::rename(&path, tmp.path().join("moved.jsonl")).unwrap();
5204 symlink(&victim, &path).unwrap();
5205
5206 assert!(load_private_records(&path, &file).is_err());
5207 assert_eq!(std::fs::read(&victim).unwrap(), b"victim\n");
5208 }
5209
5210 #[test]
5215 fn orphan_run_status_distinguishes_inprogress_from_incomplete() {
5216 let tmp = tempfile::TempDir::new().unwrap();
5217 let s = store(tmp.path().join("runs"));
5218 let stale = Utc::now() - chrono::Duration::hours(6);
5220 s.write_started(&started("run-1", "agent-a", stale))
5221 .unwrap();
5222 s.append_turns("agent-a", "run-1", &[turn(0, "first")])
5223 .unwrap();
5224
5225 let open = &s.list_runs("agent-a")[0];
5227 assert_eq!(open.status, RunStatus::InProgress);
5228
5229 let incomplete = RunRecord::Ended(RunEnded {
5231 run_id: "run-1".to_string(),
5232 client_id: Some("test-client".to_string()),
5233 agent_id: "agent-a".to_string(),
5234 termination: RunTermination::Incomplete,
5235 completion_digest: Some("test-digest".to_string()),
5236 ended_at: Utc::now(),
5237 });
5238 s.append_records("agent-a", "run-1", &[incomplete]).unwrap();
5239 let closed = &s.list_runs("agent-a")[0];
5240 assert_eq!(closed.status, RunStatus::Incomplete);
5241 }
5242
5243 #[test]
5246 fn gc_evicts_beyond_per_agent_cap_but_never_in_progress() {
5247 let tmp = tempfile::TempDir::new().unwrap();
5248 let root = tmp.path().join("runs");
5249 let s = RunStore::new(root, RetentionConfig::new(3, 30));
5250 let base = Utc::now() - chrono::Duration::days(1);
5252 for i in 0..5 {
5253 let id = format!("c{i}");
5254 let when = base + chrono::Duration::minutes(i);
5255 s.write_started(&started(&id, "agent-a", when)).unwrap();
5256 s.append_records(
5257 "agent-a",
5258 &id,
5259 &[ended(&id, "agent-a", OutcomeStatus::Success)],
5260 )
5261 .unwrap();
5262 }
5263 s.write_started(&started("live", "agent-a", Utc::now()))
5265 .unwrap();
5266
5267 let removed = s.gc();
5268 assert_eq!(removed, 2, "should evict the 2 oldest completed runs");
5271 let remaining = s.list_runs("agent-a");
5272 assert_eq!(remaining.len(), 4);
5274 assert!(
5275 remaining.iter().any(|r| r.run_id == "live"),
5276 "in-progress run must never be evicted"
5277 );
5278 assert!(!remaining.iter().any(|r| r.run_id == "c0"));
5280 assert!(!remaining.iter().any(|r| r.run_id == "c1"));
5281 }
5282
5283 #[test]
5292 fn a_zero_cap_keeps_everything_instead_of_deleting_it() {
5293 let recent = Utc::now();
5297 let ancient = Utc::now() - chrono::Duration::days(400);
5298 for (retention, count, when) in [
5299 (RetentionConfig::new(0, 30), 60, recent),
5301 (RetentionConfig::new(50, 0), 5, ancient),
5303 (RetentionConfig::new(50, -1), 5, recent),
5306 (RetentionConfig::new(0, 0), 60, ancient),
5308 ] {
5309 let tmp = tempfile::TempDir::new().unwrap();
5310 let s = RunStore::new(tmp.path().join("runs"), retention);
5311 for i in 0..count {
5312 let id = format!("r{i}");
5313 s.write_started(&started(&id, "agent-a", when)).unwrap();
5314 s.append_records(
5315 "agent-a",
5316 &id,
5317 &[ended_at(&id, "agent-a", OutcomeStatus::Success, when)],
5318 )
5319 .unwrap();
5320 }
5321
5322 assert_eq!(s.gc(), 0, "{retention:?} must evict nothing");
5323 assert_eq!(s.list_runs("agent-a").len(), count, "{retention:?}");
5324 }
5325 }
5326
5327 #[test]
5334 fn an_absurd_age_cap_disables_itself_rather_than_panicking() {
5335 assert_eq!(RetentionConfig::new(50, i64::MAX).age_cap_days(), None);
5339 assert_eq!(RetentionConfig::new(50, -1).age_cap_days(), None);
5340 assert_eq!(RetentionConfig::new(50, 30).age_cap_days(), Some(30));
5341
5342 let tmp = tempfile::TempDir::new().unwrap();
5343 let s = RunStore::new(tmp.path().join("runs"), RetentionConfig::new(50, i64::MAX));
5344 let old = Utc::now() - chrono::Duration::days(400);
5345 s.write_started(&started("r0", "agent-a", old)).unwrap();
5346 s.append_records(
5347 "agent-a",
5348 "r0",
5349 &[ended_at("r0", "agent-a", OutcomeStatus::Success, old)],
5350 )
5351 .unwrap();
5352
5353 assert_eq!(s.gc(), 0);
5354 assert_eq!(s.list_runs("agent-a").len(), 1);
5355 }
5356
5357 #[test]
5360 fn a_configured_zero_survives_loading() {
5361 let dir = tempfile::TempDir::new().unwrap();
5362 std::fs::write(
5363 dir.path().join("config.toml"),
5364 "[runs]\nmax_per_agent = 0\nmax_age_days = 0\n",
5365 )
5366 .unwrap();
5367 let cfg = RetentionConfig::from_car_dir(dir.path());
5368 assert_eq!(cfg.count_cap(), None, "a configured 0 must not become 50");
5369 assert_eq!(
5370 cfg.age_cap_days(),
5371 None,
5372 "a configured 0 must not become 30"
5373 );
5374 }
5375
5376 #[test]
5378 fn gc_evicts_runs_older_than_age_cap() {
5379 let tmp = tempfile::TempDir::new().unwrap();
5380 let s = RunStore::new(tmp.path().join("runs"), RetentionConfig::new(50, 30));
5381 let old = Utc::now() - chrono::Duration::days(40);
5385 s.write_started(&started("old", "agent-a", old)).unwrap();
5386 s.append_records(
5387 "agent-a",
5388 "old",
5389 &[ended_at("old", "agent-a", OutcomeStatus::Success, old)],
5390 )
5391 .unwrap();
5392 s.write_started(&started("fresh", "agent-a", Utc::now()))
5393 .unwrap();
5394 s.append_records(
5395 "agent-a",
5396 "fresh",
5397 &[ended("fresh", "agent-a", OutcomeStatus::Success)],
5398 )
5399 .unwrap();
5400
5401 let removed = s.gc();
5402 assert_eq!(removed, 1, "the 40-day-old run should be evicted");
5403 let remaining = s.list_runs("agent-a");
5404 assert_eq!(remaining.len(), 1);
5405 assert_eq!(remaining[0].run_id, "fresh");
5406 }
5407
5408 #[test]
5411 fn gc_never_evicts_stale_in_progress_run() {
5412 let tmp = tempfile::TempDir::new().unwrap();
5413 let s = RunStore::new(tmp.path().join("runs"), RetentionConfig::new(1, 1));
5414 let old = Utc::now() - chrono::Duration::days(40);
5415 s.write_started(&started("stale-live", "agent-a", old))
5417 .unwrap();
5418 let removed = s.gc();
5419 assert_eq!(removed, 0);
5420 assert!(s
5421 .list_runs("agent-a")
5422 .iter()
5423 .any(|r| r.run_id == "stale-live"));
5424 }
5425
5426 #[test]
5429 fn corrupt_trailing_line_loads_prior_records() {
5430 let tmp = tempfile::TempDir::new().unwrap();
5431 let root = tmp.path().join("runs");
5432 let s = store(root.clone());
5433 s.write_started(&started("run-1", "agent-a", Utc::now()))
5434 .unwrap();
5435 s.append_turns("agent-a", "run-1", &[turn(0, "first"), turn(1, "second")])
5436 .unwrap();
5437
5438 let path = root.join("agent-a").join("run-1.jsonl");
5441 let mut f = std::fs::OpenOptions::new()
5442 .append(true)
5443 .open(&path)
5444 .unwrap();
5445 f.write_all(b"{\"record\":\"turn\",\"index\":2,\"prom")
5446 .unwrap();
5447
5448 let trace = s.get_run_trace("run-1").expect("trace still loads");
5449 let turns = trace
5451 .iter()
5452 .filter(|r| matches!(r, RunRecord::Turn(_)))
5453 .count();
5454 assert_eq!(turns, 2, "prior valid turns load; corrupt line skipped");
5455 assert!(matches!(trace.first(), Some(RunRecord::Started(_))));
5456 }
5457
5458 #[test]
5460 fn list_runs_empty_for_unknown_agent() {
5461 let tmp = tempfile::TempDir::new().unwrap();
5462 let s = store(tmp.path().join("runs"));
5463 assert!(s.list_runs("nobody").is_empty());
5464 assert!(s.get_run_trace("nope").is_none());
5465 assert!(s.agent_for_run("nope").is_none());
5466 }
5467
5468 #[test]
5471 fn from_journal_dir_roots_at_car_runs() {
5472 let s = RunStore::from_journal_dir(Path::new("/home/u/.car/journals"));
5473 assert_eq!(s.root(), Path::new("/home/u/.car/runs"));
5474 }
5475
5476 #[test]
5479 fn retention_config_reads_overrides() {
5480 let tmp = tempfile::TempDir::new().unwrap();
5481 std::fs::write(
5482 tmp.path().join("config.toml"),
5483 "[runs]\nmax_per_agent = 10\n",
5484 )
5485 .unwrap();
5486 let cfg = RetentionConfig::from_car_dir(tmp.path());
5487 assert_eq!(cfg.count_cap(), Some(10));
5488 assert_eq!(cfg.age_cap_days(), Some(DEFAULT_MAX_AGE_DAYS));
5490 }
5491
5492 #[test]
5495 fn retention_config_defaults_on_missing_file() {
5496 let tmp = tempfile::TempDir::new().unwrap();
5497 let cfg = RetentionConfig::from_car_dir(tmp.path());
5498 assert_eq!(cfg.count_cap(), Some(DEFAULT_MAX_RUNS_PER_AGENT));
5499 assert_eq!(cfg.age_cap_days(), Some(DEFAULT_MAX_AGE_DAYS));
5500 }
5501
5502 fn ended_at(
5506 run_id: &str,
5507 agent_id: &str,
5508 status: OutcomeStatus,
5509 when: DateTime<Utc>,
5510 ) -> RunRecord {
5511 let outcome = AgentOutcome {
5512 status,
5513 summary: "done".to_string(),
5514 evidence: vec![],
5515 metrics: OutcomeMetrics::default(),
5516 timestamp: when,
5517 };
5518 RunRecord::Ended(RunEnded {
5519 run_id: run_id.to_string(),
5520 client_id: Some("test-client".to_string()),
5521 agent_id: agent_id.to_string(),
5522 termination: RunTermination::Outcome { status, outcome },
5523 completion_digest: Some("test-digest".to_string()),
5524 ended_at: when,
5525 })
5526 }
5527
5528 #[test]
5532 fn gc_age_cap_uses_terminal_time_not_start() {
5533 let tmp = tempfile::TempDir::new().unwrap();
5534 let s = RunStore::new(tmp.path().join("runs"), RetentionConfig::new(50, 30));
5535 let started_40d = Utc::now() - chrono::Duration::days(40);
5537 let ended_1d = Utc::now() - chrono::Duration::days(1);
5538 s.write_started(&started("long", "agent-a", started_40d))
5539 .unwrap();
5540 s.append_records(
5541 "agent-a",
5542 "long",
5543 &[ended_at(
5544 "long",
5545 "agent-a",
5546 OutcomeStatus::Success,
5547 ended_1d,
5548 )],
5549 )
5550 .unwrap();
5551
5552 let removed = s.gc();
5553 assert_eq!(
5554 removed, 0,
5555 "a run completed 1 day ago must survive the 30-day age cap, \
5556 even if it started 40 days ago"
5557 );
5558 let remaining = s.list_runs("agent-a");
5559 assert_eq!(remaining.len(), 1);
5560 assert_eq!(remaining[0].run_id, "long");
5561 }
5562
5563 #[test]
5568 fn adopt_orphans_marks_crashed_inprogress_runs_incomplete() {
5569 let tmp = tempfile::TempDir::new().unwrap();
5570 let root = tmp.path().join("runs");
5571 let s1 = store(root.clone());
5573 s1.write_started(&started("orphan", "agent-a", Utc::now()))
5574 .unwrap();
5575 s1.append_turns("agent-a", "orphan", &[turn(0, "first")])
5576 .unwrap();
5577 assert_eq!(
5578 s1.list_runs("agent-a")[0].status,
5579 RunStatus::InProgress,
5580 "precondition: orphan reads InProgress before adoption"
5581 );
5582
5583 let s2 = store(root);
5585 let adopted = s2.adopt_orphans();
5586 assert_eq!(adopted, 1, "the crash orphan should be adopted");
5587
5588 let after = &s2.list_runs("agent-a")[0];
5589 assert_eq!(
5590 after.status,
5591 RunStatus::Incomplete,
5592 "adopted orphan now reads Incomplete (terminal)"
5593 );
5594 assert!(after.ended_at.is_some(), "terminal record has an ended_at");
5595
5596 assert_eq!(s2.adopt_orphans(), 0);
5598 }
5599
5600 fn write_truncated_private_file(path: &Path) {
5601 let parent = path.parent().unwrap();
5602 car_secrets::ensure_private_dir(parent).unwrap();
5603 let mut file = car_secrets::create_private_file(path).unwrap();
5604 file.write_all(b"{").unwrap();
5605 file.sync_all().unwrap();
5606 }
5607
5608 #[test]
5609 fn corrupt_execution_marker_quarantines_orphan_adoption() {
5610 let tmp = tempfile::TempDir::new().unwrap();
5611 let root = tmp.path().join("runs");
5612 let s = store(root);
5613 s.write_started(&started("marker-corrupt", "agent-a", Utc::now()))
5614 .unwrap();
5615 write_truncated_private_file(&s.proposal_execution_path("marker-corrupt"));
5616
5617 assert!(s.execution_marker("absent-marker").unwrap().is_none());
5618 assert!(s.execution_marker("marker-corrupt").is_err());
5619
5620 assert_eq!(
5621 s.adopt_orphans(),
5622 0,
5623 "present-but-invalid marker is outcome-unknown, never an adoptable absence"
5624 );
5625 assert_eq!(s.list_runs("agent-a")[0].status, RunStatus::InProgress);
5626 }
5627
5628 #[test]
5629 fn corrupt_finalization_outbox_quarantines_orphan_adoption() {
5630 let tmp = tempfile::TempDir::new().unwrap();
5631 let root = tmp.path().join("runs");
5632 let s = store(root);
5633 s.write_started(&started("outbox-corrupt", "agent-a", Utc::now()))
5634 .unwrap();
5635 write_truncated_private_file(&s.proposal_outbox_path("outbox-corrupt"));
5636
5637 assert!(s.pending_proposal("absent-outbox").unwrap().is_none());
5638 assert!(s.pending_proposal("outbox-corrupt").is_err());
5639
5640 assert_eq!(
5641 s.adopt_orphans(),
5642 0,
5643 "present-but-invalid outbox is outcome-unknown, never an adoptable absence"
5644 );
5645 assert_eq!(s.list_runs("agent-a")[0].status, RunStatus::InProgress);
5646 }
5647
5648 #[test]
5649 fn proposal_sidecars_reject_wrong_run_identity() {
5650 let tmp = tempfile::TempDir::new().unwrap();
5651 let s = store(tmp.path().join("runs"));
5652 let marker = ProposalExecutionMarker {
5653 run_id: "other-run".to_string(),
5654 client_id: "client".to_string(),
5655 requested_policy_session_id: None,
5656 policy_session_id: None,
5657 original_proposal_id: "proposal".to_string(),
5658 original_submission: json!({"id":"proposal","source":"test","actions":[]}),
5659 original_proposal: json!({"id":"proposal","source":"test","actions":[]}),
5660 proposal_digest: "a".repeat(64),
5661 };
5662 let path = s.proposal_execution_path("requested-run");
5663 car_secrets::ensure_private_dir(path.parent().unwrap()).unwrap();
5664 let mut file = car_secrets::create_private_file(&path).unwrap();
5665 serde_json::to_writer(&mut file, &marker).unwrap();
5666 file.sync_all().unwrap();
5667
5668 let error = s.execution_marker("requested-run").unwrap_err();
5669 assert_eq!(error.kind(), std::io::ErrorKind::InvalidData);
5670 assert!(error.to_string().contains("does not match requested run"));
5671
5672 let mut pending = valid_pending("other-outbox-run");
5673 pending.client_id = "other-client".to_string();
5674 let path = s.proposal_outbox_path("requested-outbox-run");
5675 car_secrets::ensure_private_dir(path.parent().unwrap()).unwrap();
5676 let mut file = car_secrets::create_private_file(&path).unwrap();
5677 serde_json::to_writer(&mut file, &pending).unwrap();
5678 file.sync_all().unwrap();
5679 let error = s.pending_proposal("requested-outbox-run").unwrap_err();
5680 assert_eq!(error.kind(), std::io::ErrorKind::InvalidData);
5681 assert!(error.to_string().contains("does not match requested run"));
5682 }
5683
5684 #[test]
5685 fn execution_marker_reader_rejects_semantically_invalid_preimage() {
5686 let tmp = tempfile::TempDir::new().unwrap();
5687 let s = store(tmp.path().join("runs"));
5688 let pending = valid_pending("invalid-marker");
5689 let marker = ProposalExecutionMarker {
5690 run_id: pending.run_id.clone(),
5691 client_id: pending.client_id.clone(),
5692 requested_policy_session_id: None,
5693 policy_session_id: None,
5694 original_proposal_id: pending.original_proposal_id.clone(),
5695 original_submission: pending.original_submission.clone(),
5696 original_proposal: serde_json::to_value(&pending.original_proposal).unwrap(),
5697 proposal_digest: "A".repeat(64),
5698 };
5699 let path = s.proposal_execution_path(&pending.run_id);
5700 car_secrets::ensure_private_dir(path.parent().unwrap()).unwrap();
5701 let mut file = car_secrets::create_private_file(&path).unwrap();
5702 serde_json::to_writer(&mut file, &marker).unwrap();
5703 file.sync_all().unwrap();
5704
5705 let error = s.execution_marker(&pending.run_id).unwrap_err();
5706 assert_eq!(error.kind(), std::io::ErrorKind::InvalidData);
5707 assert!(error.to_string().contains("does not bind"));
5708 }
5709
5710 #[test]
5711 fn execution_marker_cleanup_requires_the_full_identity_tuple() {
5712 let tmp = tempfile::TempDir::new().unwrap();
5713 let store = store(tmp.path().join("runs"));
5714 let pending = valid_pending("marker-cleanup-tuple");
5715 let marker = marker_for_pending(&pending);
5716 store.write_execution_marker(&marker).unwrap();
5717
5718 let mut mismatched = marker.clone();
5719 mismatched.requested_policy_session_id = Some("self-claimed-policy".to_string());
5720 assert!(store.clear_execution_marker(&mismatched).is_err());
5721 assert_eq!(
5722 store.execution_marker(&marker.run_id).unwrap(),
5723 Some(marker.clone()),
5724 "tuple mismatch must preserve the exact durable marker"
5725 );
5726
5727 store.clear_execution_marker(&marker).unwrap();
5728 assert!(store.execution_marker(&marker.run_id).unwrap().is_none());
5729 }
5730
5731 #[test]
5732 fn typed_finalization_rejects_invalid_result_and_lineage() {
5733 let tmp = tempfile::TempDir::new().unwrap();
5734 let s = store(tmp.path().join("runs"));
5735 let valid = valid_pending("typed-valid");
5736 valid.validate().unwrap();
5737 write_pending_provenance(&s, &valid);
5738 s.write_pending_proposal(&valid).unwrap();
5739 assert_eq!(s.pending_proposal("typed-valid").unwrap(), Some(valid));
5740
5741 let mut missing_final = valid_pending("typed-missing-final");
5742 missing_final.proposal_result.final_proposal = None;
5743 assert!(missing_final.validate().is_err());
5744
5745 let mut wrong_digest = valid_pending("typed-wrong-digest");
5746 wrong_digest.proposal_result.replan_lineage[0].proposal_digest = Some("A".repeat(64));
5747 assert!(wrong_digest.validate().is_err());
5748
5749 let mut wrong_final = valid_pending("typed-wrong-final");
5750 wrong_final.final_proposal_id = "another-final".to_string();
5751 assert!(wrong_final.validate().is_err());
5752
5753 let mut rejected_tail = valid_pending("typed-rejected-tail");
5754 rejected_tail
5755 .proposal_result
5756 .replan_lineage
5757 .push(ProposalLineageEntry {
5758 generation: 1,
5759 proposal_id: "rejected-candidate".to_string(),
5760 proposal_digest: Some("d".repeat(64)),
5761 status: ProposalLineageStatus::Rejected,
5762 rejection_reason: Some("candidate failed quality gate".to_string()),
5763 });
5764 let value = serde_json::to_value(&rejected_tail.proposal_result).unwrap();
5765 let canonical = car_inference::catalog_identity::canonical_json(&value).unwrap();
5766 rejected_tail.result_digest = format!("{:x}", Sha256::digest(canonical.as_bytes()));
5767 rejected_tail.validate().unwrap();
5768
5769 let mut rejected_without_reason = valid_pending("typed-rejected-no-reason");
5770 rejected_without_reason
5771 .proposal_result
5772 .replan_lineage
5773 .push(ProposalLineageEntry {
5774 generation: 1,
5775 proposal_id: "rejected-candidate".to_string(),
5776 proposal_digest: None,
5777 status: ProposalLineageStatus::Rejected,
5778 rejection_reason: None,
5779 });
5780 assert!(rejected_without_reason.validate().is_err());
5781
5782 let mut rejected_zero_then_accepted = valid_pending("typed-rejected-zero");
5783 rejected_zero_then_accepted.proposal_result.replan_lineage[0].status =
5784 ProposalLineageStatus::Rejected;
5785 rejected_zero_then_accepted.proposal_result.replan_lineage[0].rejection_reason =
5786 Some("original rejected".to_string());
5787 rejected_zero_then_accepted
5788 .proposal_result
5789 .replan_lineage
5790 .push(ProposalLineageEntry {
5791 generation: 1,
5792 proposal_id: rejected_zero_then_accepted.final_proposal_id.clone(),
5793 proposal_digest: Some(
5794 proposal_digest(&rejected_zero_then_accepted.final_proposal).unwrap(),
5795 ),
5796 status: ProposalLineageStatus::Accepted,
5797 rejection_reason: None,
5798 });
5799 rejected_zero_then_accepted
5800 .accepted_proposal_preimages
5801 .push(AcceptedProposalPreimage {
5802 generation: 1,
5803 proposal: rejected_zero_then_accepted.final_proposal.clone(),
5804 });
5805 refresh_pending_result_digest(&mut rejected_zero_then_accepted);
5806 assert!(rejected_zero_then_accepted.validate().is_err());
5807 }
5808
5809 #[test]
5810 fn active_pending_requires_exact_raw_submission_shape() {
5811 let valid = valid_pending("raw-valid");
5812 valid.validate().unwrap();
5813
5814 for raw in [
5815 Value::Null,
5816 json!([]),
5817 json!({"id": valid.original_proposal_id, "source": "run-store-test"}),
5818 json!({
5819 "id": valid.original_proposal_id,
5820 "source": "run-store-test",
5821 "actions": [{"id":"not-the-typed-action","type":"state_read"}]
5822 }),
5823 ] {
5824 let mut invalid = valid.clone();
5825 invalid.original_submission = raw;
5826 assert!(
5827 invalid.validate().is_err(),
5828 "active pending accepted invalid raw submission: {}",
5829 invalid.original_submission
5830 );
5831 }
5832
5833 let mut with_extra = valid;
5834 with_extra.original_submission["caller_extension"] = json!({"kept": true});
5835 with_extra.validate().unwrap();
5836 assert_eq!(
5837 with_extra.event_data()["original_submission"]["caller_extension"],
5838 json!({"kept": true})
5839 );
5840 }
5841
5842 #[test]
5843 fn fresh_pending_write_requires_exact_run_and_marker_provenance() {
5844 let tmp = tempfile::TempDir::new().unwrap();
5845 let store = store(tmp.path().join("runs"));
5846 let pending = valid_pending("fresh-provenance");
5847
5848 assert!(
5849 store.write_pending_proposal(&pending).is_err(),
5850 "pending without durable RunStarted and execution marker must be rejected"
5851 );
5852 write_pending_provenance(&store, &pending);
5853 store.write_pending_proposal(&pending).unwrap();
5854 assert_eq!(
5855 store.pending_proposal(&pending.run_id).unwrap(),
5856 Some(pending.clone())
5857 );
5858
5859 store.clear_pending_proposal(&pending).unwrap();
5860 let mut client_mismatch = pending.clone();
5861 client_mismatch.client_id = "self-claimed-client".to_string();
5862 let mut requested_policy_mismatch = pending.clone();
5863 requested_policy_mismatch.requested_policy_session_id =
5864 Some("self-claimed-policy".to_string());
5865 let mut run_mismatch = pending.clone();
5866 run_mismatch.run_id = "self-claimed-run".to_string();
5867 let mut original_mismatch = pending;
5868 rebind_pending_original(&mut original_mismatch);
5869 for (name, mismatched) in [
5870 ("client", client_mismatch),
5871 ("requested policy", requested_policy_mismatch),
5872 ("run", run_mismatch),
5873 ("original proposal", original_mismatch),
5874 ] {
5875 assert!(
5876 store.write_pending_proposal(&mismatched).is_err(),
5877 "pending {name} self-claim must not override durable run/marker provenance"
5878 );
5879 }
5880
5881 let mut authenticated = valid_pending("fresh-auth-provenance");
5882 authenticated.requested_policy_session_id = Some("live-policy".to_string());
5883 authenticated.policy_session_id = Some("live-policy".to_string());
5884 write_pending_provenance(&store, &authenticated);
5885 authenticated.policy_session_id = None;
5886 assert!(
5887 store.write_pending_proposal(&authenticated).is_err(),
5888 "pending authenticated policy self-claim must match the durable marker"
5889 );
5890 }
5891
5892 #[test]
5893 fn outbox_reader_requires_active_v3_typed_result_fields() {
5894 let tmp = tempfile::TempDir::new().unwrap();
5895 let s = store(tmp.path().join("runs"));
5896 let pending = valid_pending("strict-result");
5897 let mut value = serde_json::to_value(&pending).unwrap();
5898 value["proposal_result"]
5899 .as_object_mut()
5900 .unwrap()
5901 .remove("cost");
5902 let path = s.proposal_outbox_path("strict-result");
5903 car_secrets::ensure_private_dir(path.parent().unwrap()).unwrap();
5904 let mut file = car_secrets::create_private_file(&path).unwrap();
5905 serde_json::to_writer(&mut file, &value).unwrap();
5906 file.sync_all().unwrap();
5907
5908 let error = s.pending_proposal("strict-result").unwrap_err();
5909 assert_eq!(error.kind(), std::io::ErrorKind::InvalidData);
5910 assert!(error.to_string().contains("missing `cost`"));
5911 }
5912
5913 #[test]
5916 fn adopt_orphans_leaves_completed_runs_alone() {
5917 let tmp = tempfile::TempDir::new().unwrap();
5918 let root = tmp.path().join("runs");
5919 let s = store(root);
5920 s.write_started(&started("done", "agent-a", Utc::now()))
5921 .unwrap();
5922 s.append_records(
5923 "agent-a",
5924 "done",
5925 &[ended("done", "agent-a", OutcomeStatus::Success)],
5926 )
5927 .unwrap();
5928 assert_eq!(s.adopt_orphans(), 0);
5929 assert_eq!(s.list_runs("agent-a")[0].status, RunStatus::Completed);
5930 }
5931
5932 #[test]
5936 fn appending_after_torn_tail_quarantines_trace() {
5937 let tmp = tempfile::TempDir::new().unwrap();
5938 let root = tmp.path().join("runs");
5939 let s = store(root.clone());
5940 s.write_started(&started("run-1", "agent-a", Utc::now()))
5941 .unwrap();
5942 s.append_turns("agent-a", "run-1", &[turn(0, "first")])
5943 .unwrap();
5944
5945 let path = root.join("agent-a").join("run-1.jsonl");
5947 {
5948 let mut f = std::fs::OpenOptions::new()
5949 .append(true)
5950 .open(&path)
5951 .unwrap();
5952 f.write_all(b"{\"record\":\"turn\",\"index\":1,\"prom")
5954 .unwrap();
5955 }
5956 let mut opened = car_secrets::open_private_append(&path).unwrap();
5957 assert!(
5958 last_byte_is_not_newline(&mut opened).unwrap(),
5959 "precondition: tail is torn (no trailing newline)"
5960 );
5961 drop(opened);
5962
5963 let append_error = s
5967 .append_turns("agent-a", "run-1", &[turn(2, "third")])
5968 .expect_err("committed middle corruption must fail the append receipt");
5969 assert!(
5970 append_error.to_string().contains("line 3"),
5971 "{append_error}"
5972 );
5973
5974 let error = s.get_run_trace_checked("run-1").unwrap_err();
5975 assert!(error.to_string().contains("line 3"), "{error}");
5976 let summary = &s.list_runs("agent-a")[0];
5977 assert_eq!(summary.status, RunStatus::Incomplete);
5978 assert_eq!(summary.trace_corruption.as_ref().unwrap().line, 3);
5979 }
5980
5981 #[test]
5982 fn durable_boundaries_fail_closed_and_retry_without_duplicate_rows() {
5983 for point in [
5984 RunStoreFailurePoint::Write,
5985 RunStoreFailurePoint::Flush,
5986 RunStoreFailurePoint::Fsync,
5987 ] {
5988 let tmp = tempfile::TempDir::new().unwrap();
5989 let root = tmp.path().join("runs");
5990 let failures = RunStoreFailureInjector::default();
5991 let store = RunStore::new(root.clone(), RetentionConfig::default())
5992 .with_failure_injector(failures.clone());
5993 let started = started("durable", "agent-a", Utc::now());
5994
5995 failures.fail_next(point);
5996 assert!(
5997 store.write_started(&started).is_err(),
5998 "{point:?} start must not acknowledge"
5999 );
6000 store
6001 .write_started(&started)
6002 .expect("exact start retry reaches durability");
6003
6004 let ended = match ended("durable", "agent-a", OutcomeStatus::Success) {
6005 RunRecord::Ended(ended) => ended,
6006 _ => unreachable!(),
6007 };
6008 failures.fail_next(point);
6009 assert!(
6010 store.write_ended(&ended).is_err(),
6011 "{point:?} terminal must not acknowledge"
6012 );
6013 store
6014 .write_ended(&ended)
6015 .expect("exact terminal retry reaches durability");
6016
6017 let restarted = RunStore::new(root, RetentionConfig::default());
6018 let trace = restarted.get_run_trace("durable").unwrap();
6019 assert_eq!(
6020 trace
6021 .iter()
6022 .filter(|row| matches!(row, RunRecord::Started(_)))
6023 .count(),
6024 1,
6025 "{point:?} retry duplicated RunStarted"
6026 );
6027 assert_eq!(
6028 trace
6029 .iter()
6030 .filter(|row| matches!(row, RunRecord::Ended(_)))
6031 .count(),
6032 1,
6033 "{point:?} retry duplicated RunEnded"
6034 );
6035 }
6036 }
6037
6038 #[test]
6039 fn retained_completed_proposal_survives_run_gc_and_restart() {
6040 let tmp = tempfile::TempDir::new().unwrap();
6041 let root = tmp.path().join("runs");
6042 let retention = RetentionConfig::new(DEFAULT_MAX_RUNS_PER_AGENT, DEFAULT_MAX_AGE_DAYS);
6048 let long_ago = Utc::now() - chrono::Duration::days(DEFAULT_MAX_AGE_DAYS + 10);
6049 let store = RunStore::new(root.clone(), retention);
6050 let pending = valid_pending("completed-after-gc");
6051 write_pending_provenance(&store, &pending);
6052 store.write_pending_proposal(&pending).unwrap();
6053 let receipt = store.write_completed_proposal(&pending).unwrap();
6054 store.cleanup_completed_proposal_guards(&receipt).unwrap();
6055 let mut ended = match ended(&pending.run_id, "agent-a", OutcomeStatus::Success) {
6056 RunRecord::Ended(ended) => ended,
6057 _ => unreachable!(),
6058 };
6059 ended.ended_at = long_ago;
6063 store.write_ended(&ended).unwrap();
6064
6065 assert_eq!(store.gc(), 1, "the completed run trace must be evicted");
6066 assert!(store.get_run_trace(&pending.run_id).is_none());
6067 assert_eq!(
6068 store
6069 .completed_proposal(
6070 &pending.run_id,
6071 &pending.client_id,
6072 pending.requested_policy_session_id.as_deref(),
6073 &pending.original_submission,
6074 )
6075 .unwrap(),
6076 Some(receipt.clone()),
6077 "receipt validity must not depend on a GC-eligible RunStarted"
6078 );
6079
6080 let restarted = RunStore::new(root, retention);
6081 assert_eq!(
6082 restarted
6083 .completed_proposal_retry_owner(
6084 pending.requested_policy_session_id.as_deref(),
6085 &pending.original_submission,
6086 )
6087 .unwrap(),
6088 Some((pending.run_id.clone(), pending.client_id.clone())),
6089 "the retry tuple owner must remain recoverable after restart"
6090 );
6091 assert_eq!(restarted.all_completed_proposals().unwrap(), vec![receipt]);
6092 }
6093
6094 #[test]
6095 fn resumed_completed_proposal_lookup_requires_one_exact_receipt() {
6096 let tmp = tempfile::TempDir::new().unwrap();
6097 let store = store(tmp.path().join("runs"));
6098 let mut first = valid_pending("resumed-policy-rotation");
6099 first.requested_policy_session_id = Some("original-policy-one".to_string());
6100 first.policy_session_id = first.requested_policy_session_id.clone();
6101 write_pending_provenance(&store, &first);
6102 store.write_pending_proposal(&first).unwrap();
6103 let first_receipt = store.write_completed_proposal(&first).unwrap();
6104 store
6105 .cleanup_completed_proposal_guards(&first_receipt)
6106 .unwrap();
6107
6108 assert_eq!(
6109 store
6110 .completed_proposal_for_resumed_owner(
6111 &first.run_id,
6112 &first.client_id,
6113 &first.original_submission,
6114 )
6115 .unwrap(),
6116 Some(first_receipt),
6117 "a single exact receipt is recoverable without the closed policy-session id"
6118 );
6119 assert!(
6120 store
6121 .completed_proposal_for_resumed_owner(
6122 &first.run_id,
6123 &first.client_id,
6124 &json!({"id":"different"}),
6125 )
6126 .unwrap()
6127 .is_none(),
6128 "a different proposal must not match"
6129 );
6130
6131 let mut second = first.clone();
6132 second.requested_policy_session_id = Some("original-policy-two".to_string());
6133 second.policy_session_id = second.requested_policy_session_id.clone();
6134 store
6135 .write_execution_marker(&marker_for_pending(&second))
6136 .unwrap();
6137 store.write_pending_proposal(&second).unwrap();
6138 let second_receipt = store.write_completed_proposal(&second).unwrap();
6139 store
6140 .cleanup_completed_proposal_guards(&second_receipt)
6141 .unwrap();
6142
6143 let error = store
6144 .completed_proposal_for_resumed_owner(
6145 &first.run_id,
6146 &first.client_id,
6147 &first.original_submission,
6148 )
6149 .unwrap_err();
6150 assert_eq!(error.kind(), std::io::ErrorKind::InvalidData);
6151 assert!(error.to_string().contains("ambiguous"));
6152 }
6153
6154 #[test]
6155 fn resumed_completed_proposal_lookup_enforces_receipt_count_before_deserialization() {
6156 let tmp = tempfile::TempDir::new().unwrap();
6157 let store = store(tmp.path().join("runs"));
6158 let target = valid_pending("resumed-receipt-count-limit");
6159 let target_receipt = write_completed_receipt_fixture(&store, target.clone(), None, false);
6160 for index in 1..MAX_RESUMED_PROPOSAL_RECEIPTS_PER_RUN {
6161 let mut unrelated = target.clone();
6162 unrelated.client_id = format!("unrelated-client-{index}");
6163 write_completed_receipt_fixture(&store, unrelated, None, false);
6164 }
6165
6166 assert_eq!(
6167 store
6168 .completed_proposal_for_resumed_owner(
6169 &target.run_id,
6170 &target.client_id,
6171 &target.original_submission,
6172 )
6173 .unwrap(),
6174 Some(target_receipt),
6175 "the exact receipt-count limit remains recoverable"
6176 );
6177
6178 let corrupt_path = store
6179 .completed_response_run_root(&target.run_id)
6180 .join("count-plus-one-is-never-deserialized.json");
6181 let mut corrupt = store.create_private_file(&corrupt_path).unwrap();
6182 corrupt.write_all(b"not-json").unwrap();
6183 corrupt.sync_all().unwrap();
6184 let error = store
6185 .completed_proposal_for_resumed_owner(
6186 &target.run_id,
6187 &target.client_id,
6188 &target.original_submission,
6189 )
6190 .unwrap_err();
6191 assert_eq!(error.kind(), std::io::ErrorKind::InvalidData);
6192 assert!(error.to_string().contains("receipt count limit"));
6193 }
6194
6195 #[test]
6196 fn resumed_completed_proposal_lookup_enforces_byte_limit_before_deserialization() {
6197 let at_limit = tempfile::TempDir::new().unwrap();
6198 let at_limit_store = store(at_limit.path().join("runs"));
6199 let target = valid_pending("resumed-receipt-byte-limit");
6200 let first_receipt_bytes = MAX_RESUMED_PROPOSAL_RECEIPT_BYTES_PER_RUN as usize / 2;
6201 let target_receipt = write_completed_receipt_fixture(
6202 &at_limit_store,
6203 target.clone(),
6204 Some(first_receipt_bytes),
6205 false,
6206 );
6207 let mut unrelated = target.clone();
6208 unrelated.client_id = "unrelated-byte-limit-client".to_string();
6209 write_completed_receipt_fixture(
6210 &at_limit_store,
6211 unrelated,
6212 Some(MAX_RESUMED_PROPOSAL_RECEIPT_BYTES_PER_RUN as usize - first_receipt_bytes),
6213 false,
6214 );
6215 assert_eq!(
6216 at_limit_store
6217 .completed_proposal_for_resumed_owner(
6218 &target.run_id,
6219 &target.client_id,
6220 &target.original_submission,
6221 )
6222 .unwrap(),
6223 Some(target_receipt),
6224 "the exact cumulative-byte limit remains recoverable"
6225 );
6226
6227 let cumulative_over_limit = tempfile::TempDir::new().unwrap();
6228 let cumulative_over_limit_store = store(cumulative_over_limit.path().join("runs"));
6229 write_completed_receipt_fixture(
6230 &cumulative_over_limit_store,
6231 target.clone(),
6232 Some(first_receipt_bytes),
6233 false,
6234 );
6235 let mut unrelated = target.clone();
6236 unrelated.client_id = "unrelated-byte-over-limit-client".to_string();
6237 write_completed_receipt_fixture(
6238 &cumulative_over_limit_store,
6239 unrelated,
6240 Some(MAX_RESUMED_PROPOSAL_RECEIPT_BYTES_PER_RUN as usize - first_receipt_bytes + 1),
6241 false,
6242 );
6243 let error = cumulative_over_limit_store
6244 .completed_proposal_for_resumed_owner(
6245 &target.run_id,
6246 &target.client_id,
6247 &target.original_submission,
6248 )
6249 .unwrap_err();
6250 assert_eq!(error.kind(), std::io::ErrorKind::InvalidData);
6251 assert!(error.to_string().contains("receipt byte limit"));
6252
6253 let invalid_over_limit = tempfile::TempDir::new().unwrap();
6254 let invalid_over_limit_store = store(invalid_over_limit.path().join("runs"));
6255 write_completed_receipt_fixture(
6256 &invalid_over_limit_store,
6257 target.clone(),
6258 Some(MAX_RESUMED_PROPOSAL_RECEIPT_BYTES_PER_RUN as usize),
6259 true,
6260 );
6261 let error = invalid_over_limit_store
6262 .completed_proposal_for_resumed_owner(
6263 &target.run_id,
6264 &target.client_id,
6265 &target.original_submission,
6266 )
6267 .unwrap_err();
6268 assert_eq!(error.kind(), std::io::ErrorKind::InvalidData);
6269 assert!(
6270 error.to_string().contains("receipt byte limit"),
6271 "the byte ceiling must reject an oversized corrupt receipt before deserialization"
6272 );
6273 }
6274
6275 #[test]
6276 fn resumed_completed_proposal_lookup_rejects_corrupt_receipt_under_limits() {
6277 let tmp = tempfile::TempDir::new().unwrap();
6278 let store = store(tmp.path().join("runs"));
6279 let target = valid_pending("resumed-corrupt-receipt");
6280 let run_root = store.completed_response_run_root(&target.run_id);
6281 store.ensure_private_dir(&run_root).unwrap();
6282 let path = store
6283 .completed_response_path(
6284 &target.run_id,
6285 &target.client_id,
6286 target.requested_policy_session_id.as_deref(),
6287 &target.original_submission,
6288 )
6289 .unwrap();
6290 let mut corrupt = store.create_private_file(&path).unwrap();
6291 corrupt.write_all(b"not-json").unwrap();
6292 corrupt.sync_all().unwrap();
6293
6294 let error = store
6295 .completed_proposal_for_resumed_owner(
6296 &target.run_id,
6297 &target.client_id,
6298 &target.original_submission,
6299 )
6300 .unwrap_err();
6301 assert_eq!(error.kind(), std::io::ErrorKind::InvalidData);
6302 assert!(!error.to_string().contains("receipt count limit"));
6303 assert!(!error.to_string().contains("receipt byte limit"));
6304 }
6305
6306 #[test]
6307 fn completed_proposal_owner_lookup_does_not_scan_unrelated_receipts() {
6308 let tmp = tempfile::TempDir::new().unwrap();
6309 let store = store(tmp.path().join("runs"));
6310 let unrelated_dir = store.completed_response_root().join("unrelated-run");
6311 car_secrets::ensure_private_dir(&unrelated_dir).unwrap();
6312 let path = unrelated_dir.join("corrupt.json");
6313 let mut file = car_secrets::create_private_file(&path).unwrap();
6314 file.write_all(b"not-json").unwrap();
6315 file.sync_all().unwrap();
6316
6317 assert_eq!(
6318 store
6319 .completed_proposal_retry_owner(None, &json!({"id": "absent"}))
6320 .unwrap(),
6321 None,
6322 "a bounded content-addressed miss must not enumerate unrelated receipts"
6323 );
6324 }
6325
6326 #[test]
6327 fn proposal_retry_reservation_is_first_writer_wins_and_exact_owner_idempotent() {
6328 let tmp = tempfile::TempDir::new().unwrap();
6329 let store = store(tmp.path().join("runs"));
6330 let submission = json!({"id":"reserved","source":"test","actions":[]});
6331 assert_eq!(
6332 store
6333 .reserve_proposal_retry_owner("first-run", "first-client", None, &submission)
6334 .unwrap(),
6335 ProposalRetryReservation::Acquired
6336 );
6337 for (run_id, client_id) in [
6338 ("first-run", "first-client"),
6339 ("second-run", "second-client"),
6340 ] {
6341 assert_eq!(
6342 store
6343 .reserve_proposal_retry_owner(run_id, client_id, None, &submission)
6344 .unwrap(),
6345 ProposalRetryReservation::Existing {
6346 run_id: "first-run".to_string(),
6347 client_id: "first-client".to_string(),
6348 }
6349 );
6350 }
6351 assert_eq!(
6352 store
6353 .completed_proposal_retry_owner(None, &submission)
6354 .unwrap(),
6355 Some(("first-run".to_string(), "first-client".to_string()))
6356 );
6357 }
6358
6359 #[test]
6360 fn partial_proposal_retry_reservation_fails_closed_without_replacement() {
6361 let tmp = tempfile::TempDir::new().unwrap();
6362 let store = store(tmp.path().join("runs"));
6363 let submission = json!({"id":"partial","source":"test","actions":[]});
6364 let root = store.completed_response_index_root();
6365 car_secrets::ensure_private_dir(&root).unwrap();
6366 let path = store
6367 .completed_response_owner_path(None, &submission)
6368 .unwrap();
6369 let mut file = car_secrets::create_private_file(&path).unwrap();
6370 file.write_all(b"{").unwrap();
6371 file.sync_all().unwrap();
6372
6373 let error = store
6374 .reserve_proposal_retry_owner("partial-run", "partial-client", None, &submission)
6375 .unwrap_err();
6376 assert_eq!(error.kind(), std::io::ErrorKind::InvalidData);
6377 assert_eq!(std::fs::read(&path).unwrap(), b"{");
6378 }
6379
6380 #[test]
6381 fn completed_proposal_owner_claim_rejects_a_different_run_for_the_same_retry_tuple() {
6382 let tmp = tempfile::TempDir::new().unwrap();
6383 let store = store(tmp.path().join("runs"));
6384 let first = valid_pending("owner-first-run");
6385 write_pending_provenance(&store, &first);
6386 store.write_pending_proposal(&first).unwrap();
6387 store.write_completed_proposal(&first).unwrap();
6388
6389 let mut conflicting = first.clone();
6390 conflicting.run_id = "owner-conflicting-run".to_string();
6391 write_pending_provenance(&store, &conflicting);
6392 store.write_pending_proposal(&conflicting).unwrap();
6393 let error = store.write_completed_proposal(&conflicting).unwrap_err();
6394 assert_eq!(error.kind(), std::io::ErrorKind::AlreadyExists);
6395 assert!(error.to_string().contains("different completed response"));
6396 assert_eq!(
6397 store
6398 .completed_proposal_retry_owner(
6399 first.requested_policy_session_id.as_deref(),
6400 &first.original_submission,
6401 )
6402 .unwrap(),
6403 Some((first.run_id.clone(), first.client_id.clone())),
6404 "a conflicting writer must not replace the first durable owner"
6405 );
6406 assert!(
6407 store
6408 .completed_proposal(
6409 &conflicting.run_id,
6410 &conflicting.client_id,
6411 conflicting.requested_policy_session_id.as_deref(),
6412 &conflicting.original_submission,
6413 )
6414 .unwrap()
6415 .is_none(),
6416 "the rejected owner must not publish a receipt"
6417 );
6418 }
6419
6420 #[test]
6421 fn startup_enumeration_backfills_a_legacy_completed_proposal_owner() {
6422 let tmp = tempfile::TempDir::new().unwrap();
6423 let root = tmp.path().join("runs");
6424 let initial = store(root.clone());
6425 let pending = valid_pending("legacy-owner-backfill");
6426 write_pending_provenance(&initial, &pending);
6427 initial.write_pending_proposal(&pending).unwrap();
6428 let receipt = initial.write_completed_proposal(&pending).unwrap();
6429 let owner_path = initial
6430 .completed_response_owner_path(
6431 pending.requested_policy_session_id.as_deref(),
6432 &pending.original_submission,
6433 )
6434 .unwrap();
6435 std::fs::remove_file(&owner_path).unwrap();
6436 sync_directory(owner_path.parent().unwrap()).unwrap();
6437
6438 let restarted = store(root);
6439 assert!(
6440 restarted
6441 .completed_proposal_retry_owner(
6442 pending.requested_policy_session_id.as_deref(),
6443 &pending.original_submission,
6444 )
6445 .unwrap()
6446 .is_none(),
6447 "pre-index receipts begin without an ownership claim"
6448 );
6449 assert_eq!(restarted.all_completed_proposals().unwrap(), vec![receipt]);
6450 assert_eq!(
6451 restarted
6452 .completed_proposal_retry_owner(
6453 pending.requested_policy_session_id.as_deref(),
6454 &pending.original_submission,
6455 )
6456 .unwrap(),
6457 Some((pending.run_id, pending.client_id)),
6458 "startup receipt enumeration must durably backfill the owner claim"
6459 );
6460 }
6461}