1use std::collections::HashMap;
2use std::fmt;
3use std::fs;
4use std::path::{Path, PathBuf};
5use std::process::{Command, Stdio};
6use std::sync::atomic::{AtomicU64, Ordering};
7use std::time::{SystemTime, UNIX_EPOCH};
8
9static ID_COUNTER: AtomicU64 = AtomicU64::new(1);
10
11pub type State = HashMap<String, Value>;
12
13pub const MODEL_EVIDENCE_SCHEMA_VERSION: &str = "agentledger.model.evidence.v1";
14
15#[derive(Clone, Debug, PartialEq)]
16pub enum Value {
17 Null,
18 Bool(bool),
19 Number(f64),
20 String(String),
21 Object(State),
22 Array(Vec<Value>),
23}
24
25impl Default for Value {
26 fn default() -> Self {
27 Value::Null
28 }
29}
30
31impl From<&str> for Value {
32 fn from(value: &str) -> Self {
33 Value::String(value.to_string())
34 }
35}
36
37impl From<String> for Value {
38 fn from(value: String) -> Self {
39 Value::String(value)
40 }
41}
42
43impl From<bool> for Value {
44 fn from(value: bool) -> Self {
45 Value::Bool(value)
46 }
47}
48
49impl From<i64> for Value {
50 fn from(value: i64) -> Self {
51 Value::Number(value as f64)
52 }
53}
54
55impl From<f64> for Value {
56 fn from(value: f64) -> Self {
57 Value::Number(value)
58 }
59}
60
61#[derive(Clone, Debug)]
62pub struct Run {
63 pub run_id: String,
64 pub session_id: String,
65 pub status: String,
66 pub state: State,
67 pub state_version: u64,
68 pub created_at: f64,
69 pub updated_at: f64,
70}
71
72#[derive(Clone, Debug)]
73pub struct Step {
74 pub step_id: String,
75 pub run_id: String,
76 pub session_id: String,
77 pub status: String,
78 pub owner: Option<String>,
79 pub lease_token: Option<String>,
80 pub lease_until: Option<f64>,
81 pub attempt: u64,
82 pub state_version: u64,
83 pub checkpoint_id: Option<String>,
84 pub last_error_type: Option<String>,
85 pub last_error: Option<String>,
86 pub cancelled_at: Option<f64>,
87 pub created_at: f64,
88 pub updated_at: f64,
89}
90
91#[derive(Clone, Debug)]
92pub struct StepClaim {
93 pub run_id: String,
94 pub session_id: String,
95 pub step_id: String,
96 pub attempt: u64,
97 pub lease_token: String,
98 pub state_version: u64,
99 pub lease_until: f64,
100}
101
102#[derive(Clone, Debug)]
103pub struct Event {
104 pub event_id: String,
105 pub run_id: String,
106 pub session_id: Option<String>,
107 pub step_id: Option<String>,
108 pub seq: u64,
109 pub event_type: String,
110 pub timestamp: f64,
111 pub agent_role: Option<String>,
112 pub state_version: Option<u64>,
113 pub causal_token: Option<String>,
114 pub payload_hash: String,
115 pub payload_ref: String,
116 pub payload: State,
117}
118
119#[derive(Clone, Debug)]
120pub struct ToolLedgerEntry {
121 pub ledger_id: String,
122 pub run_id: String,
123 pub session_id: String,
124 pub step_id: String,
125 pub tool_name: String,
126 pub tool_version: String,
127 pub tool_call_id: String,
128 pub idempotency_key: String,
129 pub causal_token: String,
130 pub request_hash: String,
131 pub request_ref: String,
132 pub status: String,
133 pub external_id: Option<String>,
134 pub response_hash: Option<String>,
135 pub response_ref: Option<String>,
136 pub error_type: Option<String>,
137 pub response: Option<Value>,
138 pub created_at: f64,
139 pub updated_at: f64,
140}
141
142#[derive(Clone, Debug)]
143pub struct ApprovalRequest {
144 pub approval_id: String,
145 pub approval_key: String,
146 pub run_id: String,
147 pub session_id: String,
148 pub step_id: String,
149 pub tool_name: String,
150 pub risk_level: String,
151 pub status: String,
152 pub reason: String,
153 pub request_hash: String,
154 pub request_ref: String,
155 pub requested_by: String,
156 pub approved_by: Option<String>,
157 pub decision_reason: Option<String>,
158 pub created_at: f64,
159 pub updated_at: f64,
160}
161
162#[derive(Clone, Debug)]
163pub struct CostRecord {
164 pub cost_id: String,
165 pub run_id: String,
166 pub session_id: String,
167 pub step_id: String,
168 pub category: String,
169 pub name: String,
170 pub amount: f64,
171 pub unit: String,
172 pub metadata: State,
173 pub created_at: f64,
174}
175
176#[derive(Clone, Debug, Default, PartialEq)]
177pub struct CostSummary {
178 pub tool_calls: f64,
179 pub model_tokens: f64,
180 pub total_usd: f64,
181 pub by_category: HashMap<String, f64>,
182}
183
184pub const MEDIA_SCHEMA_VERSION: &str = "agentledger.media.v0";
185pub const STREAM_SCHEMA_VERSION: &str = "agentledger.stream.v0";
186
187#[derive(Clone, Debug)]
188pub struct LocalBlobStore {
189 root: PathBuf,
190}
191
192impl LocalBlobStore {
193 pub fn open(root: impl AsRef<Path>) -> Result<Self> {
194 fs::create_dir_all(root.as_ref()).map_err(|err| RuntimeError(err.to_string()))?;
195 Ok(Self {
196 root: root.as_ref().to_path_buf(),
197 })
198 }
199
200 pub fn put_json(&self, value: &Value) -> Result<(String, String)> {
201 let encoded = encode_value(value);
202 let digest = stable_hash(&encoded);
203 let dir = self.root.join("sha256");
204 fs::create_dir_all(&dir).map_err(|err| RuntimeError(err.to_string()))?;
205 let path = dir.join(format!("{digest}.json"));
206 if !path.exists() {
207 let tmp = path.with_extension("json.tmp");
208 fs::write(&tmp, encoded).map_err(|err| RuntimeError(err.to_string()))?;
209 fs::rename(&tmp, &path).map_err(|err| RuntimeError(err.to_string()))?;
210 }
211 Ok((
212 format!("sha256:{digest}"),
213 format!("blob://sha256/{digest}.json"),
214 ))
215 }
216
217 pub fn get_json(&self, reference: &str) -> Result<Value> {
218 let prefix = "blob://sha256/";
219 if !reference.starts_with(prefix) {
220 return Err(RuntimeError(format!("unsupported blob ref: {reference}")));
221 }
222 let name = &reference[prefix.len()..];
223 if !name.ends_with(".json")
224 || name.contains("..")
225 || name.contains('/')
226 || name.contains('\\')
227 {
228 return Err(RuntimeError(format!("unsupported blob ref: {reference}")));
229 }
230 let body = fs::read_to_string(self.root.join("sha256").join(name))
231 .map_err(|err| RuntimeError(err.to_string()))?;
232 decode_value(&body)
233 }
234}
235
236#[derive(Clone, Debug)]
237pub struct Artifact {
238 pub artifact_id: String,
239 pub run_id: String,
240 pub step_id: Option<String>,
241 pub name: String,
242 pub blob_hash: String,
243 pub blob_ref: String,
244 pub metadata: State,
245 pub created_at: f64,
246}
247
248#[derive(Clone, Debug, Default)]
249pub struct MediaArtifactOptions {
250 pub uri: Option<String>,
251 pub content_ref: Option<String>,
252 pub media_metadata: State,
253 pub lineage: State,
254 pub derived_outputs: State,
255 pub metadata: State,
256}
257
258#[derive(Clone, Debug, Default)]
259pub struct StreamChunkRef {
260 pub stream_id: String,
261 pub chunk_id: String,
262 pub offset: Value,
263 pub content_ref: Option<String>,
264 pub content_hash: Option<String>,
265 pub sequence: Option<f64>,
266 pub event_time: Option<f64>,
267 pub metadata: State,
268}
269
270#[derive(Clone, Debug, Default)]
271pub struct StreamCheckpointOptions {
272 pub stream_id: String,
273 pub consumer_id: String,
274 pub offset: Value,
275 pub watermark: Option<Value>,
276 pub chunk: Option<StreamChunkRef>,
277 pub partial_result_ref: Option<String>,
278 pub backpressure: State,
279 pub metadata: State,
280}
281
282#[derive(Clone, Debug, Default)]
283pub struct BudgetLimits {
284 pub max_tool_calls: Option<f64>,
285 pub max_model_tokens: Option<f64>,
286 pub max_total_usd: Option<f64>,
287}
288
289#[derive(Debug, Clone)]
290pub struct RuntimeError(pub String);
291
292impl fmt::Display for RuntimeError {
293 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
294 write!(f, "{}", self.0)
295 }
296}
297
298impl std::error::Error for RuntimeError {}
299
300pub type Result<T> = std::result::Result<T, RuntimeError>;
301pub type ToolFunc = Box<dyn Fn(State) -> Result<Value> + Send + Sync>;
302pub type AgentFunc = fn(&mut AgentContext, State) -> Result<()>;
303
304pub struct SandboxPolicy {
305 pub tool_name: String,
306 pub run_id: String,
307 pub step_id: String,
308 pub executor: String,
309 pub network: String,
310 pub filesystem: String,
311 pub timeout_seconds: u64,
312 pub extra: State,
313}
314
315pub struct SandboxResult {
316 pub ok: bool,
317 pub output: Value,
318 pub error: Option<String>,
319 pub metadata: State,
320}
321
322pub trait SandboxExecutor {
323 fn run_tool(&self, args: State, policy: &SandboxPolicy) -> SandboxResult;
324}
325
326pub struct DisabledSandboxExecutor;
327
328impl SandboxExecutor for DisabledSandboxExecutor {
329 fn run_tool(&self, _args: State, policy: &SandboxPolicy) -> SandboxResult {
330 let mut metadata = State::new();
331 metadata.insert("executor".to_string(), Value::String(policy.executor.clone()));
332 metadata.insert("isolation_level".to_string(), Value::String("none".to_string()));
333 metadata.insert("fail_closed".to_string(), Value::Bool(true));
334 SandboxResult {
335 ok: false,
336 output: Value::Null,
337 error: Some(format!("sandbox executor \"{}\" is disabled", policy.executor)),
338 metadata,
339 }
340 }
341}
342
343pub struct ToolSpec {
344 pub name: String,
345 pub version: String,
346 pub side_effect: String,
347 pub risk_level: String,
348 pub idempotency_required: bool,
349 pub approval_required: bool,
350 pub sandbox_required: bool,
351 pub sandbox_executor: String,
352 pub sandbox_policy: State,
353 pub input_schema: Option<Value>,
354 pub output_schema: Option<Value>,
355 pub func: ToolFunc,
356}
357
358impl ToolSpec {
359 pub fn new(name: &str, func: ToolFunc) -> Self {
360 Self {
361 name: name.to_string(),
362 version: "v1".to_string(),
363 side_effect: "none".to_string(),
364 risk_level: "low".to_string(),
365 idempotency_required: false,
366 approval_required: false,
367 sandbox_required: false,
368 sandbox_executor: String::new(),
369 sandbox_policy: State::new(),
370 input_schema: None,
371 output_schema: None,
372 func,
373 }
374 }
375
376 pub fn side_effect(mut self, side_effect: &str) -> Self {
377 self.side_effect = side_effect.to_string();
378 self
379 }
380
381 pub fn risk_level(mut self, risk_level: &str) -> Self {
382 self.risk_level = risk_level.to_string();
383 self
384 }
385
386 pub fn idempotency_required(mut self, required: bool) -> Self {
387 self.idempotency_required = required;
388 self
389 }
390
391 pub fn approval_required(mut self, required: bool) -> Self {
392 self.approval_required = required;
393 self
394 }
395
396 pub fn sandbox_required(mut self, required: bool) -> Self {
397 self.sandbox_required = required;
398 self
399 }
400
401 pub fn sandbox_executor(mut self, executor: &str) -> Self {
402 self.sandbox_executor = executor.to_string();
403 self
404 }
405
406 pub fn sandbox_policy(mut self, policy: State) -> Self {
407 self.sandbox_policy = policy;
408 self
409 }
410
411 pub fn input_schema(mut self, schema: Value) -> Self {
412 self.input_schema = Some(schema);
413 self
414 }
415
416 pub fn output_schema(mut self, schema: Value) -> Self {
417 self.output_schema = Some(schema);
418 self
419 }
420}
421
422#[derive(Default)]
423pub struct MemoryStore {
424 runs: HashMap<String, Run>,
425 steps: HashMap<String, Step>,
426 events: HashMap<String, Vec<Event>>,
427 tool_ledger: HashMap<String, ToolLedgerEntry>,
428 approval_requests: HashMap<String, ApprovalRequest>,
429 cost_records: HashMap<String, Vec<CostRecord>>,
430 artifacts: HashMap<String, Vec<Artifact>>,
431}
432
433impl MemoryStore {
434 pub fn new() -> Self {
435 Self::default()
436 }
437
438 pub fn save_to_path(&self, path: impl AsRef<Path>) -> Result<()> {
439 fs::write(path, encode_store(self)).map_err(|err| RuntimeError(err.to_string()))
440 }
441
442 pub fn load_from_path(path: impl AsRef<Path>) -> Result<Self> {
443 let body = fs::read_to_string(path).map_err(|err| RuntimeError(err.to_string()))?;
444 decode_store(&body)
445 }
446
447 pub fn create_run(&mut self, initial_state: State) -> (String, String) {
448 let run_id = new_id("run");
449 let session_id = new_id("sess");
450 let step_id = new_id("step");
451 let now = now_seconds();
452 self.runs.insert(
453 run_id.clone(),
454 Run {
455 run_id: run_id.clone(),
456 session_id: session_id.clone(),
457 status: "pending".to_string(),
458 state: initial_state.clone(),
459 state_version: 0,
460 created_at: now,
461 updated_at: now,
462 },
463 );
464 self.steps.insert(
465 step_id.clone(),
466 Step {
467 step_id: step_id.clone(),
468 run_id: run_id.clone(),
469 session_id: session_id.clone(),
470 status: "pending".to_string(),
471 owner: None,
472 lease_token: None,
473 lease_until: None,
474 attempt: 0,
475 state_version: 0,
476 checkpoint_id: None,
477 last_error_type: None,
478 last_error: None,
479 cancelled_at: None,
480 created_at: now,
481 updated_at: now,
482 },
483 );
484 let mut payload = State::new();
485 payload.insert("initial_state".to_string(), Value::Object(initial_state));
486 self.append_event(
487 &run_id,
488 Some(&session_id),
489 None,
490 "run_created",
491 payload,
492 None,
493 None,
494 None,
495 );
496 let mut step_payload = State::new();
497 step_payload.insert("step_id".to_string(), Value::String(step_id.clone()));
498 self.append_event(
499 &run_id,
500 Some(&session_id),
501 Some(&step_id),
502 "step_created",
503 step_payload,
504 None,
505 None,
506 None,
507 );
508 (run_id, step_id)
509 }
510
511 pub fn create_external_step(&mut self, run_id: &str) -> Result<Step> {
512 let run = self
513 .runs
514 .get(run_id)
515 .ok_or_else(|| RuntimeError(format!("run not found: {run_id}")))?
516 .clone();
517 let step_id = new_id("step");
518 let now = now_seconds();
519 let step = Step {
520 step_id: step_id.clone(),
521 run_id: run_id.to_string(),
522 session_id: run.session_id.clone(),
523 status: "pending".to_string(),
524 owner: None,
525 lease_token: None,
526 lease_until: None,
527 attempt: 0,
528 state_version: run.state_version,
529 checkpoint_id: None,
530 last_error_type: None,
531 last_error: None,
532 cancelled_at: None,
533 created_at: now,
534 updated_at: now,
535 };
536 self.steps.insert(step_id.clone(), step.clone());
537 if let Some(run) = self.runs.get_mut(run_id) {
538 run.status = "pending".to_string();
539 run.updated_at = now;
540 }
541 let mut payload = State::new();
542 payload.insert("step_id".to_string(), Value::String(step_id));
543 payload.insert("external".to_string(), Value::Bool(true));
544 self.append_event(
545 run_id,
546 Some(&run.session_id),
547 Some(&step.step_id),
548 "step_created",
549 payload,
550 None,
551 Some(run.state_version),
552 None,
553 );
554 Ok(step)
555 }
556
557 pub fn claim_step(
558 &mut self,
559 worker_id: &str,
560 run_id: &str,
561 lease_seconds: f64,
562 ) -> Result<StepClaim> {
563 let mut candidates: Vec<Step> = self
564 .steps
565 .values()
566 .filter(|step| {
567 (run_id.is_empty() || step.run_id == run_id)
568 && (step.status == "pending" || step.status == "retry_scheduled")
569 })
570 .cloned()
571 .collect();
572 candidates.sort_by(|a, b| {
573 a.created_at
574 .partial_cmp(&b.created_at)
575 .unwrap()
576 .then(a.step_id.cmp(&b.step_id))
577 });
578 let candidate = candidates
579 .first()
580 .ok_or_else(|| RuntimeError("agentledger: no runnable step".to_string()))?;
581 let now = now_seconds();
582 let step = self.steps.get_mut(&candidate.step_id).unwrap();
583 step.status = "running".to_string();
584 step.owner = Some(worker_id.to_string());
585 step.lease_token = Some(new_id("lease"));
586 step.lease_until = Some(now + lease_seconds);
587 step.attempt += 1;
588 step.updated_at = now;
589 let claim = StepClaim {
590 run_id: step.run_id.clone(),
591 session_id: step.session_id.clone(),
592 step_id: step.step_id.clone(),
593 attempt: step.attempt,
594 lease_token: step.lease_token.clone().unwrap(),
595 state_version: step.state_version,
596 lease_until: step.lease_until.unwrap(),
597 };
598 let run = self.runs.get_mut(&claim.run_id).unwrap();
599 run.status = "running".to_string();
600 run.updated_at = now;
601 let mut payload = State::new();
602 payload.insert(
603 "worker_id".to_string(),
604 Value::String(worker_id.to_string()),
605 );
606 payload.insert(
607 "lease_token".to_string(),
608 Value::String(claim.lease_token.clone()),
609 );
610 payload.insert("attempt".to_string(), Value::Number(claim.attempt as f64));
611 self.append_event(
612 &claim.run_id,
613 Some(&claim.session_id),
614 Some(&claim.step_id),
615 "step_claimed",
616 payload,
617 None,
618 None,
619 None,
620 );
621 Ok(claim)
622 }
623
624 pub fn load_state(&self, run_id: &str) -> Result<(State, u64, String)> {
625 let run = self
626 .runs
627 .get(run_id)
628 .ok_or_else(|| RuntimeError(format!("run not found: {run_id}")))?;
629 Ok((run.state.clone(), run.state_version, run.session_id.clone()))
630 }
631
632 pub fn recover_expired_leases(&mut self) -> usize {
633 let now = now_seconds();
634 let ids: Vec<String> = self.steps.keys().cloned().collect();
635 let mut recovered = 0;
636 for step_id in ids {
637 let should_recover = self
638 .steps
639 .get(&step_id)
640 .map(|step| {
641 step.status == "running" && step.lease_until.is_some_and(|until| until <= now)
642 })
643 .unwrap_or(false);
644 if !should_recover {
645 continue;
646 }
647 let (run_id, session_id, previous_owner, attempt) = {
648 let step = self.steps.get_mut(&step_id).unwrap();
649 let previous_owner = step.owner.clone().unwrap_or_default();
650 step.status = "retry_scheduled".to_string();
651 step.owner = None;
652 step.lease_token = None;
653 step.lease_until = None;
654 step.updated_at = now;
655 (
656 step.run_id.clone(),
657 step.session_id.clone(),
658 previous_owner,
659 step.attempt,
660 )
661 };
662 let run = self.runs.get_mut(&run_id).unwrap();
663 run.status = "retry_scheduled".to_string();
664 run.updated_at = now;
665 recovered += 1;
666 let mut payload = State::new();
667 payload.insert("previous_owner".to_string(), Value::String(previous_owner));
668 payload.insert("attempt".to_string(), Value::Number(attempt as f64));
669 self.append_event(
670 &run_id,
671 Some(&session_id),
672 Some(&step_id),
673 "lease_expired",
674 payload,
675 None,
676 None,
677 None,
678 );
679 let mut retry_payload = State::new();
680 retry_payload.insert("step_id".to_string(), Value::String(step_id.clone()));
681 retry_payload.insert(
682 "reason".to_string(),
683 Value::String("lease_expired".to_string()),
684 );
685 self.append_event(
686 &run_id,
687 Some(&session_id),
688 Some(&step_id),
689 "step_retry_scheduled",
690 retry_payload,
691 None,
692 None,
693 None,
694 );
695 }
696 recovered
697 }
698
699 pub fn cancel_run(&mut self, run_id: &str, reason: &str) -> Result<usize> {
700 let session_id = self
701 .runs
702 .get(run_id)
703 .ok_or_else(|| RuntimeError(format!("run not found: {run_id}")))?
704 .session_id
705 .clone();
706 let mut payload = State::new();
707 payload.insert("reason".to_string(), Value::String(reason.to_string()));
708 self.append_event(
709 run_id,
710 Some(&session_id),
711 None,
712 "run_cancel_requested",
713 payload,
714 None,
715 None,
716 None,
717 );
718 let ids: Vec<String> = self.steps.keys().cloned().collect();
719 let now = now_seconds();
720 let mut cancelled = 0;
721 for step_id in ids {
722 let should_cancel = self
723 .steps
724 .get(&step_id)
725 .map(|step| {
726 step.run_id == run_id
727 && !["completed", "failed", "cancelled"].contains(&step.status.as_str())
728 })
729 .unwrap_or(false);
730 if !should_cancel {
731 continue;
732 }
733 let step_session = {
734 let step = self.steps.get_mut(&step_id).unwrap();
735 step.status = "cancelled".to_string();
736 step.owner = None;
737 step.lease_token = None;
738 step.lease_until = None;
739 step.cancelled_at = Some(now);
740 step.updated_at = now;
741 step.session_id.clone()
742 };
743 cancelled += 1;
744 let mut step_payload = State::new();
745 step_payload.insert("reason".to_string(), Value::String(reason.to_string()));
746 self.append_event(
747 run_id,
748 Some(&step_session),
749 Some(&step_id),
750 "step_cancelled",
751 step_payload,
752 None,
753 None,
754 None,
755 );
756 }
757 let run = self.runs.get_mut(run_id).unwrap();
758 run.status = "cancelled".to_string();
759 run.updated_at = now;
760 let mut done_payload = State::new();
761 done_payload.insert("reason".to_string(), Value::String(reason.to_string()));
762 done_payload.insert(
763 "cancelled_steps".to_string(),
764 Value::Number(cancelled as f64),
765 );
766 self.append_event(
767 run_id,
768 Some(&session_id),
769 None,
770 "run_cancelled",
771 done_payload,
772 None,
773 None,
774 None,
775 );
776 Ok(cancelled)
777 }
778
779 pub fn commit_state_patch(
780 &mut self,
781 run_id: &str,
782 step_id: &str,
783 lease_token: &str,
784 base_version: u64,
785 patch: State,
786 ) -> Result<u64> {
787 self.validate_lease(step_id, lease_token)?;
788 let run = self
789 .runs
790 .get(run_id)
791 .ok_or_else(|| RuntimeError(format!("run not found: {run_id}")))?;
792 if run.state_version != base_version {
793 return Err(RuntimeError(format!(
794 "state version conflict: expected {base_version}, got {}",
795 run.state_version
796 )));
797 }
798 let now = now_seconds();
799 let new_version = base_version + 1;
800 let session_id = self.steps.get(step_id).unwrap().session_id.clone();
801 let run = self.runs.get_mut(run_id).unwrap();
802 run.state = merge_patch(&run.state, &patch);
803 run.state_version = new_version;
804 run.status = "completed".to_string();
805 run.updated_at = now;
806 let step = self.steps.get_mut(step_id).unwrap();
807 step.status = "completed".to_string();
808 step.state_version = new_version;
809 step.checkpoint_id = Some(format!("ckpt:{run_id}:{step_id}:{}", step.attempt));
810 step.updated_at = now;
811 let mut payload = State::new();
812 payload.insert("patch".to_string(), Value::Object(patch));
813 payload.insert(
814 "state_version".to_string(),
815 Value::Number(new_version as f64),
816 );
817 self.append_event(
818 run_id,
819 Some(&session_id),
820 Some(step_id),
821 "state_patch_committed",
822 payload,
823 None,
824 Some(new_version),
825 None,
826 );
827 let mut complete_payload = State::new();
828 complete_payload.insert("step_id".to_string(), Value::String(step_id.to_string()));
829 self.append_event(
830 run_id,
831 Some(&session_id),
832 Some(step_id),
833 "step_completed",
834 complete_payload,
835 None,
836 Some(new_version),
837 None,
838 );
839 Ok(new_version)
840 }
841
842 pub fn apply_system_state_patch(
843 &mut self,
844 run_id: &str,
845 patch: State,
846 reason: &str,
847 ) -> Result<u64> {
848 let run = self
849 .runs
850 .get(run_id)
851 .ok_or_else(|| RuntimeError(format!("run not found: {run_id}")))?
852 .clone();
853 let new_version = run.state_version + 1;
854 let now = now_seconds();
855 if let Some(item) = self.runs.get_mut(run_id) {
856 item.state = merge_patch(&item.state, &patch);
857 item.state_version = new_version;
858 item.updated_at = now;
859 }
860 let mut payload = State::new();
861 payload.insert("patch".to_string(), Value::Object(patch));
862 payload.insert("reason".to_string(), Value::String(reason.to_string()));
863 payload.insert(
864 "state_version".to_string(),
865 Value::Number(new_version as f64),
866 );
867 self.append_event(
868 run_id,
869 Some(&run.session_id),
870 None,
871 "system_state_patch_applied",
872 payload,
873 None,
874 Some(new_version),
875 None,
876 );
877 Ok(new_version)
878 }
879
880 pub fn mark_retry(&mut self, run_id: &str, step_id: &str, error_type: &str, message: &str) {
881 let now = now_seconds();
882 let session_id = {
883 let step = self.steps.get_mut(step_id).unwrap();
884 step.status = "retry_scheduled".to_string();
885 step.owner = None;
886 step.lease_token = None;
887 step.lease_until = None;
888 step.last_error_type = Some(error_type.to_string());
889 step.last_error = Some(message.to_string());
890 step.updated_at = now;
891 step.session_id.clone()
892 };
893 let run = self.runs.get_mut(run_id).unwrap();
894 run.status = "retry_scheduled".to_string();
895 run.updated_at = now;
896 let mut classified = State::new();
897 classified.insert("error".to_string(), Value::String(message.to_string()));
898 classified.insert(
899 "error_type".to_string(),
900 Value::String(error_type.to_string()),
901 );
902 classified.insert("retryable".to_string(), Value::Bool(true));
903 classified.insert("source".to_string(), Value::String("agent".to_string()));
904 self.append_event(
905 run_id,
906 Some(&session_id),
907 Some(step_id),
908 "failure_classified",
909 classified,
910 None,
911 None,
912 None,
913 );
914 let mut payload = State::new();
915 payload.insert("error".to_string(), Value::String(message.to_string()));
916 payload.insert(
917 "error_type".to_string(),
918 Value::String(error_type.to_string()),
919 );
920 self.append_event(
921 run_id,
922 Some(&session_id),
923 Some(step_id),
924 "error_raised",
925 payload,
926 None,
927 None,
928 None,
929 );
930 let mut retry_payload = State::new();
931 retry_payload.insert("step_id".to_string(), Value::String(step_id.to_string()));
932 self.append_event(
933 run_id,
934 Some(&session_id),
935 Some(step_id),
936 "step_retry_scheduled",
937 retry_payload,
938 None,
939 None,
940 None,
941 );
942 }
943
944 pub fn mark_waiting_human(
945 &mut self,
946 run_id: &str,
947 step_id: &str,
948 reason: &str,
949 approval_id: &str,
950 ) {
951 let now = now_seconds();
952 let session_id = {
953 let step = self.steps.get_mut(step_id).unwrap();
954 step.status = "waiting_human".to_string();
955 step.owner = None;
956 step.lease_token = None;
957 step.lease_until = None;
958 step.last_error_type = Some("ApprovalRequired".to_string());
959 step.last_error = Some(reason.to_string());
960 step.updated_at = now;
961 step.session_id.clone()
962 };
963 let run = self.runs.get_mut(run_id).unwrap();
964 run.status = "waiting_human".to_string();
965 run.updated_at = now;
966 let mut payload = State::new();
967 payload.insert("reason".to_string(), Value::String(reason.to_string()));
968 payload.insert(
969 "approval_id".to_string(),
970 Value::String(approval_id.to_string()),
971 );
972 self.append_event(
973 run_id,
974 Some(&session_id),
975 Some(step_id),
976 "step_waiting_human",
977 payload,
978 None,
979 None,
980 None,
981 );
982 }
983
984 pub fn mark_failed(&mut self, run_id: &str, step_id: &str, error_type: &str, message: &str) {
985 let now = now_seconds();
986 let session_id = {
987 let step = self.steps.get_mut(step_id).unwrap();
988 step.status = "failed".to_string();
989 step.owner = None;
990 step.lease_token = None;
991 step.lease_until = None;
992 step.last_error_type = Some(error_type.to_string());
993 step.last_error = Some(message.to_string());
994 step.updated_at = now;
995 step.session_id.clone()
996 };
997 let run = self.runs.get_mut(run_id).unwrap();
998 run.status = "failed".to_string();
999 run.updated_at = now;
1000 let mut classified = State::new();
1001 classified.insert("error".to_string(), Value::String(message.to_string()));
1002 classified.insert(
1003 "error_type".to_string(),
1004 Value::String(error_type.to_string()),
1005 );
1006 classified.insert("retryable".to_string(), Value::Bool(false));
1007 classified.insert(
1008 "source".to_string(),
1009 Value::String(failure_source(error_type).to_string()),
1010 );
1011 self.append_event(
1012 run_id,
1013 Some(&session_id),
1014 Some(step_id),
1015 "failure_classified",
1016 classified,
1017 None,
1018 None,
1019 None,
1020 );
1021 let mut error = State::new();
1022 error.insert("error".to_string(), Value::String(message.to_string()));
1023 error.insert(
1024 "error_type".to_string(),
1025 Value::String(error_type.to_string()),
1026 );
1027 self.append_event(
1028 run_id,
1029 Some(&session_id),
1030 Some(step_id),
1031 "error_raised",
1032 error,
1033 None,
1034 None,
1035 None,
1036 );
1037 let mut payload = State::new();
1038 payload.insert("step_id".to_string(), Value::String(step_id.to_string()));
1039 payload.insert(
1040 "error_type".to_string(),
1041 Value::String(error_type.to_string()),
1042 );
1043 self.append_event(
1044 run_id,
1045 Some(&session_id),
1046 Some(step_id),
1047 "step_failed",
1048 payload,
1049 None,
1050 None,
1051 None,
1052 );
1053 }
1054
1055 pub fn append_event(
1056 &mut self,
1057 run_id: &str,
1058 session_id: Option<&str>,
1059 step_id: Option<&str>,
1060 event_type: &str,
1061 payload: State,
1062 agent_role: Option<&str>,
1063 state_version: Option<u64>,
1064 causal_token: Option<&str>,
1065 ) -> Event {
1066 let events = self.events.entry(run_id.to_string()).or_default();
1067 let payload_ref = format_state(&payload);
1068 let event = Event {
1069 event_id: new_id("evt"),
1070 run_id: run_id.to_string(),
1071 session_id: session_id.map(str::to_string),
1072 step_id: step_id.map(str::to_string),
1073 seq: events.len() as u64 + 1,
1074 event_type: event_type.to_string(),
1075 timestamp: now_seconds(),
1076 agent_role: agent_role.map(str::to_string),
1077 state_version,
1078 causal_token: causal_token.map(str::to_string),
1079 payload_hash: stable_hash(&payload_ref),
1080 payload_ref,
1081 payload,
1082 };
1083 events.push(event.clone());
1084 event
1085 }
1086
1087 pub fn reserve_ledger(&mut self, key: &str, entry: ToolLedgerEntry) -> Option<ToolLedgerEntry> {
1088 if let Some(existing) = self.tool_ledger.get(key) {
1089 return Some(existing.clone());
1090 }
1091 self.tool_ledger.insert(key.to_string(), entry);
1092 None
1093 }
1094
1095 pub fn update_ledger(
1096 &mut self,
1097 key: &str,
1098 status: &str,
1099 response: Option<Value>,
1100 error_type: Option<String>,
1101 ) {
1102 if let Some(entry) = self.tool_ledger.get_mut(key) {
1103 entry.status = status.to_string();
1104 entry.response = response.clone();
1105 entry.response_ref = response.as_ref().map(format_value);
1106 entry.response_hash = entry.response_ref.as_ref().map(|value| stable_hash(value));
1107 entry.error_type = error_type;
1108 entry.updated_at = now_seconds();
1109 }
1110 }
1111
1112 pub fn request_approval(
1113 &mut self,
1114 approval_key: &str,
1115 run_id: &str,
1116 session_id: &str,
1117 step_id: &str,
1118 tool_name: &str,
1119 risk_level: &str,
1120 reason: &str,
1121 request_hash: &str,
1122 request_ref: &str,
1123 requested_by: &str,
1124 ) -> ApprovalRequest {
1125 if let Some(existing) = self.approval_requests.get(approval_key) {
1126 return existing.clone();
1127 }
1128 let now = now_seconds();
1129 let approval = ApprovalRequest {
1130 approval_id: new_id("approval"),
1131 approval_key: approval_key.to_string(),
1132 run_id: run_id.to_string(),
1133 session_id: session_id.to_string(),
1134 step_id: step_id.to_string(),
1135 tool_name: tool_name.to_string(),
1136 risk_level: risk_level.to_string(),
1137 status: "PENDING".to_string(),
1138 reason: reason.to_string(),
1139 request_hash: request_hash.to_string(),
1140 request_ref: request_ref.to_string(),
1141 requested_by: requested_by.to_string(),
1142 approved_by: None,
1143 decision_reason: None,
1144 created_at: now,
1145 updated_at: now,
1146 };
1147 self.approval_requests
1148 .insert(approval_key.to_string(), approval.clone());
1149 approval
1150 }
1151
1152 pub fn approval_for_key(&self, approval_key: &str) -> Option<ApprovalRequest> {
1153 self.approval_requests.get(approval_key).cloned()
1154 }
1155
1156 pub fn approval_requests(&self, run_id: &str) -> Vec<ApprovalRequest> {
1157 self.approval_requests
1158 .values()
1159 .filter(|item| run_id.is_empty() || item.run_id == run_id)
1160 .cloned()
1161 .collect()
1162 }
1163
1164 pub fn approve_request(
1165 &mut self,
1166 approval_id: &str,
1167 approver: &str,
1168 reason: &str,
1169 ) -> Result<ApprovalRequest> {
1170 self.decide_approval(approval_id, "APPROVED", approver, reason)
1171 }
1172
1173 pub fn deny_request(
1174 &mut self,
1175 approval_id: &str,
1176 approver: &str,
1177 reason: &str,
1178 ) -> Result<ApprovalRequest> {
1179 self.decide_approval(approval_id, "DENIED", approver, reason)
1180 }
1181
1182 fn decide_approval(
1183 &mut self,
1184 approval_id: &str,
1185 status: &str,
1186 approver: &str,
1187 reason: &str,
1188 ) -> Result<ApprovalRequest> {
1189 let key = self
1190 .approval_requests
1191 .iter()
1192 .find(|(_, item)| item.approval_id == approval_id)
1193 .map(|(key, _)| key.clone())
1194 .ok_or_else(|| RuntimeError(format!("approval not found: {approval_id}")))?;
1195 let mut approval = self.approval_requests.get(&key).unwrap().clone();
1196 approval.status = status.to_string();
1197 approval.approved_by = Some(approver.to_string());
1198 approval.decision_reason = Some(reason.to_string());
1199 approval.updated_at = now_seconds();
1200 self.approval_requests.insert(key, approval.clone());
1201 let mut payload = State::new();
1202 payload.insert(
1203 "approval_id".to_string(),
1204 Value::String(approval_id.to_string()),
1205 );
1206 payload.insert(
1207 "tool".to_string(),
1208 Value::String(approval.tool_name.clone()),
1209 );
1210 payload.insert("status".to_string(), Value::String(status.to_string()));
1211 self.append_event(
1212 &approval.run_id,
1213 Some(&approval.session_id),
1214 Some(&approval.step_id),
1215 "tool_approval_decided",
1216 payload,
1217 None,
1218 None,
1219 None,
1220 );
1221 if self
1222 .steps
1223 .get(&approval.step_id)
1224 .map(|step| step.status.as_str())
1225 == Some("waiting_human")
1226 {
1227 if status == "APPROVED" {
1228 let step = self.steps.get_mut(&approval.step_id).unwrap();
1229 step.status = "pending".to_string();
1230 step.owner = None;
1231 step.lease_token = None;
1232 step.lease_until = None;
1233 step.updated_at = now_seconds();
1234 let run = self.runs.get_mut(&approval.run_id).unwrap();
1235 run.status = "pending".to_string();
1236 let mut retry = State::new();
1237 retry.insert(
1238 "step_id".to_string(),
1239 Value::String(approval.step_id.clone()),
1240 );
1241 retry.insert(
1242 "reason".to_string(),
1243 Value::String("approval_granted".to_string()),
1244 );
1245 self.append_event(
1246 &approval.run_id,
1247 Some(&approval.session_id),
1248 Some(&approval.step_id),
1249 "step_retry_scheduled",
1250 retry,
1251 None,
1252 None,
1253 None,
1254 );
1255 } else if status == "DENIED" {
1256 self.mark_failed(
1257 &approval.run_id,
1258 &approval.step_id,
1259 "ApprovalDenied",
1260 reason,
1261 );
1262 }
1263 }
1264 Ok(approval)
1265 }
1266
1267 pub fn record_cost(
1268 &mut self,
1269 run_id: &str,
1270 session_id: &str,
1271 step_id: &str,
1272 category: &str,
1273 name: &str,
1274 amount: f64,
1275 unit: &str,
1276 metadata: State,
1277 ) -> String {
1278 let cost_id = new_id("cost");
1279 let record = CostRecord {
1280 cost_id: cost_id.clone(),
1281 run_id: run_id.to_string(),
1282 session_id: session_id.to_string(),
1283 step_id: step_id.to_string(),
1284 category: category.to_string(),
1285 name: name.to_string(),
1286 amount,
1287 unit: unit.to_string(),
1288 metadata,
1289 created_at: now_seconds(),
1290 };
1291 self.cost_records
1292 .entry(run_id.to_string())
1293 .or_default()
1294 .push(record);
1295 let mut payload = State::new();
1296 payload.insert("cost_id".to_string(), Value::String(cost_id.clone()));
1297 payload.insert("category".to_string(), Value::String(category.to_string()));
1298 payload.insert("name".to_string(), Value::String(name.to_string()));
1299 payload.insert("amount".to_string(), Value::Number(amount));
1300 payload.insert("unit".to_string(), Value::String(unit.to_string()));
1301 self.append_event(
1302 run_id,
1303 Some(session_id),
1304 Some(step_id),
1305 "cost_recorded",
1306 payload,
1307 None,
1308 None,
1309 None,
1310 );
1311 cost_id
1312 }
1313
1314 pub fn cost_records(&self, run_id: &str) -> Vec<CostRecord> {
1315 self.cost_records.get(run_id).cloned().unwrap_or_default()
1316 }
1317
1318 pub fn cost_summary(&self, run_id: &str) -> CostSummary {
1319 let mut summary = CostSummary::default();
1320 for record in self.cost_records(run_id) {
1321 add_cost(&mut summary, &record);
1322 }
1323 summary
1324 }
1325
1326 pub fn create_artifact(
1327 &mut self,
1328 run_id: &str,
1329 step_id: Option<&str>,
1330 name: &str,
1331 content: State,
1332 metadata: State,
1333 ) -> Artifact {
1334 let blob_ref = format_state(&content);
1335 let artifact = Artifact {
1336 artifact_id: new_id("art"),
1337 run_id: run_id.to_string(),
1338 step_id: step_id.map(str::to_string),
1339 name: name.to_string(),
1340 blob_hash: format!("sha256:{}", stable_hash(&blob_ref)),
1341 blob_ref,
1342 metadata,
1343 created_at: now_seconds(),
1344 };
1345 self.artifacts
1346 .entry(run_id.to_string())
1347 .or_default()
1348 .push(artifact.clone());
1349 artifact
1350 }
1351
1352 pub fn artifacts(&self, run_id: &str) -> Vec<Artifact> {
1353 let mut items = self.artifacts.get(run_id).cloned().unwrap_or_default();
1354 items.sort_by(|a, b| {
1355 a.created_at
1356 .partial_cmp(&b.created_at)
1357 .unwrap()
1358 .then(a.artifact_id.cmp(&b.artifact_id))
1359 });
1360 items
1361 }
1362
1363 pub fn validate_lease(&self, step_id: &str, lease_token: &str) -> Result<()> {
1364 let step = self
1365 .steps
1366 .get(step_id)
1367 .ok_or_else(|| RuntimeError(format!("step not found: {step_id}")))?;
1368 if step.status != "running" || step.lease_token.as_deref() != Some(lease_token) {
1369 return Err(RuntimeError("invalid or stale lease token".to_string()));
1370 }
1371 if step.lease_until.is_some_and(|until| until <= now_seconds()) {
1372 return Err(RuntimeError("lease expired".to_string()));
1373 }
1374 Ok(())
1375 }
1376
1377 pub fn final_state(&self, run_id: &str) -> Result<State> {
1378 Ok(self.load_state(run_id)?.0)
1379 }
1380
1381 pub fn run(&self, run_id: &str) -> Result<Run> {
1382 self.runs
1383 .get(run_id)
1384 .cloned()
1385 .ok_or_else(|| RuntimeError(format!("run not found: {run_id}")))
1386 }
1387
1388 pub fn steps(&self, run_id: &str) -> Vec<Step> {
1389 self.steps
1390 .values()
1391 .filter(|step| step.run_id == run_id)
1392 .cloned()
1393 .collect()
1394 }
1395
1396 pub fn events(&self, run_id: &str) -> Vec<Event> {
1397 self.events.get(run_id).cloned().unwrap_or_default()
1398 }
1399
1400 pub fn ledger(&self, run_id: &str) -> Vec<ToolLedgerEntry> {
1401 self.tool_ledger
1402 .values()
1403 .filter(|entry| entry.run_id == run_id)
1404 .cloned()
1405 .collect()
1406 }
1407}
1408
1409pub struct ToolRegistry {
1410 tools: HashMap<String, ToolSpec>,
1411}
1412
1413impl ToolRegistry {
1414 pub fn new() -> Self {
1415 Self {
1416 tools: HashMap::new(),
1417 }
1418 }
1419
1420 pub fn register(&mut self, spec: ToolSpec) {
1421 self.tools.insert(spec.name.clone(), spec);
1422 }
1423
1424 pub fn get(&self, name: &str) -> Result<&ToolSpec> {
1425 self.tools
1426 .get(name)
1427 .ok_or_else(|| RuntimeError(format!("tool not registered: {name}")))
1428 }
1429}
1430
1431impl Default for ToolRegistry {
1432 fn default() -> Self {
1433 Self::new()
1434 }
1435}
1436
1437pub struct Runtime {
1438 pub store: MemoryStore,
1439 pub registry: ToolRegistry,
1440 pub budget: BudgetLimits,
1441 pub sandbox: Box<dyn SandboxExecutor>,
1442}
1443
1444impl Runtime {
1445 pub fn new() -> Self {
1446 Self {
1447 store: MemoryStore::new(),
1448 registry: ToolRegistry::new(),
1449 budget: BudgetLimits::default(),
1450 sandbox: Box::new(DisabledSandboxExecutor),
1451 }
1452 }
1453
1454 pub fn register_tool(&mut self, spec: ToolSpec) {
1455 self.registry.register(spec);
1456 }
1457
1458 pub fn set_budget(&mut self, budget: BudgetLimits) {
1459 self.budget = budget;
1460 }
1461
1462 pub fn set_sandbox(&mut self, sandbox: Box<dyn SandboxExecutor>) {
1463 self.sandbox = sandbox;
1464 }
1465
1466 pub fn create_run(&mut self, initial_state: State) -> (String, String) {
1467 self.store.create_run(initial_state)
1468 }
1469
1470 pub fn run_once(
1471 &mut self,
1472 run_id: &str,
1473 worker_id: &str,
1474 agent_role: &str,
1475 lease_seconds: f64,
1476 agent: AgentFunc,
1477 ) -> Result<bool> {
1478 let claim = match self.store.claim_step(worker_id, run_id, lease_seconds) {
1479 Ok(claim) => claim,
1480 Err(err) if err.0.contains("no runnable step") => return Ok(false),
1481 Err(err) => return Err(err),
1482 };
1483 let (state, version, session_id) = self.store.load_state(&claim.run_id)?;
1484 let mut payload = State::new();
1485 payload.insert(
1486 "agent_role".to_string(),
1487 Value::String(agent_role.to_string()),
1488 );
1489 payload.insert("attempt".to_string(), Value::Number(claim.attempt as f64));
1490 self.store.append_event(
1491 &claim.run_id,
1492 Some(&session_id),
1493 Some(&claim.step_id),
1494 "agent_started",
1495 payload,
1496 Some(agent_role),
1497 Some(version),
1498 None,
1499 );
1500 let mut ctx = AgentContext {
1501 run_id: claim.run_id.clone(),
1502 session_id,
1503 step_id: claim.step_id.clone(),
1504 agent_role: agent_role.to_string(),
1505 lease_token: claim.lease_token.clone(),
1506 attempt: claim.attempt,
1507 state_version: version,
1508 pending_patch: State::new(),
1509 };
1510 match agent(&mut ctx, state) {
1511 Ok(()) => {
1512 self.store.commit_state_patch(
1513 &claim.run_id,
1514 &claim.step_id,
1515 &claim.lease_token,
1516 version,
1517 ctx.pending_patch,
1518 )?;
1519 Ok(true)
1520 }
1521 Err(err) if err.0 == "retryable" => {
1522 self.store.mark_retry(
1523 &claim.run_id,
1524 &claim.step_id,
1525 "RetryableAgentError",
1526 "retryable",
1527 );
1528 Ok(false)
1529 }
1530 Err(err) if err.0.starts_with("approval required:") => {
1531 let approval_id = err.0.trim_start_matches("approval required:");
1532 self.store
1533 .mark_waiting_human(&claim.run_id, &claim.step_id, &err.0, approval_id);
1534 Ok(false)
1535 }
1536 Err(err) => {
1537 self.store.mark_failed(
1538 &claim.run_id,
1539 &claim.step_id,
1540 classify_runtime_error(&err.0),
1541 &err.0,
1542 );
1543 Err(err)
1544 }
1545 }
1546 }
1547
1548 pub fn call_tool(&mut self, ctx: &AgentContext, tool_name: &str, args: State) -> Result<Value> {
1549 let (
1550 version,
1551 side_effect,
1552 risk_level,
1553 idempotency_required,
1554 approval_required,
1555 sandbox_required,
1556 sandbox_executor,
1557 sandbox_policy,
1558 input_schema,
1559 output_schema,
1560 ) = {
1561 let spec = self.registry.get(tool_name)?;
1562 (
1563 spec.version.clone(),
1564 spec.side_effect.clone(),
1565 spec.risk_level.clone(),
1566 spec.idempotency_required,
1567 spec.approval_required,
1568 spec.sandbox_required,
1569 spec.sandbox_executor.clone(),
1570 spec.sandbox_policy.clone(),
1571 spec.input_schema.clone(),
1572 spec.output_schema.clone(),
1573 )
1574 };
1575 let mut request = State::new();
1576 request.insert("tool".to_string(), Value::String(tool_name.to_string()));
1577 request.insert("args".to_string(), Value::Object(args.clone()));
1578 let request_ref = format_state(&request);
1579 let request_hash = stable_hash(&request_ref);
1580 let causal_token = format!(
1581 "{}:{}:{}:{}",
1582 ctx.run_id, ctx.step_id, ctx.attempt, ctx.state_version
1583 );
1584 let idempotency_key = format!(
1585 "{}:{}:{}:{}",
1586 ctx.run_id, ctx.step_id, tool_name, request_hash
1587 );
1588 let managed = side_effect != "none" || idempotency_required;
1589 self.store.append_event(
1590 &ctx.run_id,
1591 Some(&ctx.session_id),
1592 Some(&ctx.step_id),
1593 "tool_call_requested",
1594 request,
1595 Some(&ctx.agent_role),
1596 Some(ctx.state_version),
1597 Some(&causal_token),
1598 );
1599 if let Some(schema) = input_schema.as_ref() {
1600 if let Err(err) = validate_tool_schema(schema, &Value::Object(args.clone()), "$arg") {
1601 let mut failed = State::new();
1602 failed.insert("tool".to_string(), Value::String(tool_name.to_string()));
1603 failed.insert("error".to_string(), Value::String(err.0.clone()));
1604 failed.insert(
1605 "phase".to_string(),
1606 Value::String("input_validation".to_string()),
1607 );
1608 self.store.append_event(
1609 &ctx.run_id,
1610 Some(&ctx.session_id),
1611 Some(&ctx.step_id),
1612 "tool_call_failed",
1613 failed,
1614 Some(&ctx.agent_role),
1615 Some(ctx.state_version),
1616 Some(&causal_token),
1617 );
1618 return Err(err);
1619 }
1620 }
1621
1622 let mut allowed = !is_high_risk(&risk_level);
1623 let mut reason = if allowed {
1624 "default allow for low/medium risk in local runtime".to_string()
1625 } else {
1626 "high-risk tool denied by default".to_string()
1627 };
1628 if let Some(approval) = self.store.approval_for_key(&idempotency_key) {
1629 if approval.status == "DENIED" {
1630 reason = format!("approval denied for tool {tool_name}");
1631 self.record_permission(ctx, tool_name, false, &reason, &causal_token);
1632 return Err(RuntimeError(reason));
1633 }
1634 if approval.status == "APPROVED" {
1635 allowed = true;
1636 reason = format!(
1637 "approved by {}",
1638 approval
1639 .approved_by
1640 .unwrap_or_else(|| "operator".to_string())
1641 );
1642 }
1643 } else if approval_required {
1644 let approval = self.store.request_approval(
1645 &idempotency_key,
1646 &ctx.run_id,
1647 &ctx.session_id,
1648 &ctx.step_id,
1649 tool_name,
1650 &risk_level,
1651 "tool requires approval",
1652 &request_hash,
1653 &request_ref,
1654 &ctx.agent_role,
1655 );
1656 self.record_permission(ctx, tool_name, false, "approval required", &causal_token);
1657 let mut payload = State::new();
1658 payload.insert("tool".to_string(), Value::String(tool_name.to_string()));
1659 payload.insert(
1660 "approval_id".to_string(),
1661 Value::String(approval.approval_id.clone()),
1662 );
1663 payload.insert(
1664 "approval_key".to_string(),
1665 Value::String(idempotency_key.clone()),
1666 );
1667 payload.insert("risk_level".to_string(), Value::String(risk_level));
1668 self.store.append_event(
1669 &ctx.run_id,
1670 Some(&ctx.session_id),
1671 Some(&ctx.step_id),
1672 "tool_approval_required",
1673 payload,
1674 Some(&ctx.agent_role),
1675 Some(ctx.state_version),
1676 Some(&causal_token),
1677 );
1678 return Err(RuntimeError(format!(
1679 "approval required:{}",
1680 approval.approval_id
1681 )));
1682 }
1683 self.record_permission(ctx, tool_name, allowed, &reason, &causal_token);
1684 if !allowed {
1685 return Err(RuntimeError(reason));
1686 }
1687 if let Some(max) = self.budget.max_tool_calls {
1688 let used = self.store.cost_summary(&ctx.run_id).tool_calls;
1689 if used >= max {
1690 let message = format!("tool call budget exceeded: {used}/{max}");
1691 let mut payload = State::new();
1692 payload.insert("category".to_string(), Value::String("tool".to_string()));
1693 payload.insert("tool".to_string(), Value::String(tool_name.to_string()));
1694 payload.insert("error".to_string(), Value::String(message.clone()));
1695 self.store.append_event(
1696 &ctx.run_id,
1697 Some(&ctx.session_id),
1698 Some(&ctx.step_id),
1699 "budget_check_failed",
1700 payload,
1701 Some(&ctx.agent_role),
1702 Some(ctx.state_version),
1703 Some(&causal_token),
1704 );
1705 return Err(RuntimeError(message));
1706 }
1707 }
1708 if managed {
1709 let entry = ToolLedgerEntry {
1710 ledger_id: new_id("ledger"),
1711 run_id: ctx.run_id.clone(),
1712 session_id: ctx.session_id.clone(),
1713 step_id: ctx.step_id.clone(),
1714 tool_name: tool_name.to_string(),
1715 tool_version: version,
1716 tool_call_id: new_id("toolcall"),
1717 idempotency_key: idempotency_key.clone(),
1718 causal_token: causal_token.clone(),
1719 request_hash: request_hash.clone(),
1720 request_ref: request_ref.clone(),
1721 status: "RESERVED".to_string(),
1722 external_id: None,
1723 response_hash: None,
1724 response_ref: None,
1725 error_type: None,
1726 response: None,
1727 created_at: now_seconds(),
1728 updated_at: now_seconds(),
1729 };
1730 if let Some(existing) = self.store.reserve_ledger(&idempotency_key, entry) {
1731 if existing.status == "SUCCEEDED" {
1732 let mut payload = State::new();
1733 payload.insert("tool".to_string(), Value::String(tool_name.to_string()));
1734 payload.insert("replayed_from_ledger".to_string(), Value::Bool(true));
1735 self.store.append_event(
1736 &ctx.run_id,
1737 Some(&ctx.session_id),
1738 Some(&ctx.step_id),
1739 "tool_call_completed",
1740 payload,
1741 Some(&ctx.agent_role),
1742 Some(ctx.state_version),
1743 Some(&causal_token),
1744 );
1745 self.store.record_cost(
1746 &ctx.run_id,
1747 &ctx.session_id,
1748 &ctx.step_id,
1749 "tool",
1750 tool_name,
1751 1.0,
1752 "call",
1753 State::new(),
1754 );
1755 return existing
1756 .response
1757 .ok_or_else(|| RuntimeError("missing ledger response".to_string()));
1758 }
1759 return Err(RuntimeError(
1760 "tool side effect already in progress".to_string(),
1761 ));
1762 }
1763 self.store
1764 .update_ledger(&idempotency_key, "RUNNING", None, None);
1765 }
1766 if sandbox_required {
1767 let executor = if sandbox_executor.is_empty() { "default".to_string() } else { sandbox_executor.clone() };
1768 let network = match sandbox_policy.get("network") {
1769 Some(Value::String(value)) => value.clone(),
1770 _ => "deny".to_string(),
1771 };
1772 let filesystem = match sandbox_policy.get("filesystem") {
1773 Some(Value::String(value)) => value.clone(),
1774 _ => "read-only".to_string(),
1775 };
1776 let timeout_seconds = match sandbox_policy.get("timeout_seconds") {
1777 Some(Value::Number(value)) if *value > 0.0 => *value as u64,
1778 _ => 30,
1779 };
1780 let policy = SandboxPolicy {
1781 tool_name: tool_name.to_string(),
1782 run_id: ctx.run_id.clone(),
1783 step_id: ctx.step_id.clone(),
1784 executor: executor.clone(),
1785 network,
1786 filesystem,
1787 timeout_seconds,
1788 extra: sandbox_policy.clone(),
1789 };
1790 let mut sandbox_payload = State::new();
1791 sandbox_payload.insert("tool_name".to_string(), Value::String(tool_name.to_string()));
1792 sandbox_payload.insert("executor".to_string(), Value::String(executor));
1793 sandbox_payload.insert("network".to_string(), Value::String(policy.network.clone()));
1794 sandbox_payload.insert("filesystem".to_string(), Value::String(policy.filesystem.clone()));
1795 sandbox_payload.insert("timeout_seconds".to_string(), Value::Number(policy.timeout_seconds as f64));
1796 self.store.append_event(
1797 &ctx.run_id,
1798 Some(&ctx.session_id),
1799 Some(&ctx.step_id),
1800 "sandbox_started",
1801 sandbox_payload,
1802 Some(&ctx.agent_role),
1803 Some(ctx.state_version),
1804 Some(&causal_token),
1805 );
1806 let result = self.sandbox.run_tool(args, &policy);
1807 let mut completed = State::new();
1808 completed.insert("ok".to_string(), Value::Bool(result.ok));
1809 completed.insert("metadata".to_string(), Value::Object(result.metadata.clone()));
1810 if let Some(error) = &result.error {
1811 completed.insert("error".to_string(), Value::String(error.clone()));
1812 }
1813 self.store.append_event(
1814 &ctx.run_id,
1815 Some(&ctx.session_id),
1816 Some(&ctx.step_id),
1817 "sandbox_completed",
1818 completed,
1819 Some(&ctx.agent_role),
1820 Some(ctx.state_version),
1821 Some(&causal_token),
1822 );
1823 if !result.ok {
1824 let message = result.error.unwrap_or_else(|| "sandboxed tool failed".to_string());
1825 let mut failed = State::new();
1826 failed.insert("tool".to_string(), Value::String(tool_name.to_string()));
1827 failed.insert("error".to_string(), Value::String(message.clone()));
1828 self.store.append_event(
1829 &ctx.run_id,
1830 Some(&ctx.session_id),
1831 Some(&ctx.step_id),
1832 "tool_call_failed",
1833 failed,
1834 Some(&ctx.agent_role),
1835 Some(ctx.state_version),
1836 Some(&causal_token),
1837 );
1838 return Err(RuntimeError(message));
1839 }
1840 let value = result.output;
1841 if let Some(schema) = output_schema.as_ref() {
1842 validate_tool_schema(schema, &value, "$result")?;
1843 }
1844 if managed {
1845 self.store.update_ledger(&idempotency_key, "SUCCEEDED", Some(value.clone()), None);
1846 }
1847 let mut payload = State::new();
1848 payload.insert("tool".to_string(), Value::String(tool_name.to_string()));
1849 payload.insert("idempotency_key".to_string(), Value::String(idempotency_key));
1850 self.store.append_event(
1851 &ctx.run_id,
1852 Some(&ctx.session_id),
1853 Some(&ctx.step_id),
1854 "tool_call_completed",
1855 payload,
1856 Some(&ctx.agent_role),
1857 Some(ctx.state_version),
1858 Some(&causal_token),
1859 );
1860 self.store.record_cost(
1861 &ctx.run_id,
1862 &ctx.session_id,
1863 &ctx.step_id,
1864 "tool",
1865 tool_name,
1866 1.0,
1867 "call",
1868 State::new(),
1869 );
1870 return Ok(value);
1871 }
1872 let result = {
1873 let spec = self.registry.get(tool_name)?;
1874 (spec.func)(args)
1875 };
1876 match result {
1877 Ok(value) => {
1878 if let Some(schema) = output_schema.as_ref() {
1879 if let Err(err) = validate_tool_schema(schema, &value, "$result") {
1880 if managed {
1881 self.store.update_ledger(
1882 &idempotency_key,
1883 "PENDING_VERIFICATION",
1884 None,
1885 Some("ToolOutputValidationError".to_string()),
1886 );
1887 }
1888 let mut failed = State::new();
1889 failed.insert("tool".to_string(), Value::String(tool_name.to_string()));
1890 failed.insert("error".to_string(), Value::String(err.0.clone()));
1891 failed.insert(
1892 "phase".to_string(),
1893 Value::String("output_validation".to_string()),
1894 );
1895 self.store.append_event(
1896 &ctx.run_id,
1897 Some(&ctx.session_id),
1898 Some(&ctx.step_id),
1899 "tool_call_failed",
1900 failed,
1901 Some(&ctx.agent_role),
1902 Some(ctx.state_version),
1903 Some(&causal_token),
1904 );
1905 return Err(err);
1906 }
1907 }
1908 if managed {
1909 self.store.update_ledger(
1910 &idempotency_key,
1911 "SUCCEEDED",
1912 Some(value.clone()),
1913 None,
1914 );
1915 }
1916 let mut payload = State::new();
1917 payload.insert("tool".to_string(), Value::String(tool_name.to_string()));
1918 payload.insert(
1919 "idempotency_key".to_string(),
1920 Value::String(idempotency_key),
1921 );
1922 self.store.append_event(
1923 &ctx.run_id,
1924 Some(&ctx.session_id),
1925 Some(&ctx.step_id),
1926 "tool_call_completed",
1927 payload,
1928 Some(&ctx.agent_role),
1929 Some(ctx.state_version),
1930 Some(&causal_token),
1931 );
1932 self.store.record_cost(
1933 &ctx.run_id,
1934 &ctx.session_id,
1935 &ctx.step_id,
1936 "tool",
1937 tool_name,
1938 1.0,
1939 "call",
1940 State::new(),
1941 );
1942 Ok(value)
1943 }
1944 Err(err) => {
1945 if managed {
1946 self.store.update_ledger(
1947 &idempotency_key,
1948 "PENDING_VERIFICATION",
1949 None,
1950 Some("RuntimeError".to_string()),
1951 );
1952 }
1953 let mut payload = State::new();
1954 payload.insert("tool".to_string(), Value::String(tool_name.to_string()));
1955 payload.insert("error".to_string(), Value::String(err.0.clone()));
1956 self.store.append_event(
1957 &ctx.run_id,
1958 Some(&ctx.session_id),
1959 Some(&ctx.step_id),
1960 "tool_call_failed",
1961 payload,
1962 Some(&ctx.agent_role),
1963 Some(ctx.state_version),
1964 Some(&causal_token),
1965 );
1966 Err(err)
1967 }
1968 }
1969 }
1970
1971 fn record_permission(
1972 &mut self,
1973 ctx: &AgentContext,
1974 tool_name: &str,
1975 allowed: bool,
1976 reason: &str,
1977 causal_token: &str,
1978 ) {
1979 let mut permission = State::new();
1980 permission.insert("tool".to_string(), Value::String(tool_name.to_string()));
1981 permission.insert("allowed".to_string(), Value::Bool(allowed));
1982 permission.insert("reason".to_string(), Value::String(reason.to_string()));
1983 self.store.append_event(
1984 &ctx.run_id,
1985 Some(&ctx.session_id),
1986 Some(&ctx.step_id),
1987 "tool_permission_decided",
1988 permission,
1989 Some(&ctx.agent_role),
1990 Some(ctx.state_version),
1991 Some(causal_token),
1992 );
1993 }
1994
1995 pub fn create_artifact(
1996 &mut self,
1997 ctx: &AgentContext,
1998 name: &str,
1999 content: State,
2000 metadata: State,
2001 ) -> Result<String> {
2002 let artifact =
2003 self.store
2004 .create_artifact(&ctx.run_id, Some(&ctx.step_id), name, content, metadata);
2005 let mut payload = State::new();
2006 payload.insert(
2007 "artifact_id".to_string(),
2008 Value::String(artifact.artifact_id.clone()),
2009 );
2010 payload.insert("name".to_string(), Value::String(name.to_string()));
2011 self.store.append_event(
2012 &ctx.run_id,
2013 Some(&ctx.session_id),
2014 Some(&ctx.step_id),
2015 "artifact_created",
2016 payload,
2017 Some(&ctx.agent_role),
2018 Some(ctx.state_version),
2019 None,
2020 );
2021 Ok(artifact.artifact_id)
2022 }
2023
2024 pub fn create_media_artifact(
2025 &mut self,
2026 ctx: &AgentContext,
2027 name: &str,
2028 kind: &str,
2029 options: MediaArtifactOptions,
2030 ) -> Result<String> {
2031 if !is_media_kind(kind) {
2032 return Err(RuntimeError(format!("unsupported media kind: {kind}")));
2033 }
2034 let mut media_metadata = options.media_metadata.clone();
2035 media_metadata.insert(
2036 "schema_version".to_string(),
2037 Value::String(MEDIA_SCHEMA_VERSION.to_string()),
2038 );
2039 media_metadata.insert("kind".to_string(), Value::String(kind.to_string()));
2040 let mut content = State::new();
2041 content.insert(
2042 "schema_version".to_string(),
2043 Value::String(MEDIA_SCHEMA_VERSION.to_string()),
2044 );
2045 content.insert("kind".to_string(), Value::String(kind.to_string()));
2046 if let Some(uri) = &options.uri {
2047 content.insert("uri".to_string(), Value::String(uri.clone()));
2048 }
2049 if let Some(content_ref) = &options.content_ref {
2050 content.insert(
2051 "content_ref".to_string(),
2052 Value::String(content_ref.clone()),
2053 );
2054 }
2055 content.insert(
2056 "metadata".to_string(),
2057 Value::Object(media_metadata.clone()),
2058 );
2059 if !options.lineage.is_empty() {
2060 content.insert(
2061 "lineage".to_string(),
2062 Value::Object(options.lineage.clone()),
2063 );
2064 }
2065 if !options.derived_outputs.is_empty() {
2066 content.insert(
2067 "derived_outputs".to_string(),
2068 Value::Object(options.derived_outputs.clone()),
2069 );
2070 }
2071 let mut media = State::new();
2072 media.insert(
2073 "schema_version".to_string(),
2074 Value::String(MEDIA_SCHEMA_VERSION.to_string()),
2075 );
2076 media.insert("kind".to_string(), Value::String(kind.to_string()));
2077 if let Some(uri) = options.uri {
2078 media.insert("uri".to_string(), Value::String(uri));
2079 }
2080 if let Some(content_ref) = options.content_ref {
2081 media.insert("content_ref".to_string(), Value::String(content_ref));
2082 }
2083 media.insert("metadata".to_string(), Value::Object(media_metadata));
2084 media.insert("lineage".to_string(), Value::Object(options.lineage));
2085 let mut artifact_metadata = options.metadata;
2086 artifact_metadata.insert("agentledger_media".to_string(), Value::Object(media));
2087 self.create_artifact(ctx, name, content, artifact_metadata)
2088 }
2089
2090 pub fn create_stream_checkpoint(
2091 &mut self,
2092 ctx: &AgentContext,
2093 name: &str,
2094 options: StreamCheckpointOptions,
2095 ) -> Result<String> {
2096 if options.stream_id.is_empty() || options.consumer_id.is_empty() {
2097 return Err(RuntimeError(
2098 "stream_id and consumer_id are required".to_string(),
2099 ));
2100 }
2101 let chunk = options.chunk.map(stream_chunk_to_state);
2102 let mut content = State::new();
2103 content.insert(
2104 "schema_version".to_string(),
2105 Value::String(STREAM_SCHEMA_VERSION.to_string()),
2106 );
2107 content.insert(
2108 "stream_id".to_string(),
2109 Value::String(options.stream_id.clone()),
2110 );
2111 content.insert(
2112 "consumer_id".to_string(),
2113 Value::String(options.consumer_id.clone()),
2114 );
2115 content.insert("offset".to_string(), options.offset.clone());
2116 if let Some(watermark) = &options.watermark {
2117 content.insert("watermark".to_string(), watermark.clone());
2118 }
2119 if let Some(chunk_state) = &chunk {
2120 content.insert("chunk".to_string(), Value::Object(chunk_state.clone()));
2121 }
2122 if let Some(partial_ref) = &options.partial_result_ref {
2123 content.insert(
2124 "partial_result_ref".to_string(),
2125 Value::String(partial_ref.clone()),
2126 );
2127 }
2128 if !options.backpressure.is_empty() {
2129 content.insert(
2130 "backpressure".to_string(),
2131 Value::Object(options.backpressure.clone()),
2132 );
2133 }
2134 if !options.metadata.is_empty() {
2135 content.insert("metadata".to_string(), Value::Object(options.metadata));
2136 }
2137 let mut stream = State::new();
2138 stream.insert(
2139 "schema_version".to_string(),
2140 Value::String(STREAM_SCHEMA_VERSION.to_string()),
2141 );
2142 stream.insert("stream_id".to_string(), Value::String(options.stream_id));
2143 stream.insert(
2144 "consumer_id".to_string(),
2145 Value::String(options.consumer_id),
2146 );
2147 stream.insert("offset".to_string(), options.offset);
2148 if let Some(watermark) = options.watermark {
2149 stream.insert("watermark".to_string(), watermark);
2150 }
2151 if let Some(chunk_state) = chunk {
2152 stream.insert("chunk".to_string(), Value::Object(chunk_state));
2153 }
2154 if let Some(partial_ref) = options.partial_result_ref {
2155 stream.insert("partial_result_ref".to_string(), Value::String(partial_ref));
2156 }
2157 if !options.backpressure.is_empty() {
2158 stream.insert(
2159 "backpressure".to_string(),
2160 Value::Object(options.backpressure),
2161 );
2162 }
2163 let mut artifact_metadata = State::new();
2164 artifact_metadata.insert("agentledger_stream".to_string(), Value::Object(stream));
2165 self.create_artifact(ctx, name, content, artifact_metadata)
2166 }
2167
2168 pub fn record_model_call(
2169 &mut self,
2170 ctx: &AgentContext,
2171 model: &str,
2172 input_tokens: f64,
2173 output_tokens: f64,
2174 total_usd: f64,
2175 ) -> Result<()> {
2176 let mut usage = State::new();
2177 usage.insert("input_tokens".to_string(), Value::Number(input_tokens));
2178 usage.insert("output_tokens".to_string(), Value::Number(output_tokens));
2179 self.record_model_call_evidence(ctx, "custom", model, State::new(), State::new(), usage, total_usd, State::new())
2180 }
2181
2182 pub fn record_model_call_evidence(
2183 &mut self,
2184 ctx: &AgentContext,
2185 provider: &str,
2186 model: &str,
2187 request: State,
2188 response: State,
2189 usage: State,
2190 total_usd: f64,
2191 metadata: State,
2192 ) -> Result<()> {
2193 let tokens = usage_total_tokens(&usage);
2194 if let Some(max) = self.budget.max_model_tokens {
2195 let used = self.store.cost_summary(&ctx.run_id).model_tokens;
2196 if used + tokens > max {
2197 let message = format!("model token budget exceeded: {used}+{tokens}/{max}");
2198 let mut payload = State::new();
2199 payload.insert("category".to_string(), Value::String("model".to_string()));
2200 payload.insert("provider".to_string(), Value::String(provider.to_string()));
2201 payload.insert("model".to_string(), Value::String(model.to_string()));
2202 payload.insert("error".to_string(), Value::String(message.clone()));
2203 self.store.append_event(
2204 &ctx.run_id,
2205 Some(&ctx.session_id),
2206 Some(&ctx.step_id),
2207 "budget_check_failed",
2208 payload,
2209 Some(&ctx.agent_role),
2210 Some(ctx.state_version),
2211 None,
2212 );
2213 return Err(RuntimeError(message));
2214 }
2215 }
2216 let mut request_payload = State::new();
2217 request_payload.insert("schema_version".to_string(), Value::String(MODEL_EVIDENCE_SCHEMA_VERSION.to_string()));
2218 request_payload.insert("provider".to_string(), Value::String(provider.to_string()));
2219 request_payload.insert("model".to_string(), Value::String(model.to_string()));
2220 request_payload.insert("request".to_string(), Value::Object(request));
2221 request_payload.insert("metadata".to_string(), Value::Object(metadata.clone()));
2222 self.store.append_event(
2223 &ctx.run_id,
2224 Some(&ctx.session_id),
2225 Some(&ctx.step_id),
2226 "model_call_requested",
2227 request_payload,
2228 Some(&ctx.agent_role),
2229 Some(ctx.state_version),
2230 None,
2231 );
2232 let mut response_payload = State::new();
2233 response_payload.insert("schema_version".to_string(), Value::String(MODEL_EVIDENCE_SCHEMA_VERSION.to_string()));
2234 response_payload.insert("provider".to_string(), Value::String(provider.to_string()));
2235 response_payload.insert("model".to_string(), Value::String(model.to_string()));
2236 response_payload.insert("response".to_string(), Value::Object(response));
2237 response_payload.insert("usage".to_string(), Value::Object(usage.clone()));
2238 response_payload.insert("total_usd".to_string(), Value::Number(total_usd));
2239 response_payload.insert("metadata".to_string(), Value::Object(metadata));
2240 self.store.append_event(
2241 &ctx.run_id,
2242 Some(&ctx.session_id),
2243 Some(&ctx.step_id),
2244 "model_call_completed",
2245 response_payload,
2246 Some(&ctx.agent_role),
2247 Some(ctx.state_version),
2248 None,
2249 );
2250 self.record_model_costs(ctx, provider, model, &usage, total_usd);
2251 Ok(())
2252 }
2253
2254 pub fn record_model_failure(
2255 &mut self,
2256 ctx: &AgentContext,
2257 provider: &str,
2258 model: &str,
2259 error_type: &str,
2260 message: &str,
2261 retryable: Option<bool>,
2262 request: State,
2263 usage: State,
2264 total_usd: f64,
2265 metadata: State,
2266 ) -> Result<()> {
2267 let mut payload = State::new();
2268 payload.insert("schema_version".to_string(), Value::String(MODEL_EVIDENCE_SCHEMA_VERSION.to_string()));
2269 payload.insert("provider".to_string(), Value::String(provider.to_string()));
2270 payload.insert("model".to_string(), Value::String(model.to_string()));
2271 payload.insert("error_type".to_string(), Value::String(error_type.to_string()));
2272 payload.insert("error".to_string(), Value::String(message.to_string()));
2273 if let Some(value) = retryable {
2274 payload.insert("retryable".to_string(), Value::Bool(value));
2275 }
2276 payload.insert("request".to_string(), Value::Object(request));
2277 payload.insert("usage".to_string(), Value::Object(usage.clone()));
2278 payload.insert("total_usd".to_string(), Value::Number(total_usd));
2279 payload.insert("metadata".to_string(), Value::Object(metadata));
2280 self.store.append_event(
2281 &ctx.run_id,
2282 Some(&ctx.session_id),
2283 Some(&ctx.step_id),
2284 "model_call_failed",
2285 payload,
2286 Some(&ctx.agent_role),
2287 Some(ctx.state_version),
2288 None,
2289 );
2290 self.record_model_costs(ctx, provider, model, &usage, total_usd);
2291 Ok(())
2292 }
2293
2294 pub fn record_tool_call_proposal(
2295 &mut self,
2296 ctx: &AgentContext,
2297 tool_name: &str,
2298 arguments: State,
2299 provider: Option<&str>,
2300 model: Option<&str>,
2301 model_call_ref: Option<&str>,
2302 confidence: Option<f64>,
2303 reason: Option<&str>,
2304 metadata: State,
2305 ) {
2306 let mut payload = State::new();
2307 payload.insert("schema_version".to_string(), Value::String(MODEL_EVIDENCE_SCHEMA_VERSION.to_string()));
2308 payload.insert("tool".to_string(), Value::String(tool_name.to_string()));
2309 payload.insert("args".to_string(), Value::Object(arguments));
2310 if let Some(value) = provider {
2311 payload.insert("provider".to_string(), Value::String(value.to_string()));
2312 }
2313 if let Some(value) = model {
2314 payload.insert("model".to_string(), Value::String(value.to_string()));
2315 }
2316 if let Some(value) = model_call_ref {
2317 payload.insert("model_call_ref".to_string(), Value::String(value.to_string()));
2318 }
2319 if let Some(value) = confidence {
2320 payload.insert("confidence".to_string(), Value::Number(value));
2321 }
2322 if let Some(value) = reason {
2323 payload.insert("reason".to_string(), Value::String(value.to_string()));
2324 }
2325 payload.insert("metadata".to_string(), Value::Object(metadata));
2326 self.store.append_event(
2327 &ctx.run_id,
2328 Some(&ctx.session_id),
2329 Some(&ctx.step_id),
2330 "tool_call_proposed",
2331 payload,
2332 Some(&ctx.agent_role),
2333 Some(ctx.state_version),
2334 None,
2335 );
2336 }
2337
2338 fn record_model_costs(&mut self, ctx: &AgentContext, provider: &str, model: &str, usage: &State, total_usd: f64) {
2339 let tokens = usage_total_tokens(usage);
2340 let mut metadata = State::new();
2341 metadata.insert("provider".to_string(), Value::String(provider.to_string()));
2342 metadata.insert("model".to_string(), Value::String(model.to_string()));
2343 metadata.insert("usage".to_string(), Value::Object(usage.clone()));
2344 if tokens > 0.0 {
2345 self.store.record_cost(&ctx.run_id, &ctx.session_id, &ctx.step_id, "model", model, tokens, "token", metadata.clone());
2346 }
2347 if total_usd > 0.0 {
2348 self.store.record_cost(&ctx.run_id, &ctx.session_id, &ctx.step_id, "model", model, total_usd, "usd", metadata);
2349 }
2350 }
2351}
2352
2353impl Default for Runtime {
2354 fn default() -> Self {
2355 Self::new()
2356 }
2357}
2358
2359#[derive(Clone, Debug)]
2360pub struct AgentContext {
2361 pub run_id: String,
2362 pub session_id: String,
2363 pub step_id: String,
2364 pub agent_role: String,
2365 pub lease_token: String,
2366 pub attempt: u64,
2367 pub state_version: u64,
2368 pub pending_patch: State,
2369}
2370
2371impl AgentContext {
2372 pub fn write_state(&mut self, key: &str, value: Value) {
2373 self.pending_patch.insert(key.to_string(), value);
2374 }
2375}
2376
2377pub const OMP_ADAPTER_SCHEMA_VERSION: &str = "agentledger.omp.adapter.v1";
2378
2379#[derive(Clone, Debug, Default)]
2380pub struct OmpSession {
2381 pub session_id: String,
2382 pub initial_state: State,
2383 pub metadata: State,
2384 pub run_id: Option<String>,
2385}
2386
2387#[derive(Clone, Debug, Default)]
2388pub struct OmpTurn {
2389 pub session_id: String,
2390 pub turn_id: String,
2391 pub agent_role: String,
2392 pub state_patch: State,
2393 pub metadata: State,
2394}
2395
2396#[derive(Clone, Debug, Default)]
2397pub struct OmpModelCall {
2398 pub session_id: String,
2399 pub turn_id: String,
2400 pub provider: String,
2401 pub model: String,
2402 pub request: State,
2403 pub response: State,
2404 pub usage: State,
2405 pub total_usd: f64,
2406 pub metadata: State,
2407}
2408
2409#[derive(Clone, Debug, Default)]
2410pub struct OmpToolProposal {
2411 pub session_id: String,
2412 pub turn_id: String,
2413 pub tool_name: String,
2414 pub arguments: State,
2415 pub provider: Option<String>,
2416 pub model: Option<String>,
2417 pub model_call_ref: Option<String>,
2418 pub confidence: Option<f64>,
2419 pub reason: Option<String>,
2420 pub metadata: State,
2421}
2422
2423#[derive(Clone, Debug)]
2424pub struct OmpToolExecution {
2425 pub session_id: String,
2426 pub turn_id: String,
2427 pub tool_name: String,
2428 pub arguments: State,
2429 pub result: Option<Value>,
2430 pub tool_call_id: Option<String>,
2431 pub tool_version: String,
2432 pub idempotency_key: Option<String>,
2433 pub ledger_status: Option<String>,
2434 pub error_type: Option<String>,
2435 pub error_message: Option<String>,
2436 pub external_id: Option<String>,
2437 pub causal_token: Option<String>,
2438 pub metadata: State,
2439}
2440
2441impl Default for OmpToolExecution {
2442 fn default() -> Self {
2443 Self {
2444 session_id: String::new(),
2445 turn_id: String::new(),
2446 tool_name: String::new(),
2447 arguments: State::new(),
2448 result: None,
2449 tool_call_id: None,
2450 tool_version: "external".to_string(),
2451 idempotency_key: None,
2452 ledger_status: None,
2453 error_type: None,
2454 error_message: None,
2455 external_id: None,
2456 causal_token: None,
2457 metadata: State::new(),
2458 }
2459 }
2460}
2461
2462#[derive(Clone, Debug)]
2463pub struct OmpFailure {
2464 pub session_id: String,
2465 pub turn_id: String,
2466 pub error_type: String,
2467 pub message: String,
2468 pub retryable: Option<bool>,
2469 pub status: String,
2470 pub terminal: Option<bool>,
2471 pub category: String,
2472 pub provider: Option<String>,
2473 pub model: Option<String>,
2474 pub request: State,
2475 pub usage: State,
2476 pub total_usd: f64,
2477 pub metadata: State,
2478 pub approval_id: Option<String>,
2479}
2480
2481impl Default for OmpFailure {
2482 fn default() -> Self {
2483 Self {
2484 session_id: String::new(),
2485 turn_id: String::new(),
2486 error_type: String::new(),
2487 message: String::new(),
2488 retryable: None,
2489 status: "failed".to_string(),
2490 terminal: None,
2491 category: "runtime".to_string(),
2492 provider: None,
2493 model: None,
2494 request: State::new(),
2495 usage: State::new(),
2496 total_usd: 0.0,
2497 metadata: State::new(),
2498 approval_id: None,
2499 }
2500 }
2501}
2502
2503#[derive(Clone, Debug)]
2504pub struct OmpStateChange {
2505 pub session_id: String,
2506 pub turn_id: Option<String>,
2507 pub reason: String,
2508 pub patch: State,
2509 pub label: String,
2510 pub commit_status: String,
2511 pub before_snapshot: Option<Value>,
2512 pub after_snapshot: Option<Value>,
2513 pub diff: Option<Value>,
2514 pub metadata: State,
2515}
2516
2517impl Default for OmpStateChange {
2518 fn default() -> Self {
2519 Self {
2520 session_id: String::new(),
2521 turn_id: None,
2522 reason: String::new(),
2523 patch: State::new(),
2524 label: "state".to_string(),
2525 commit_status: "committed".to_string(),
2526 before_snapshot: None,
2527 after_snapshot: None,
2528 diff: None,
2529 metadata: State::new(),
2530 }
2531 }
2532}
2533
2534#[derive(Clone, Debug)]
2535struct OmpBridgeSession {
2536 run_id: String,
2537 session_id: String,
2538 initial_step_id: Option<String>,
2539}
2540
2541#[derive(Clone, Debug)]
2542struct OmpActiveTurn {
2543 run_id: String,
2544 session_id: String,
2545 step_id: String,
2546 lease_token: String,
2547 attempt: u64,
2548 state_version: u64,
2549 agent_role: String,
2550}
2551
2552pub struct OmpLedgerBridge {
2553 pub runtime: Runtime,
2554 pub app_name: String,
2555 pub worker_id: String,
2556 pub lease_seconds: f64,
2557 sessions: HashMap<String, OmpBridgeSession>,
2558 active_turns: HashMap<String, OmpActiveTurn>,
2559}
2560
2561impl OmpLedgerBridge {
2562 pub fn new(runtime: Runtime, app_name: &str) -> Self {
2563 let app = if app_name.is_empty() { "omp" } else { app_name };
2564 Self {
2565 runtime,
2566 app_name: app.to_string(),
2567 worker_id: format!("omp:{app}"),
2568 lease_seconds: 60.0,
2569 sessions: HashMap::new(),
2570 active_turns: HashMap::new(),
2571 }
2572 }
2573
2574 pub fn record_session_started(&mut self, session: OmpSession) -> Result<String> {
2575 if let Some(existing) = self.sessions.get(&session.session_id) {
2576 return Ok(existing.run_id.clone());
2577 }
2578 let bridge_session = self.ensure_session(session.clone())?;
2579 let run = self.runtime.store.run(&bridge_session.run_id)?;
2580 let payload = compact_state([
2581 ("schema_version", OMP_ADAPTER_SCHEMA_VERSION.into()),
2582 ("adapter", "omp-ledger-bridge".into()),
2583 ("app_name", self.app_name.clone().into()),
2584 ("external_session_id", session.session_id.clone().into()),
2585 ("metadata", Value::Object(session.metadata)),
2586 ]);
2587 self.runtime.store.append_event(
2588 &bridge_session.run_id,
2589 Some(&bridge_session.session_id),
2590 None,
2591 "omp_session_started",
2592 payload,
2593 None,
2594 Some(run.state_version),
2595 None,
2596 );
2597 Ok(bridge_session.run_id)
2598 }
2599
2600 pub fn record_turn_started(&mut self, turn: OmpTurn) -> Result<String> {
2601 let key = self.turn_key(&turn.session_id, &turn.turn_id);
2602 if let Some(active) = self.active_turns.get(&key) {
2603 return Ok(active.step_id.clone());
2604 }
2605 let mut session = self.ensure_session(OmpSession {
2606 session_id: turn.session_id.clone(),
2607 ..Default::default()
2608 })?;
2609 if session.initial_step_id.is_some() {
2610 session.initial_step_id = None;
2611 self.sessions.insert(turn.session_id.clone(), session.clone());
2612 } else if self.next_runnable_step_id(&session.run_id).is_none() {
2613 self.runtime.store.create_external_step(&session.run_id)?;
2614 }
2615 let claim = self.runtime.store.claim_step(&self.worker_id, &session.run_id, self.lease_seconds)?;
2616 let role = if turn.agent_role.is_empty() { "OMPAgent".to_string() } else { turn.agent_role.clone() };
2617 let active = OmpActiveTurn {
2618 run_id: claim.run_id.clone(),
2619 session_id: claim.session_id.clone(),
2620 step_id: claim.step_id.clone(),
2621 lease_token: claim.lease_token,
2622 attempt: claim.attempt,
2623 state_version: claim.state_version,
2624 agent_role: role.clone(),
2625 };
2626 self.active_turns.insert(key, active.clone());
2627 let payload = compact_state([
2628 ("schema_version", OMP_ADAPTER_SCHEMA_VERSION.into()),
2629 ("adapter", "omp-ledger-bridge".into()),
2630 ("app_name", self.app_name.clone().into()),
2631 ("external_session_id", turn.session_id.clone().into()),
2632 ("external_turn_id", turn.turn_id.clone().into()),
2633 ("metadata", Value::Object(turn.metadata)),
2634 ]);
2635 self.runtime.store.append_event(
2636 &active.run_id,
2637 Some(&active.session_id),
2638 Some(&active.step_id),
2639 "omp_turn_started",
2640 payload,
2641 Some(&active.agent_role),
2642 Some(active.state_version),
2643 None,
2644 );
2645 Ok(active.step_id)
2646 }
2647
2648 pub fn record_turn_completed(&mut self, turn: OmpTurn) -> Result<u64> {
2649 let active = self.require_turn(&turn.session_id, &turn.turn_id)?;
2650 let next_version = self.runtime.store.commit_state_patch(
2651 &active.run_id,
2652 &active.step_id,
2653 &active.lease_token,
2654 active.state_version,
2655 turn.state_patch,
2656 )?;
2657 let payload = compact_state([
2658 ("schema_version", OMP_ADAPTER_SCHEMA_VERSION.into()),
2659 ("adapter", "omp-ledger-bridge".into()),
2660 ("app_name", self.app_name.clone().into()),
2661 ("external_session_id", turn.session_id.clone().into()),
2662 ("external_turn_id", turn.turn_id.clone().into()),
2663 ("metadata", Value::Object(turn.metadata)),
2664 ]);
2665 self.runtime.store.append_event(
2666 &active.run_id,
2667 Some(&active.session_id),
2668 Some(&active.step_id),
2669 "omp_turn_completed",
2670 payload,
2671 Some(&active.agent_role),
2672 Some(next_version),
2673 None,
2674 );
2675 let key = self.turn_key(&turn.session_id, &turn.turn_id);
2676 self.active_turns.remove(&key);
2677 Ok(next_version)
2678 }
2679
2680 pub fn record_model_call(&mut self, record: OmpModelCall) -> Result<()> {
2681 let ctx = self.context_for(&record.session_id, &record.turn_id)?;
2682 self.runtime.record_model_call_evidence(
2683 &ctx,
2684 &record.provider,
2685 &record.model,
2686 record.request,
2687 record.response,
2688 record.usage,
2689 record.total_usd,
2690 record.metadata,
2691 )
2692 }
2693
2694 pub fn record_tool_proposal(&mut self, proposal: OmpToolProposal) -> Result<()> {
2695 let ctx = self.context_for(&proposal.session_id, &proposal.turn_id)?;
2696 self.runtime.record_tool_call_proposal(
2697 &ctx,
2698 &proposal.tool_name,
2699 proposal.arguments,
2700 proposal.provider.as_deref(),
2701 proposal.model.as_deref(),
2702 proposal.model_call_ref.as_deref(),
2703 proposal.confidence,
2704 proposal.reason.as_deref(),
2705 proposal.metadata,
2706 );
2707 Ok(())
2708 }
2709
2710 pub fn record_tool_execution(&mut self, execution: OmpToolExecution) -> Result<State> {
2711 let active = self.require_turn(&execution.session_id, &execution.turn_id)?;
2712 let tool_call_id = execution.tool_call_id.clone().unwrap_or_else(|| new_id("toolcall"));
2713 let tool_version = if execution.tool_version.is_empty() { "external".to_string() } else { execution.tool_version.clone() };
2714 let request = compact_state([
2715 ("schema_version", OMP_ADAPTER_SCHEMA_VERSION.into()),
2716 ("tool", execution.tool_name.clone().into()),
2717 ("args", Value::Object(execution.arguments.clone())),
2718 ("tool_call_id", tool_call_id.clone().into()),
2719 ("metadata", Value::Object(execution.metadata.clone())),
2720 ]);
2721 let request_ref = format_state(&request);
2722 let request_hash = stable_hash(&request_ref);
2723 let idempotency_key = execution.idempotency_key.clone().unwrap_or_else(|| {
2724 format!(
2725 "omp:{}:{}:{}:{}",
2726 execution.session_id, execution.turn_id, execution.tool_name, tool_call_id
2727 )
2728 });
2729 let causal_token = execution.causal_token.clone().unwrap_or_else(|| format!("omp:{}:{}:{}", execution.session_id, execution.turn_id, tool_call_id));
2730 self.runtime.store.append_event(
2731 &active.run_id,
2732 Some(&active.session_id),
2733 Some(&active.step_id),
2734 "tool_call_requested",
2735 request,
2736 Some(&active.agent_role),
2737 Some(active.state_version),
2738 Some(&causal_token),
2739 );
2740 let now = now_seconds();
2741 let entry = ToolLedgerEntry {
2742 ledger_id: new_id("ledger"),
2743 run_id: active.run_id.clone(),
2744 session_id: active.session_id.clone(),
2745 step_id: active.step_id.clone(),
2746 tool_name: execution.tool_name.clone(),
2747 tool_version,
2748 tool_call_id: tool_call_id.clone(),
2749 idempotency_key: idempotency_key.clone(),
2750 causal_token: causal_token.clone(),
2751 request_hash,
2752 request_ref,
2753 status: "RESERVED".to_string(),
2754 external_id: None,
2755 response_hash: None,
2756 response_ref: None,
2757 error_type: None,
2758 response: None,
2759 created_at: now,
2760 updated_at: now,
2761 };
2762 if let Some(existing) = self.runtime.store.reserve_ledger(&idempotency_key, entry) {
2763 if existing.status == "SUCCEEDED" {
2764 let mut payload = State::new();
2765 payload.insert("tool".into(), execution.tool_name.clone().into());
2766 payload.insert("replayed_from_ledger".into(), Value::Bool(true));
2767 payload.insert("idempotency_key".into(), idempotency_key.clone().into());
2768 payload.insert("tool_call_id".into(), tool_call_id.clone().into());
2769 self.runtime.store.append_event(
2770 &active.run_id,
2771 Some(&active.session_id),
2772 Some(&active.step_id),
2773 "tool_call_completed",
2774 payload,
2775 Some(&active.agent_role),
2776 Some(active.state_version),
2777 Some(&causal_token),
2778 );
2779 return Ok(compact_state([
2780 ("ledger_status", "SUCCEEDED".into()),
2781 ("replayed_from_ledger", Value::Bool(true)),
2782 ("idempotency_key", idempotency_key.into()),
2783 ("tool_call_id", tool_call_id.into()),
2784 ]));
2785 }
2786 if existing.status == "PENDING_VERIFICATION" {
2787 return Err(RuntimeError("tool side effect pending verification".to_string()));
2788 }
2789 if existing.status == "RESERVED" || existing.status == "RUNNING" {
2790 return Err(RuntimeError("tool side effect already in progress".to_string()));
2791 }
2792 }
2793 let ledger_status = normalize_omp_ledger_status(
2794 execution.ledger_status.as_deref(),
2795 execution.error_message.as_deref().or(execution.error_type.as_deref()),
2796 )?;
2797 let (response_hash, response_ref) = execution.result.as_ref().map(|value| {
2798 let response_ref = format_value(value);
2799 (Some(stable_hash(&response_ref)), Some(response_ref))
2800 }).unwrap_or((None, None));
2801 self.runtime.store.update_ledger(
2802 &idempotency_key,
2803 &ledger_status,
2804 execution.result.clone(),
2805 execution.error_type.clone(),
2806 );
2807 if let Some(entry) = self.runtime.store.tool_ledger.get_mut(&idempotency_key) {
2808 entry.external_id = execution.external_id.clone().or_else(|| external_id_from_value(execution.result.as_ref()));
2809 entry.response_hash = response_hash.clone();
2810 entry.response_ref = response_ref.clone();
2811 }
2812 let event_type = if ledger_status == "SUCCEEDED" || ledger_status == "COMPENSATED" { "tool_call_completed" } else { "tool_call_failed" };
2813 let payload = compact_state([
2814 ("tool", execution.tool_name.clone().into()),
2815 ("tool_call_id", tool_call_id.clone().into()),
2816 ("idempotency_key", idempotency_key.clone().into()),
2817 ("ledger_status", ledger_status.clone().into()),
2818 ("error", execution.error_message.unwrap_or_default().into()),
2819 ("error_type", execution.error_type.unwrap_or_default().into()),
2820 ]);
2821 self.runtime.store.append_event(
2822 &active.run_id,
2823 Some(&active.session_id),
2824 Some(&active.step_id),
2825 event_type,
2826 payload,
2827 Some(&active.agent_role),
2828 Some(active.state_version),
2829 Some(&causal_token),
2830 );
2831 if ledger_status != "RESERVED" && ledger_status != "RUNNING" {
2832 let mut metadata = State::new();
2833 metadata.insert("external_runtime".into(), "omp".into());
2834 metadata.insert("ledger_status".into(), ledger_status.clone().into());
2835 self.runtime.store.record_cost(&active.run_id, &active.session_id, &active.step_id, "tool", &execution.tool_name, 1.0, "call", metadata);
2836 }
2837 Ok(compact_state([
2838 ("ledger_status", ledger_status.into()),
2839 ("idempotency_key", idempotency_key.into()),
2840 ("tool_call_id", tool_call_id.into()),
2841 ]))
2842 }
2843
2844 pub fn record_failure(&mut self, failure: OmpFailure) -> Result<()> {
2845 let active = self.require_turn(&failure.session_id, &failure.turn_id)?;
2846 if failure.category == "model" {
2847 let ctx = self.context_for(&failure.session_id, &failure.turn_id)?;
2848 self.runtime.record_model_failure(
2849 &ctx,
2850 failure.provider.as_deref().unwrap_or("custom"),
2851 failure.model.as_deref().unwrap_or("unknown"),
2852 &failure.error_type,
2853 &failure.message,
2854 failure.retryable,
2855 failure.request,
2856 failure.usage,
2857 failure.total_usd,
2858 failure.metadata,
2859 )?;
2860 }
2861 if failure.terminal == Some(false) {
2862 return Ok(());
2863 }
2864 let status = if failure.status.is_empty() { "failed".to_string() } else { failure.status.to_lowercase() };
2865 if status == "waiting_human" || status == "approval_required" {
2866 self.runtime.store.mark_waiting_human(&active.run_id, &active.step_id, &failure.message, failure.approval_id.as_deref().unwrap_or(""));
2867 } else if status == "retry_scheduled" || status == "retry" || failure.retryable == Some(true) {
2868 self.runtime.store.mark_retry(&active.run_id, &active.step_id, &failure.error_type, &failure.message);
2869 } else {
2870 self.runtime.store.mark_failed(&active.run_id, &active.step_id, &failure.error_type, &failure.message);
2871 }
2872 let key = self.turn_key(&failure.session_id, &failure.turn_id);
2873 self.active_turns.remove(&key);
2874 Ok(())
2875 }
2876
2877 pub fn record_state_change(&mut self, change: OmpStateChange) -> Result<u64> {
2878 let session = self.ensure_session(OmpSession { session_id: change.session_id.clone(), ..Default::default() })?;
2879 let active_key = change.turn_id.as_ref().map(|turn_id| self.turn_key(&change.session_id, turn_id));
2880 let active = active_key.as_ref().and_then(|key| self.active_turns.get(key)).cloned();
2881 let mut artifacts = State::new();
2882 let label = if change.label.is_empty() { "state".to_string() } else { change.label.clone() };
2883 if let Some(value) = change.before_snapshot.clone() {
2884 let id = self.store_artifact(
2885 &session.run_id,
2886 active.as_ref().map(|item| item.step_id.as_str()),
2887 &format!("omp-{label}-before"),
2888 value,
2889 compact_state([
2890 ("schema_version", OMP_ADAPTER_SCHEMA_VERSION.into()),
2891 ("kind", "before_snapshot".into()),
2892 ("external_session_id", change.session_id.clone().into()),
2893 ]),
2894 active.as_ref(),
2895 )?;
2896 artifacts.insert("before_artifact_id".into(), id.into());
2897 }
2898 if let Some(value) = change.after_snapshot.clone() {
2899 let id = self.store_artifact(
2900 &session.run_id,
2901 active.as_ref().map(|item| item.step_id.as_str()),
2902 &format!("omp-{label}-after"),
2903 value,
2904 compact_state([
2905 ("schema_version", OMP_ADAPTER_SCHEMA_VERSION.into()),
2906 ("kind", "after_snapshot".into()),
2907 ("external_session_id", change.session_id.clone().into()),
2908 ]),
2909 active.as_ref(),
2910 )?;
2911 artifacts.insert("after_artifact_id".into(), id.into());
2912 }
2913 if let Some(value) = change.diff.clone() {
2914 let id = self.store_artifact(
2915 &session.run_id,
2916 active.as_ref().map(|item| item.step_id.as_str()),
2917 &format!("omp-{label}-diff"),
2918 value,
2919 compact_state([
2920 ("schema_version", OMP_ADAPTER_SCHEMA_VERSION.into()),
2921 ("kind", "diff".into()),
2922 ("external_session_id", change.session_id.clone().into()),
2923 ]),
2924 active.as_ref(),
2925 )?;
2926 artifacts.insert("diff_artifact_id".into(), id.into());
2927 }
2928 let mut version = self.runtime.store.run(&session.run_id)?.state_version;
2929 let commit_status = if change.commit_status.is_empty() { "committed".to_string() } else { change.commit_status.clone() };
2930 if !change.patch.is_empty() && (commit_status == "committed" || commit_status == "applied") {
2931 version = self.runtime.store.apply_system_state_patch(&session.run_id, change.patch.clone(), &change.reason)?;
2932 if let Some(key) = active_key.as_ref() {
2933 if let Some(item) = self.active_turns.get_mut(key) {
2934 item.state_version = version;
2935 }
2936 }
2937 }
2938 let payload = compact_state([
2939 ("schema_version", OMP_ADAPTER_SCHEMA_VERSION.into()),
2940 ("adapter", "omp-ledger-bridge".into()),
2941 ("app_name", self.app_name.clone().into()),
2942 ("external_session_id", change.session_id.clone().into()),
2943 ("external_turn_id", change.turn_id.clone().unwrap_or_default().into()),
2944 ("reason", change.reason.into()),
2945 ("commit_status", commit_status.into()),
2946 ("patch", Value::Object(change.patch)),
2947 ("artifacts", Value::Object(artifacts)),
2948 ("metadata", Value::Object(change.metadata)),
2949 ]);
2950 let active_after = active_key.as_ref().and_then(|key| self.active_turns.get(key)).cloned();
2951 self.runtime.store.append_event(
2952 &session.run_id,
2953 Some(&session.session_id),
2954 active_after.as_ref().map(|item| item.step_id.as_str()),
2955 "omp_state_change_recorded",
2956 payload,
2957 active_after.as_ref().map(|item| item.agent_role.as_str()),
2958 Some(version),
2959 None,
2960 );
2961 Ok(version)
2962 }
2963
2964 fn ensure_session(&mut self, session: OmpSession) -> Result<OmpBridgeSession> {
2965 if let Some(existing) = self.sessions.get(&session.session_id) {
2966 return Ok(existing.clone());
2967 }
2968 let bridge_session = if let Some(run_id) = session.run_id.clone() {
2969 let run = self.runtime.store.run(&run_id)?;
2970 let initial_step_id = self
2971 .runtime
2972 .store
2973 .steps(&run_id)
2974 .into_iter()
2975 .find(|step| step.status == "pending" || step.status == "retry_scheduled")
2976 .map(|step| step.step_id);
2977 OmpBridgeSession { run_id, session_id: run.session_id, initial_step_id }
2978 } else {
2979 let (run_id, step_id) = self.runtime.create_run(session.initial_state);
2980 let run = self.runtime.store.run(&run_id)?;
2981 OmpBridgeSession { run_id, session_id: run.session_id, initial_step_id: Some(step_id) }
2982 };
2983 self.sessions.insert(session.session_id, bridge_session.clone());
2984 Ok(bridge_session)
2985 }
2986
2987 fn require_turn(&self, session_id: &str, turn_id: &str) -> Result<OmpActiveTurn> {
2988 self.active_turns
2989 .get(&self.turn_key(session_id, turn_id))
2990 .cloned()
2991 .ok_or_else(|| RuntimeError(format!("OMP turn not active: {session_id}/{turn_id}")))
2992 }
2993
2994 fn context_for(&self, session_id: &str, turn_id: &str) -> Result<AgentContext> {
2995 let active = self.require_turn(session_id, turn_id)?;
2996 Ok(AgentContext {
2997 run_id: active.run_id,
2998 session_id: active.session_id,
2999 step_id: active.step_id,
3000 agent_role: active.agent_role,
3001 lease_token: active.lease_token,
3002 attempt: active.attempt,
3003 state_version: active.state_version,
3004 pending_patch: State::new(),
3005 })
3006 }
3007
3008 fn store_artifact(
3009 &mut self,
3010 run_id: &str,
3011 step_id: Option<&str>,
3012 name: &str,
3013 content: Value,
3014 metadata: State,
3015 active: Option<&OmpActiveTurn>,
3016 ) -> Result<String> {
3017 let mut artifact_content = State::new();
3018 artifact_content.insert("value".to_string(), content);
3019 let artifact = self.runtime.store.create_artifact(run_id, step_id, name, artifact_content, metadata);
3020 let run = self.runtime.store.run(run_id)?;
3021 let mut payload = State::new();
3022 payload.insert("artifact_id".to_string(), artifact.artifact_id.clone().into());
3023 payload.insert("name".to_string(), name.into());
3024 self.runtime.store.append_event(
3025 run_id,
3026 Some(&run.session_id),
3027 step_id,
3028 "artifact_created",
3029 payload,
3030 active.map(|item| item.agent_role.as_str()),
3031 Some(active.map(|item| item.state_version).unwrap_or(run.state_version)),
3032 None,
3033 );
3034 Ok(artifact.artifact_id)
3035 }
3036
3037 fn turn_key(&self, session_id: &str, turn_id: &str) -> String {
3038 format!("{session_id}\x1f{turn_id}")
3039 }
3040
3041 fn next_runnable_step_id(&self, run_id: &str) -> Option<String> {
3042 self.runtime
3043 .store
3044 .steps(run_id)
3045 .into_iter()
3046 .find(|step| step.status == "pending" || step.status == "retry_scheduled")
3047 .map(|step| step.step_id)
3048 }
3049}
3050
3051fn compact_state<const N: usize>(items: [(&str, Value); N]) -> State {
3052 let mut out = State::new();
3053 for (key, value) in items {
3054 match &value {
3055 Value::Null => continue,
3056 Value::String(item) if item.is_empty() => continue,
3057 Value::Object(item) if item.is_empty() => continue,
3058 _ => {}
3059 }
3060 out.insert(key.to_string(), value);
3061 }
3062 out
3063}
3064
3065fn normalize_omp_ledger_status(value: Option<&str>, error_message: Option<&str>) -> Result<String> {
3066 let raw = value.unwrap_or("");
3067 if raw.is_empty() {
3068 return Ok(if error_message.unwrap_or("").is_empty() { "SUCCEEDED" } else { "PENDING_VERIFICATION" }.to_string());
3069 }
3070 let normalized = raw.trim().to_ascii_uppercase();
3071 let status = match normalized.as_str() {
3072 "SUCCESS" | "SUCCEEDED" | "COMPLETED" | "OK" => "SUCCEEDED",
3073 "FAILED" | "PENDING_VERIFICATION" | "UNKNOWN" => "PENDING_VERIFICATION",
3074 "FAILED_NO_EFFECT" | "NO_EFFECT" => "FAILED_NO_EFFECT",
3075 "COMPENSATED" => "COMPENSATED",
3076 "RUNNING" => "RUNNING",
3077 "RESERVED" => "RESERVED",
3078 other => other,
3079 };
3080 match status {
3081 "SUCCEEDED" | "COMPENSATED" | "FAILED_NO_EFFECT" | "PENDING_VERIFICATION" | "RESERVED" | "RUNNING" => Ok(status.to_string()),
3082 _ => Err(RuntimeError(format!("unsupported Tool Ledger status: {raw}"))),
3083 }
3084}
3085
3086fn external_id_from_value(value: Option<&Value>) -> Option<String> {
3087 match value {
3088 Some(Value::Object(item)) => item.get("external_id").map(format_value),
3089 _ => None,
3090 }
3091}
3092
3093#[derive(Clone, Debug)]
3094pub struct EvidenceBundle {
3095 pub schema_version: String,
3096 pub bundle_hash: String,
3097 pub run: Run,
3098 pub steps: Vec<Step>,
3099 pub events: Vec<Event>,
3100 pub tool_ledger: Vec<ToolLedgerEntry>,
3101 pub approvals: Vec<ApprovalRequest>,
3102 pub artifacts: Vec<Artifact>,
3103 pub media_artifacts: Vec<State>,
3104 pub stream_checkpoints: Vec<State>,
3105 pub cost_records: Vec<CostRecord>,
3106 pub cost_summary: CostSummary,
3107 pub summary: State,
3108 pub final_state: State,
3109}
3110
3111#[derive(Clone, Debug, Default)]
3112pub struct WorkerRunSummary {
3113 pub worker_id: String,
3114 pub run_id: Option<String>,
3115 pub iterations: u64,
3116 pub attempts: u64,
3117 pub succeeded_attempts: u64,
3118 pub recovered_leases: u64,
3119 pub final_status: Option<String>,
3120 pub stopped_reason: String,
3121}
3122
3123pub struct LocalWorker {
3124 pub worker_id: String,
3125 pub agent_role: String,
3126 pub lease_seconds: f64,
3127 pub recover_expired: bool,
3128}
3129
3130impl LocalWorker {
3131 pub fn new(worker_id: &str, agent_role: &str) -> Self {
3132 Self {
3133 worker_id: worker_id.to_string(),
3134 agent_role: agent_role.to_string(),
3135 lease_seconds: 60.0,
3136 recover_expired: true,
3137 }
3138 }
3139
3140 pub fn run_until_idle(
3141 &self,
3142 runtime: &mut Runtime,
3143 run_id: &str,
3144 max_iterations: u64,
3145 agent: AgentFunc,
3146 ) -> Result<WorkerRunSummary> {
3147 let mut summary = WorkerRunSummary {
3148 worker_id: self.worker_id.clone(),
3149 run_id: Some(run_id.to_string()),
3150 stopped_reason: "max_iterations".to_string(),
3151 ..WorkerRunSummary::default()
3152 };
3153 for index in 1..=max_iterations {
3154 summary.iterations = index;
3155 if self.recover_expired {
3156 summary.recovered_leases += runtime.store.recover_expired_leases() as u64;
3157 }
3158 let status = runtime.store.run(run_id)?.status;
3159 if is_terminal_status(&status) {
3160 summary.final_status = Some(status);
3161 summary.stopped_reason = "terminal_status".to_string();
3162 break;
3163 }
3164 let ok = runtime.run_once(
3165 run_id,
3166 &self.worker_id,
3167 &self.agent_role,
3168 self.lease_seconds,
3169 agent,
3170 )?;
3171 if !ok {
3172 summary.stopped_reason = "idle".to_string();
3173 break;
3174 }
3175 summary.attempts += 1;
3176 summary.succeeded_attempts += 1;
3177 }
3178 let status = runtime.store.run(run_id)?.status;
3179 summary.final_status = Some(status.clone());
3180 if is_terminal_status(&status) {
3181 summary.stopped_reason = "terminal_status".to_string();
3182 }
3183 Ok(summary)
3184 }
3185}
3186
3187#[derive(Clone, Debug, Default)]
3188pub struct WorkerServiceSummary {
3189 pub worker_id: String,
3190 pub run_id: Option<String>,
3191 pub loops: u64,
3192 pub attempts: u64,
3193 pub succeeded_attempts: u64,
3194 pub recovered_leases: u64,
3195 pub idle_polls: u64,
3196 pub stopped_reason: String,
3197 pub final_status: Option<String>,
3198 pub stop_requested: bool,
3199}
3200
3201pub struct WorkerService {
3202 pub worker: LocalWorker,
3203 pub stop_requested: bool,
3204 pub stop_reason: String,
3205}
3206
3207impl WorkerService {
3208 pub fn new(worker: LocalWorker) -> Self {
3209 Self {
3210 worker,
3211 stop_requested: false,
3212 stop_reason: "stop_requested".to_string(),
3213 }
3214 }
3215
3216 pub fn request_stop(&mut self, reason: &str) {
3217 self.stop_requested = true;
3218 self.stop_reason = reason.to_string();
3219 }
3220
3221 pub fn serve(
3222 &mut self,
3223 runtime: &mut Runtime,
3224 run_id: Option<&str>,
3225 max_loops: u64,
3226 max_idle_polls: Option<u64>,
3227 agent: AgentFunc,
3228 ) -> Result<WorkerServiceSummary> {
3229 let mut summary = WorkerServiceSummary {
3230 worker_id: self.worker.worker_id.clone(),
3231 run_id: run_id.map(str::to_string),
3232 stopped_reason: "max_loops".to_string(),
3233 ..WorkerServiceSummary::default()
3234 };
3235 while summary.loops < max_loops {
3236 if self.stop_requested {
3237 summary.stopped_reason = self.stop_reason.clone();
3238 summary.stop_requested = true;
3239 break;
3240 }
3241 summary.loops += 1;
3242 let Some(run_id) = run_id else {
3243 summary.idle_polls += 1;
3244 if max_idle_polls.is_some_and(|limit| summary.idle_polls >= limit) {
3245 summary.stopped_reason = "idle".to_string();
3246 break;
3247 }
3248 continue;
3249 };
3250 let run_summary = self.worker.run_until_idle(runtime, run_id, 1, agent)?;
3251 summary.attempts += run_summary.attempts;
3252 summary.succeeded_attempts += run_summary.succeeded_attempts;
3253 summary.recovered_leases += run_summary.recovered_leases;
3254 summary.final_status = run_summary.final_status.clone();
3255 if summary
3256 .final_status
3257 .as_deref()
3258 .is_some_and(is_terminal_status)
3259 {
3260 summary.stopped_reason = "terminal_status".to_string();
3261 break;
3262 }
3263 if run_summary.attempts == 0 {
3264 summary.idle_polls += 1;
3265 if max_idle_polls.is_some_and(|limit| summary.idle_polls >= limit) {
3266 summary.stopped_reason = "idle".to_string();
3267 break;
3268 }
3269 } else {
3270 summary.idle_polls = 0;
3271 }
3272 }
3273 Ok(summary)
3274 }
3275}
3276
3277fn is_terminal_status(status: &str) -> bool {
3278 matches!(status, "completed" | "failed" | "cancelled")
3279}
3280
3281pub fn export_evidence(store: &MemoryStore, run_id: &str) -> Result<EvidenceBundle> {
3282 let run = store.run(run_id)?;
3283 let steps = store.steps(run_id);
3284 let events = store.events(run_id);
3285 let tool_ledger = store.ledger(run_id);
3286 let approvals = store.approval_requests(run_id);
3287 let artifacts = store.artifacts(run_id);
3288 let media_artifacts = media_artifacts_from(&artifacts);
3289 let stream_checkpoints = stream_checkpoints_from(&artifacts);
3290 let cost_records = store.cost_records(run_id);
3291 let cost_summary = store.cost_summary(run_id);
3292 let summary = evidence_summary(
3293 &steps,
3294 &events,
3295 &tool_ledger,
3296 &approvals,
3297 &artifacts,
3298 &media_artifacts,
3299 &stream_checkpoints,
3300 &cost_records,
3301 &cost_summary,
3302 );
3303 let final_state = store.final_state(run_id)?;
3304 let basis = format!(
3305 "{}:{}:{}:{}:{}",
3306 run_id,
3307 steps.len(),
3308 events.len(),
3309 tool_ledger.len(),
3310 cost_records.len()
3311 );
3312 Ok(EvidenceBundle {
3313 schema_version: "agentledger.evidence.v1".to_string(),
3314 bundle_hash: stable_hash(&basis),
3315 run,
3316 steps,
3317 events,
3318 tool_ledger,
3319 approvals,
3320 artifacts,
3321 media_artifacts,
3322 stream_checkpoints,
3323 cost_records,
3324 cost_summary,
3325 summary,
3326 final_state,
3327 })
3328}
3329
3330fn evidence_summary(
3331 steps: &[Step],
3332 events: &[Event],
3333 ledger: &[ToolLedgerEntry],
3334 approvals: &[ApprovalRequest],
3335 artifacts: &[Artifact],
3336 media_artifacts: &[State],
3337 stream_checkpoints: &[State],
3338 cost_records: &[CostRecord],
3339 cost_summary: &CostSummary,
3340) -> State {
3341 let mut summary = State::new();
3342 summary.insert("event_count".into(), Value::Number(events.len() as f64));
3343 summary.insert("step_count".into(), Value::Number(steps.len() as f64));
3344 summary.insert(
3345 "tool_ledger_count".into(),
3346 Value::Number(ledger.len() as f64),
3347 );
3348 summary.insert(
3349 "approval_count".into(),
3350 Value::Number(approvals.len() as f64),
3351 );
3352 summary.insert(
3353 "artifact_count".into(),
3354 Value::Number(artifacts.len() as f64),
3355 );
3356 summary.insert(
3357 "media_artifact_count".into(),
3358 Value::Number(media_artifacts.len() as f64),
3359 );
3360 summary.insert(
3361 "stream_checkpoint_count".into(),
3362 Value::Number(stream_checkpoints.len() as f64),
3363 );
3364 summary.insert(
3365 "cost_record_count".into(),
3366 Value::Number(cost_records.len() as f64),
3367 );
3368 summary.insert(
3369 "has_failed_steps".into(),
3370 Value::Bool(steps.iter().any(|step| step.status == "failed")),
3371 );
3372 summary.insert(
3373 "has_pending_verification".into(),
3374 Value::Bool(
3375 ledger
3376 .iter()
3377 .any(|row| row.status == "PENDING_VERIFICATION"),
3378 ),
3379 );
3380 summary.insert(
3381 "has_pending_approvals".into(),
3382 Value::Bool(approvals.iter().any(|row| row.status == "PENDING")),
3383 );
3384 let mut cost = State::new();
3385 cost.insert("tool_calls".into(), Value::Number(cost_summary.tool_calls));
3386 cost.insert(
3387 "model_tokens".into(),
3388 Value::Number(cost_summary.model_tokens),
3389 );
3390 cost.insert("total_usd".into(), Value::Number(cost_summary.total_usd));
3391 summary.insert("cost_summary".into(), Value::Object(cost));
3392 summary
3393}
3394
3395#[derive(Clone, Debug)]
3396pub struct ReplaySummary {
3397 pub run_id: String,
3398 pub event_count: usize,
3399 pub tool_call_count: usize,
3400 pub final_state: State,
3401 pub event_hash: String,
3402 pub replay_safe: bool,
3403 pub artifact_count: usize,
3404 pub media_artifact_count: usize,
3405 pub stream_checkpoint_count: usize,
3406}
3407
3408pub fn replay(store: &MemoryStore, run_id: &str) -> Result<ReplaySummary> {
3409 let events = store.events(run_id);
3410 let tool_call_count = events
3411 .iter()
3412 .filter(|event| event.event_type.starts_with("tool_call_"))
3413 .count();
3414 let digest = events
3415 .iter()
3416 .map(|event| format!("{}:{}:{}", event.seq, event.event_type, event.payload_hash))
3417 .collect::<Vec<_>>()
3418 .join("|");
3419 let artifacts = store.artifacts(run_id);
3420 Ok(ReplaySummary {
3421 run_id: run_id.to_string(),
3422 event_count: events.len(),
3423 tool_call_count,
3424 final_state: store.final_state(run_id)?,
3425 event_hash: stable_hash(&digest),
3426 replay_safe: true,
3427 artifact_count: artifacts.len(),
3428 media_artifact_count: media_artifacts_from(&artifacts).len(),
3429 stream_checkpoint_count: stream_checkpoints_from(&artifacts).len(),
3430 })
3431}
3432
3433#[derive(Clone, Debug)]
3434pub struct CostAttributionReport {
3435 pub run_id: String,
3436 pub total: CostSummary,
3437 pub by_agent: HashMap<String, CostSummary>,
3438 pub by_step: HashMap<String, CostSummary>,
3439 pub by_name: HashMap<String, CostSummary>,
3440}
3441
3442pub fn cost_attribution(store: &MemoryStore, run_id: &str) -> CostAttributionReport {
3443 let mut step_roles = HashMap::new();
3444 for event in store.events(run_id) {
3445 if let (Some(step_id), Some(agent_role)) = (event.step_id, event.agent_role) {
3446 step_roles.insert(step_id, agent_role);
3447 }
3448 }
3449 let mut report = CostAttributionReport {
3450 run_id: run_id.to_string(),
3451 total: CostSummary::default(),
3452 by_agent: HashMap::new(),
3453 by_step: HashMap::new(),
3454 by_name: HashMap::new(),
3455 };
3456 for record in store.cost_records(run_id) {
3457 add_cost(&mut report.total, &record);
3458 let agent = step_roles
3459 .get(&record.step_id)
3460 .cloned()
3461 .unwrap_or_else(|| "<unknown>".to_string());
3462 add_cost(report.by_agent.entry(agent).or_default(), &record);
3463 add_cost(
3464 report.by_step.entry(record.step_id.clone()).or_default(),
3465 &record,
3466 );
3467 add_cost(
3468 report.by_name.entry(record.name.clone()).or_default(),
3469 &record,
3470 );
3471 }
3472 report
3473}
3474
3475#[derive(Clone, Debug)]
3476pub struct FailureAttributionReport {
3477 pub run_id: String,
3478 pub run_status: String,
3479 pub failed_steps: Vec<Step>,
3480 pub pending_verification: Vec<ToolLedgerEntry>,
3481 pub pending_approvals: Vec<ApprovalRequest>,
3482 pub failure_events: Vec<Event>,
3483 pub failure_envelopes: Vec<State>,
3484 pub failure_lifecycle: State,
3485 pub failure_causal_graph: State,
3486 pub failure_replay_plan: State,
3487 pub failure_alerts: State,
3488 pub failure_export: State,
3489 pub summary: State,
3490}
3491
3492pub fn failure_attribution(store: &MemoryStore, run_id: &str) -> Result<FailureAttributionReport> {
3493 let run = store.run(run_id)?;
3494 let failed_steps: Vec<Step> = store
3495 .steps(run_id)
3496 .into_iter()
3497 .filter(|step| step.status == "failed")
3498 .collect();
3499 let pending_verification: Vec<ToolLedgerEntry> = store
3500 .ledger(run_id)
3501 .into_iter()
3502 .filter(|entry| entry.status == "PENDING_VERIFICATION")
3503 .collect();
3504 let pending_approvals: Vec<ApprovalRequest> = store
3505 .approval_requests(run_id)
3506 .into_iter()
3507 .filter(|entry| entry.status == "PENDING")
3508 .collect();
3509 let failure_events: Vec<Event> = store
3510 .events(run_id)
3511 .into_iter()
3512 .filter(|event| is_failure_event(&event.event_type))
3513 .collect();
3514 let steps = store.steps(run_id);
3515 let ledger = store.ledger(run_id);
3516 let approvals = store.approval_requests(run_id);
3517 let events = store.events(run_id);
3518 let costs = store.cost_records(run_id);
3519 let failure_envelopes = failure_envelopes(run_id, &run.status, &steps, &ledger, &approvals, &failure_events);
3520 let failure_lifecycle = failure_lifecycle(run_id, &run.status, &failure_envelopes);
3521 let failure_causal_graph = failure_causal_graph(run_id, &run.status, &failure_envelopes, &steps, &ledger, &approvals, &events, &costs);
3522 let failure_replay_plan = failure_replay_plan(run_id, &failure_envelopes, &ledger, &events);
3523 let failure_alerts = failure_alerts(run_id, &failure_envelopes, &failure_replay_plan);
3524 let mut summary = State::new();
3525 summary.insert(
3526 "failed_step_count".to_string(),
3527 Value::Number(failed_steps.len() as f64),
3528 );
3529 summary.insert(
3530 "pending_verification_count".to_string(),
3531 Value::Number(pending_verification.len() as f64),
3532 );
3533 summary.insert(
3534 "pending_approval_count".to_string(),
3535 Value::Number(pending_approvals.len() as f64),
3536 );
3537 summary.insert(
3538 "failure_event_count".to_string(),
3539 Value::Number(failure_events.len() as f64),
3540 );
3541 summary.insert(
3542 "failure_envelope_count".to_string(),
3543 Value::Number(failure_envelopes.len() as f64),
3544 );
3545 summary.insert(
3546 "failure_lifecycle_event_count".to_string(),
3547 Value::Number(state_array_len(&failure_lifecycle, "events") as f64),
3548 );
3549 summary.insert(
3550 "failure_alert_count".to_string(),
3551 failure_alerts
3552 .get("alert_count")
3553 .cloned()
3554 .unwrap_or(Value::Number(0.0)),
3555 );
3556 summary.insert(
3557 "unsafe_replay_side_effect_count".to_string(),
3558 failure_replay_plan
3559 .get("unsafe_side_effect_count")
3560 .cloned()
3561 .unwrap_or(Value::Number(0.0)),
3562 );
3563 summary.insert(
3564 "terminal_failure_count".to_string(),
3565 Value::Number(count_state_field(&failure_envelopes, "status", "terminal") as f64),
3566 );
3567 summary.insert(
3568 "recoverable_failure_count".to_string(),
3569 Value::Number(count_recoverable_failures(&failure_envelopes) as f64),
3570 );
3571 let failure_export = failure_export(
3572 run_id,
3573 &run.status,
3574 &summary,
3575 &failure_envelopes,
3576 &failure_lifecycle,
3577 &failure_causal_graph,
3578 &failure_replay_plan,
3579 &failure_alerts,
3580 );
3581 Ok(FailureAttributionReport {
3582 run_id: run_id.to_string(),
3583 run_status: run.status,
3584 failed_steps,
3585 pending_verification,
3586 pending_approvals,
3587 failure_events,
3588 failure_envelopes,
3589 failure_lifecycle,
3590 failure_causal_graph,
3591 failure_replay_plan,
3592 failure_alerts,
3593 failure_export,
3594 summary,
3595 })
3596}
3597
3598fn failure_envelopes(
3599 run_id: &str,
3600 run_status: &str,
3601 steps: &[Step],
3602 ledger: &[ToolLedgerEntry],
3603 approvals: &[ApprovalRequest],
3604 events: &[Event],
3605) -> Vec<State> {
3606 let mut rows = Vec::new();
3607 for step in steps {
3608 if step.status == "failed" || step.status == "retry_scheduled" || step.status == "waiting_human" {
3609 let (status, severity, recoverability, retryability) = if step.status == "retry_scheduled" {
3610 ("recovery_scheduled", "warn", "auto_retry", "retryable")
3611 } else if step.status == "waiting_human" {
3612 ("waiting_human", "warn", "human_required", "unknown")
3613 } else {
3614 ("terminal", "risk", "terminal", "not_retryable")
3615 };
3616 rows.push(failure_envelope(
3617 run_id,
3618 "step",
3619 &step.step_id,
3620 &failure_category(&format!(
3621 "{} {}",
3622 step.last_error_type.clone().unwrap_or_default(),
3623 step.last_error.clone().unwrap_or_default()
3624 ), "agent"),
3625 status,
3626 severity,
3627 recoverability,
3628 retryability,
3629 "agent",
3630 &first_text(&[
3631 step.last_error.as_deref(),
3632 step.last_error_type.as_deref(),
3633 Some("step failure"),
3634 ]),
3635 state(&[
3636 ("step_id", step.step_id.clone().into()),
3637 ("occurred_at", Value::Number(step.updated_at)),
3638 ]),
3639 vec![ref_state("step", &step.step_id)],
3640 ));
3641 }
3642 }
3643 for entry in ledger {
3644 if entry.status == "PENDING_VERIFICATION" || entry.status == "FAILED" || entry.status == "ERROR" {
3645 let terminal = entry.status == "FAILED" || entry.status == "ERROR";
3646 rows.push(failure_envelope(
3647 run_id,
3648 "tool_ledger",
3649 &first_text(&[Some(&entry.ledger_id), Some(&entry.tool_name), Some(&entry.step_id)]),
3650 "tool",
3651 if terminal { "terminal" } else { "unknown_side_effect" },
3652 if terminal { "risk" } else { "warn" },
3653 if terminal { "terminal" } else { "manual_verification" },
3654 if terminal { "not_retryable" } else { "unknown" },
3655 "tool",
3656 &first_text(&[entry.error_type.as_deref(), Some("tool side effect requires verification")]),
3657 state(&[
3658 ("step_id", entry.step_id.clone().into()),
3659 ("tool_name", entry.tool_name.clone().into()),
3660 ("occurred_at", Value::Number(entry.updated_at)),
3661 ]),
3662 vec![ref_state("step", &entry.step_id), ref_state("tool", &entry.tool_name)],
3663 ));
3664 }
3665 }
3666 for approval in approvals {
3667 if approval.status == "PENDING" || approval.status == "DENIED" {
3668 let denied = approval.status == "DENIED";
3669 rows.push(failure_envelope(
3670 run_id,
3671 "approval",
3672 &first_text(&[Some(&approval.approval_id), Some(&approval.tool_name), Some(&approval.step_id)]),
3673 if denied { "policy" } else { "approval" },
3674 if denied { "blocked" } else { "waiting_human" },
3675 if denied { "risk" } else { "warn" },
3676 if denied { "terminal" } else { "human_required" },
3677 if denied { "not_retryable" } else { "unknown" },
3678 "policy",
3679 &first_text(&[
3680 approval.decision_reason.as_deref(),
3681 Some(&approval.reason),
3682 Some(if denied { "approval denied" } else { "approval pending" }),
3683 ]),
3684 state(&[
3685 ("step_id", approval.step_id.clone().into()),
3686 ("tool_name", approval.tool_name.clone().into()),
3687 ("approval_id", approval.approval_id.clone().into()),
3688 ("occurred_at", Value::Number(approval.updated_at)),
3689 ]),
3690 vec![
3691 ref_state("step", &approval.step_id),
3692 ref_state("tool", &approval.tool_name),
3693 ref_state("approval", &approval.approval_id),
3694 ],
3695 ));
3696 }
3697 }
3698 for event in events {
3699 let category = event_failure_category(event);
3700 let status = event_failure_status(&event.event_type, run_status);
3701 let message = first_text(&[
3702 state_string(&event.payload, "error").as_deref(),
3703 state_string(&event.payload, "reason").as_deref(),
3704 state_string(&event.payload, "error_type").as_deref(),
3705 Some(&event.event_type),
3706 ]);
3707 rows.push(failure_envelope(
3708 run_id,
3709 "event",
3710 &event.seq.to_string(),
3711 &category,
3712 &status,
3713 if status == "terminal" || status == "blocked" || status == "failed" { "risk" } else { "warn" },
3714 &event_recoverability(&event.event_type, run_status),
3715 &event_retryability(&event.event_type),
3716 &owner_for_failure_category(&category),
3717 &message,
3718 state(&[
3719 ("step_id", event.step_id.clone().unwrap_or_default().into()),
3720 ("event_seq", Value::Number(event.seq as f64)),
3721 ("event_type", event.event_type.clone().into()),
3722 ("occurred_at", Value::Number(event.timestamp)),
3723 ]),
3724 vec![
3725 ref_state("event", &event.seq.to_string()),
3726 ref_state("step", event.step_id.as_deref().unwrap_or("")),
3727 ],
3728 ));
3729 }
3730 dedupe_states(rows, "failure_id")
3731}
3732
3733fn failure_envelope(
3734 run_id: &str,
3735 source_kind: &str,
3736 source_id: &str,
3737 category: &str,
3738 status: &str,
3739 severity: &str,
3740 recoverability: &str,
3741 retryability: &str,
3742 owner: &str,
3743 message: &str,
3744 extra: State,
3745 refs: Vec<State>,
3746) -> State {
3747 let mut row = state(&[
3748 ("schema_version", "agentledger.failure.envelope.v1".into()),
3749 ("failure_id", format!("failure-{}", slug(&format!("{run_id}-{source_kind}-{source_id}"))).into()),
3750 ("run_id", run_id.into()),
3751 ("source_kind", source_kind.into()),
3752 ("source_id", source_id.into()),
3753 ("category", category.into()),
3754 ("status", status.into()),
3755 ("severity", severity.into()),
3756 ("recoverability", recoverability.into()),
3757 ("retryability", retryability.into()),
3758 ("owner", owner.into()),
3759 ("message", message.into()),
3760 ("causal_refs", Value::Array(refs.iter().cloned().map(Value::Object).collect())),
3761 ("evidence_refs", Value::Array(refs.into_iter().map(Value::Object).collect())),
3762 ]);
3763 for (key, value) in extra {
3764 if value != Value::String(String::new()) && value != Value::Null {
3765 row.insert(key, value);
3766 }
3767 }
3768 row
3769}
3770
3771fn failure_lifecycle(run_id: &str, run_status: &str, envelopes: &[State]) -> State {
3772 let mut events = Vec::new();
3773 for env in envelopes {
3774 events.push(lifecycle_row(run_id, env, "failure_detected", state_value_string(env, "message"), state_value_string(env, "severity")));
3775 events.push(lifecycle_row(run_id, env, "failure_classified", state_value_string(env, "category"), state_value_string(env, "severity")));
3776 let status = state_value_string(env, "status");
3777 let recoverability = state_value_string(env, "recoverability");
3778 if ["recovery_scheduled", "waiting_human", "unknown_side_effect"].contains(&status.as_str()) || ["auto_retry", "human_required", "manual_verification"].contains(&recoverability.as_str()) {
3779 events.push(lifecycle_row(run_id, env, "failure_recovery_scheduled", "recovery scheduled".to_string(), "warn".to_string()));
3780 }
3781 if ["terminal", "blocked"].contains(&status.as_str()) || recoverability == "terminal" {
3782 events.push(lifecycle_row(run_id, env, "failure_terminal", state_value_string(env, "message"), "risk".to_string()));
3783 }
3784 }
3785 state(&[
3786 ("schema_version", "agentledger.failure.lifecycle.v1".into()),
3787 ("run_id", run_id.into()),
3788 ("run_status", run_status.into()),
3789 ("events", Value::Array(events.iter().cloned().map(Value::Object).collect())),
3790 ("terminal", Value::Bool(events.iter().any(|row| state_value_string(row, "stage") == "failure_terminal"))),
3791 ("recoverable", Value::Bool(events.iter().any(|row| state_value_string(row, "stage") == "failure_recovery_scheduled"))),
3792 ])
3793}
3794
3795fn lifecycle_row(run_id: &str, env: &State, stage: &str, message: String, severity: String) -> State {
3796 state(&[
3797 ("schema_version", "agentledger.failure.lifecycle.v1".into()),
3798 ("stage", stage.into()),
3799 ("run_id", run_id.into()),
3800 ("failure_id", state_value_string(env, "failure_id").into()),
3801 ("category", state_value_string(env, "category").into()),
3802 ("recoverability", state_value_string(env, "recoverability").into()),
3803 ("retryability", state_value_string(env, "retryability").into()),
3804 ("owner", state_value_string(env, "owner").into()),
3805 ("message", message.into()),
3806 ("severity", severity.into()),
3807 ("causal_refs", env.get("causal_refs").cloned().unwrap_or(Value::Array(vec![]))),
3808 ])
3809}
3810
3811fn failure_causal_graph(
3812 run_id: &str,
3813 run_status: &str,
3814 envelopes: &[State],
3815 steps: &[Step],
3816 ledger: &[ToolLedgerEntry],
3817 approvals: &[ApprovalRequest],
3818 events: &[Event],
3819 costs: &[CostRecord],
3820) -> State {
3821 let mut nodes = vec![state(&[("id", format!("run:{}", slug(run_id)).into()), ("kind", "run".into()), ("status", run_status.into())])];
3822 let mut edges = Vec::new();
3823 for step in steps {
3824 nodes.push(state(&[("id", format!("step:{}", slug(&step.step_id)).into()), ("kind", "step".into()), ("status", step.status.clone().into())]));
3825 edges.push(state(&[("source", format!("run:{}", slug(run_id)).into()), ("target", format!("step:{}", slug(&step.step_id)).into()), ("kind", "contains_step".into())]));
3826 }
3827 for event in events {
3828 nodes.push(state(&[("id", format!("event:{}", event.seq).into()), ("kind", "event".into()), ("event_type", event.event_type.clone().into())]));
3829 edges.push(state(&[("source", format!("run:{}", slug(run_id)).into()), ("target", format!("event:{}", event.seq).into()), ("kind", "emitted_event".into())]));
3830 }
3831 for entry in ledger {
3832 nodes.push(state(&[("id", format!("tool:{}", slug(&entry.tool_name)).into()), ("kind", "tool".into()), ("status", entry.status.clone().into())]));
3833 }
3834 for approval in approvals {
3835 nodes.push(state(&[("id", format!("approval:{}", slug(&approval.approval_id)).into()), ("kind", "approval".into()), ("status", approval.status.clone().into())]));
3836 }
3837 for cost in costs {
3838 nodes.push(state(&[("id", format!("cost:{}", slug(&cost.cost_id)).into()), ("kind", "cost".into()), ("category", cost.category.clone().into()), ("amount", Value::Number(cost.amount)), ("unit", cost.unit.clone().into())]));
3839 }
3840 for env in envelopes {
3841 let id = format!("failure:{}", slug(&state_value_string(env, "failure_id")));
3842 nodes.push(state(&[("id", id.clone().into()), ("kind", "failure".into()), ("category", state_value_string(env, "category").into()), ("status", state_value_string(env, "status").into()), ("owner", state_value_string(env, "owner").into())]));
3843 edges.push(state(&[("source", format!("run:{}", slug(run_id)).into()), ("target", id.into()), ("kind", "has_failure".into())]));
3844 }
3845 let nodes = dedupe_states(nodes, "id");
3846 state(&[
3847 ("schema_version", "agentledger.failure.causal_graph.v1".into()),
3848 ("run_id", run_id.into()),
3849 ("nodes", Value::Array(nodes.iter().cloned().map(Value::Object).collect())),
3850 ("edges", Value::Array(edges.iter().cloned().map(Value::Object).collect())),
3851 ("summary", Value::Object(state(&[
3852 ("node_count", Value::Number(nodes.len() as f64)),
3853 ("edge_count", Value::Number(edges.len() as f64)),
3854 ("failure_node_count", Value::Number(count_state_field(&nodes, "kind", "failure") as f64)),
3855 ]))),
3856 ])
3857}
3858
3859fn failure_replay_plan(run_id: &str, envelopes: &[State], ledger: &[ToolLedgerEntry], events: &[Event]) -> State {
3860 let mut actions = Vec::new();
3861 let mut unsafe_count = 0;
3862 let mut manual_count = 0;
3863 for env in envelopes {
3864 let status = state_value_string(env, "status");
3865 let recoverability = state_value_string(env, "recoverability");
3866 let mut action = state(&[
3867 ("failure_id", state_value_string(env, "failure_id").into()),
3868 ("category", state_value_string(env, "category").into()),
3869 ("status", status.clone().into()),
3870 ("replay_action", "reuse_recorded_evidence".into()),
3871 ("replay_safe", Value::Bool(true)),
3872 ("requires_manual_verification", Value::Bool(false)),
3873 ("reason", "recorded runtime evidence can be inspected without calling external systems".into()),
3874 ]);
3875 if status == "unknown_side_effect" || recoverability == "manual_verification" {
3876 action.insert("replay_action".into(), "manual_verify_side_effect".into());
3877 action.insert("replay_safe".into(), Value::Bool(false));
3878 action.insert("requires_manual_verification".into(), Value::Bool(true));
3879 action.insert("reason".into(), "Tool Ledger recorded an unknown side-effect state".into());
3880 unsafe_count += 1;
3881 manual_count += 1;
3882 } else if status == "waiting_human" {
3883 action.insert("replay_action".into(), "resume_after_approval".into());
3884 } else if status == "recovery_scheduled" {
3885 action.insert("replay_action".into(), "retry_from_checkpoint".into());
3886 } else if status == "terminal" || status == "blocked" {
3887 action.insert("replay_action".into(), "terminal_stop".into());
3888 }
3889 actions.push(action);
3890 }
3891 state(&[
3892 ("schema_version", "agentledger.failure.replay_plan.v1".into()),
3893 ("run_id", run_id.into()),
3894 ("mode", "evidence_only".into()),
3895 ("safe_to_replay", Value::Bool(unsafe_count == 0)),
3896 ("unsafe_side_effect_count", Value::Number(unsafe_count as f64)),
3897 ("manual_verification_count", Value::Number(manual_count as f64)),
3898 ("recorded_tool_call_count", Value::Number(ledger.len() as f64)),
3899 ("recorded_event_count", Value::Number(events.len() as f64)),
3900 ("actions", Value::Array(actions.into_iter().map(Value::Object).collect())),
3901 ])
3902}
3903
3904fn failure_alerts(run_id: &str, envelopes: &[State], replay_plan: &State) -> State {
3905 let mut alerts = Vec::new();
3906 if count_state_field(envelopes, "status", "terminal") > 0 {
3907 alerts.push(alert_state(run_id, "terminal_failure", "risk", "terminal failure recorded"));
3908 }
3909 if count_state_field(envelopes, "status", "unknown_side_effect") > 0 {
3910 alerts.push(alert_state(run_id, "unknown_side_effect", "risk", "tool side-effect state requires manual verification"));
3911 }
3912 if state_number(replay_plan, "unsafe_side_effect_count") > 0.0 {
3913 alerts.push(alert_state(run_id, "unsafe_replay_blocked", "risk", "failure replay plan blocks unsafe automatic replay"));
3914 }
3915 state(&[
3916 ("schema_version", "agentledger.failure.alerts.v1".into()),
3917 ("run_id", run_id.into()),
3918 ("alerts", Value::Array(alerts.iter().cloned().map(Value::Object).collect())),
3919 ("alert_count", Value::Number(alerts.len() as f64)),
3920 ])
3921}
3922
3923fn alert_state(run_id: &str, kind: &str, severity: &str, message: &str) -> State {
3924 state(&[
3925 ("schema_version", "agentledger.failure.alerts.v1".into()),
3926 ("run_id", run_id.into()),
3927 ("kind", kind.into()),
3928 ("severity", severity.into()),
3929 ("message", message.into()),
3930 ])
3931}
3932
3933fn failure_export(
3934 run_id: &str,
3935 run_status: &str,
3936 summary: &State,
3937 envelopes: &[State],
3938 lifecycle: &State,
3939 graph: &State,
3940 replay_plan: &State,
3941 alerts: &State,
3942) -> State {
3943 state(&[
3944 ("schema_version", "agentledger.failure.export.v1".into()),
3945 ("run_id", run_id.into()),
3946 ("run_status", run_status.into()),
3947 ("summary", Value::Object(summary.clone())),
3948 ("failure_envelopes", Value::Array(envelopes.iter().cloned().map(Value::Object).collect())),
3949 ("failure_lifecycle", Value::Object(lifecycle.clone())),
3950 ("failure_causal_graph", Value::Object(graph.clone())),
3951 ("failure_replay_plan", Value::Object(replay_plan.clone())),
3952 ("failure_alerts", Value::Object(alerts.clone())),
3953 ("external_mappings", Value::Object(state(&[
3954 ("opentelemetry", Value::Object(state(&[("span_event_count", Value::Number(state_array_len(lifecycle, "events") as f64))]))),
3955 ("langfuse", Value::Object(state(&[("trace_id", run_id.into()), ("observation_count", Value::Number(envelopes.len() as f64))]))),
3956 ("langsmith", Value::Object(state(&[("run_id", run_id.into()), ("feedback_count", Value::Number(envelopes.len() as f64))]))),
3957 ("temporal", Value::Object(state(&[("workflow_id", run_id.into()), ("failure_count", Value::Number(envelopes.len() as f64)), ("safe_to_replay", replay_plan.get("safe_to_replay").cloned().unwrap_or(Value::Bool(false)))]))),
3958 ]))),
3959 ])
3960}
3961
3962fn encode_store(store: &MemoryStore) -> String {
3963 let mut lines = vec!["AGENTLEDGER_RUST_STORE_V1".to_string()];
3964 let mut runs: Vec<_> = store.runs.values().collect();
3965 runs.sort_by(|a, b| a.run_id.cmp(&b.run_id));
3966 for run in runs {
3967 lines.push(join_fields(&[
3968 "R".to_string(),
3969 hex_encode(&run.run_id),
3970 hex_encode(&run.session_id),
3971 hex_encode(&run.status),
3972 run.state_version.to_string(),
3973 run.created_at.to_string(),
3974 run.updated_at.to_string(),
3975 encode_state(&run.state),
3976 ]));
3977 }
3978
3979 let mut steps: Vec<_> = store.steps.values().collect();
3980 steps.sort_by(|a, b| a.step_id.cmp(&b.step_id));
3981 for step in steps {
3982 lines.push(join_fields(&[
3983 "S".to_string(),
3984 hex_encode(&step.step_id),
3985 hex_encode(&step.run_id),
3986 hex_encode(&step.session_id),
3987 hex_encode(&step.status),
3988 encode_option_string(&step.owner),
3989 encode_option_string(&step.lease_token),
3990 encode_option_f64(step.lease_until),
3991 step.attempt.to_string(),
3992 step.state_version.to_string(),
3993 encode_option_string(&step.checkpoint_id),
3994 encode_option_string(&step.last_error_type),
3995 encode_option_string(&step.last_error),
3996 encode_option_f64(step.cancelled_at),
3997 step.created_at.to_string(),
3998 step.updated_at.to_string(),
3999 ]));
4000 }
4001
4002 let mut run_ids: Vec<_> = store.events.keys().collect();
4003 run_ids.sort();
4004 for run_id in run_ids {
4005 if let Some(events) = store.events.get(run_id) {
4006 for event in events {
4007 lines.push(join_fields(&[
4008 "E".to_string(),
4009 hex_encode(&event.event_id),
4010 hex_encode(&event.run_id),
4011 encode_option_string(&event.session_id),
4012 encode_option_string(&event.step_id),
4013 event.seq.to_string(),
4014 hex_encode(&event.event_type),
4015 event.timestamp.to_string(),
4016 encode_option_string(&event.agent_role),
4017 encode_option_u64(event.state_version),
4018 encode_option_string(&event.causal_token),
4019 hex_encode(&event.payload_hash),
4020 hex_encode(&event.payload_ref),
4021 encode_state(&event.payload),
4022 ]));
4023 }
4024 }
4025 }
4026
4027 let mut ledgers: Vec<_> = store.tool_ledger.values().collect();
4028 ledgers.sort_by(|a, b| a.idempotency_key.cmp(&b.idempotency_key));
4029 for entry in ledgers {
4030 lines.push(join_fields(&[
4031 "L".to_string(),
4032 hex_encode(&entry.ledger_id),
4033 hex_encode(&entry.run_id),
4034 hex_encode(&entry.session_id),
4035 hex_encode(&entry.step_id),
4036 hex_encode(&entry.tool_name),
4037 hex_encode(&entry.tool_version),
4038 hex_encode(&entry.tool_call_id),
4039 hex_encode(&entry.idempotency_key),
4040 hex_encode(&entry.causal_token),
4041 hex_encode(&entry.request_hash),
4042 hex_encode(&entry.request_ref),
4043 hex_encode(&entry.status),
4044 encode_option_string(&entry.external_id),
4045 encode_option_string(&entry.response_hash),
4046 encode_option_string(&entry.response_ref),
4047 encode_option_string(&entry.error_type),
4048 encode_option_value(&entry.response),
4049 entry.created_at.to_string(),
4050 entry.updated_at.to_string(),
4051 ]));
4052 }
4053
4054 let mut approvals: Vec<_> = store.approval_requests.values().collect();
4055 approvals.sort_by(|a, b| a.approval_key.cmp(&b.approval_key));
4056 for approval in approvals {
4057 lines.push(join_fields(&[
4058 "A".to_string(),
4059 hex_encode(&approval.approval_id),
4060 hex_encode(&approval.approval_key),
4061 hex_encode(&approval.run_id),
4062 hex_encode(&approval.session_id),
4063 hex_encode(&approval.step_id),
4064 hex_encode(&approval.tool_name),
4065 hex_encode(&approval.risk_level),
4066 hex_encode(&approval.status),
4067 hex_encode(&approval.reason),
4068 hex_encode(&approval.request_hash),
4069 hex_encode(&approval.request_ref),
4070 hex_encode(&approval.requested_by),
4071 encode_option_string(&approval.approved_by),
4072 encode_option_string(&approval.decision_reason),
4073 approval.created_at.to_string(),
4074 approval.updated_at.to_string(),
4075 ]));
4076 }
4077
4078 let mut cost_run_ids: Vec<_> = store.cost_records.keys().collect();
4079 cost_run_ids.sort();
4080 for run_id in cost_run_ids {
4081 if let Some(records) = store.cost_records.get(run_id) {
4082 for record in records {
4083 lines.push(join_fields(&[
4084 "C".to_string(),
4085 hex_encode(&record.cost_id),
4086 hex_encode(&record.run_id),
4087 hex_encode(&record.session_id),
4088 hex_encode(&record.step_id),
4089 hex_encode(&record.category),
4090 hex_encode(&record.name),
4091 record.amount.to_string(),
4092 hex_encode(&record.unit),
4093 encode_state(&record.metadata),
4094 record.created_at.to_string(),
4095 ]));
4096 }
4097 }
4098 }
4099
4100 let mut artifact_run_ids: Vec<_> = store.artifacts.keys().collect();
4101 artifact_run_ids.sort();
4102 for run_id in artifact_run_ids {
4103 if let Some(artifacts) = store.artifacts.get(run_id) {
4104 for artifact in artifacts {
4105 lines.push(join_fields(&[
4106 "F".to_string(),
4107 hex_encode(&artifact.artifact_id),
4108 hex_encode(&artifact.run_id),
4109 encode_option_string(&artifact.step_id),
4110 hex_encode(&artifact.name),
4111 hex_encode(&artifact.blob_hash),
4112 hex_encode(&artifact.blob_ref),
4113 encode_state(&artifact.metadata),
4114 artifact.created_at.to_string(),
4115 ]));
4116 }
4117 }
4118 }
4119
4120 lines.push(String::new());
4121 lines.join("\n")
4122}
4123
4124fn decode_store(body: &str) -> Result<MemoryStore> {
4125 let mut lines = body.lines();
4126 match lines.next() {
4127 Some("AGENTLEDGER_RUST_STORE_V1") => {}
4128 _ => {
4129 return Err(RuntimeError(
4130 "invalid Rust store snapshot header".to_string(),
4131 ))
4132 }
4133 }
4134 let mut store = MemoryStore::new();
4135 for line in lines.filter(|line| !line.trim().is_empty()) {
4136 let fields: Vec<&str> = line.split('\t').collect();
4137 let tag = fields.first().copied().unwrap_or_default();
4138 match tag {
4139 "R" => {
4140 require_len(tag, &fields, 8)?;
4141 let run = Run {
4142 run_id: hex_decode(fields[1])?,
4143 session_id: hex_decode(fields[2])?,
4144 status: hex_decode(fields[3])?,
4145 state_version: parse_u64(fields[4])?,
4146 created_at: parse_f64(fields[5])?,
4147 updated_at: parse_f64(fields[6])?,
4148 state: decode_state(fields[7])?,
4149 };
4150 store.runs.insert(run.run_id.clone(), run);
4151 }
4152 "S" => {
4153 require_len(tag, &fields, 16)?;
4154 let step = Step {
4155 step_id: hex_decode(fields[1])?,
4156 run_id: hex_decode(fields[2])?,
4157 session_id: hex_decode(fields[3])?,
4158 status: hex_decode(fields[4])?,
4159 owner: decode_option_string(fields[5])?,
4160 lease_token: decode_option_string(fields[6])?,
4161 lease_until: decode_option_f64(fields[7])?,
4162 attempt: parse_u64(fields[8])?,
4163 state_version: parse_u64(fields[9])?,
4164 checkpoint_id: decode_option_string(fields[10])?,
4165 last_error_type: decode_option_string(fields[11])?,
4166 last_error: decode_option_string(fields[12])?,
4167 cancelled_at: decode_option_f64(fields[13])?,
4168 created_at: parse_f64(fields[14])?,
4169 updated_at: parse_f64(fields[15])?,
4170 };
4171 store.steps.insert(step.step_id.clone(), step);
4172 }
4173 "E" => {
4174 require_len(tag, &fields, 14)?;
4175 let event = Event {
4176 event_id: hex_decode(fields[1])?,
4177 run_id: hex_decode(fields[2])?,
4178 session_id: decode_option_string(fields[3])?,
4179 step_id: decode_option_string(fields[4])?,
4180 seq: parse_u64(fields[5])?,
4181 event_type: hex_decode(fields[6])?,
4182 timestamp: parse_f64(fields[7])?,
4183 agent_role: decode_option_string(fields[8])?,
4184 state_version: decode_option_u64(fields[9])?,
4185 causal_token: decode_option_string(fields[10])?,
4186 payload_hash: hex_decode(fields[11])?,
4187 payload_ref: hex_decode(fields[12])?,
4188 payload: decode_state(fields[13])?,
4189 };
4190 store
4191 .events
4192 .entry(event.run_id.clone())
4193 .or_default()
4194 .push(event);
4195 }
4196 "L" => {
4197 require_len(tag, &fields, 20)?;
4198 let entry = ToolLedgerEntry {
4199 ledger_id: hex_decode(fields[1])?,
4200 run_id: hex_decode(fields[2])?,
4201 session_id: hex_decode(fields[3])?,
4202 step_id: hex_decode(fields[4])?,
4203 tool_name: hex_decode(fields[5])?,
4204 tool_version: hex_decode(fields[6])?,
4205 tool_call_id: hex_decode(fields[7])?,
4206 idempotency_key: hex_decode(fields[8])?,
4207 causal_token: hex_decode(fields[9])?,
4208 request_hash: hex_decode(fields[10])?,
4209 request_ref: hex_decode(fields[11])?,
4210 status: hex_decode(fields[12])?,
4211 external_id: decode_option_string(fields[13])?,
4212 response_hash: decode_option_string(fields[14])?,
4213 response_ref: decode_option_string(fields[15])?,
4214 error_type: decode_option_string(fields[16])?,
4215 response: decode_option_value(fields[17])?,
4216 created_at: parse_f64(fields[18])?,
4217 updated_at: parse_f64(fields[19])?,
4218 };
4219 store
4220 .tool_ledger
4221 .insert(entry.idempotency_key.clone(), entry);
4222 }
4223 "A" => {
4224 require_len(tag, &fields, 17)?;
4225 let approval = ApprovalRequest {
4226 approval_id: hex_decode(fields[1])?,
4227 approval_key: hex_decode(fields[2])?,
4228 run_id: hex_decode(fields[3])?,
4229 session_id: hex_decode(fields[4])?,
4230 step_id: hex_decode(fields[5])?,
4231 tool_name: hex_decode(fields[6])?,
4232 risk_level: hex_decode(fields[7])?,
4233 status: hex_decode(fields[8])?,
4234 reason: hex_decode(fields[9])?,
4235 request_hash: hex_decode(fields[10])?,
4236 request_ref: hex_decode(fields[11])?,
4237 requested_by: hex_decode(fields[12])?,
4238 approved_by: decode_option_string(fields[13])?,
4239 decision_reason: decode_option_string(fields[14])?,
4240 created_at: parse_f64(fields[15])?,
4241 updated_at: parse_f64(fields[16])?,
4242 };
4243 store
4244 .approval_requests
4245 .insert(approval.approval_key.clone(), approval);
4246 }
4247 "C" => {
4248 require_len(tag, &fields, 11)?;
4249 let record = CostRecord {
4250 cost_id: hex_decode(fields[1])?,
4251 run_id: hex_decode(fields[2])?,
4252 session_id: hex_decode(fields[3])?,
4253 step_id: hex_decode(fields[4])?,
4254 category: hex_decode(fields[5])?,
4255 name: hex_decode(fields[6])?,
4256 amount: parse_f64(fields[7])?,
4257 unit: hex_decode(fields[8])?,
4258 metadata: decode_state(fields[9])?,
4259 created_at: parse_f64(fields[10])?,
4260 };
4261 store
4262 .cost_records
4263 .entry(record.run_id.clone())
4264 .or_default()
4265 .push(record);
4266 }
4267 "F" => {
4268 require_len(tag, &fields, 9)?;
4269 let artifact = Artifact {
4270 artifact_id: hex_decode(fields[1])?,
4271 run_id: hex_decode(fields[2])?,
4272 step_id: decode_option_string(fields[3])?,
4273 name: hex_decode(fields[4])?,
4274 blob_hash: hex_decode(fields[5])?,
4275 blob_ref: hex_decode(fields[6])?,
4276 metadata: decode_state(fields[7])?,
4277 created_at: parse_f64(fields[8])?,
4278 };
4279 store
4280 .artifacts
4281 .entry(artifact.run_id.clone())
4282 .or_default()
4283 .push(artifact);
4284 }
4285 _ => {
4286 return Err(RuntimeError(format!(
4287 "unknown Rust store snapshot row: {tag}"
4288 )))
4289 }
4290 }
4291 }
4292 Ok(store)
4293}
4294
4295fn join_fields(fields: &[String]) -> String {
4296 fields.join("\t")
4297}
4298
4299fn require_len(tag: &str, fields: &[&str], expected: usize) -> Result<()> {
4300 if fields.len() != expected {
4301 return Err(RuntimeError(format!(
4302 "invalid {tag} row: expected {expected} fields, got {}",
4303 fields.len()
4304 )));
4305 }
4306 Ok(())
4307}
4308
4309fn encode_option_string(value: &Option<String>) -> String {
4310 value
4311 .as_ref()
4312 .map(|item| hex_encode(item))
4313 .unwrap_or_else(|| "-".to_string())
4314}
4315
4316fn decode_option_string(value: &str) -> Result<Option<String>> {
4317 if value == "-" {
4318 Ok(None)
4319 } else {
4320 Ok(Some(hex_decode(value)?))
4321 }
4322}
4323
4324fn encode_option_f64(value: Option<f64>) -> String {
4325 value
4326 .map(|item| item.to_string())
4327 .unwrap_or_else(|| "-".to_string())
4328}
4329
4330fn decode_option_f64(value: &str) -> Result<Option<f64>> {
4331 if value == "-" {
4332 Ok(None)
4333 } else {
4334 Ok(Some(parse_f64(value)?))
4335 }
4336}
4337
4338fn encode_option_u64(value: Option<u64>) -> String {
4339 value
4340 .map(|item| item.to_string())
4341 .unwrap_or_else(|| "-".to_string())
4342}
4343
4344fn decode_option_u64(value: &str) -> Result<Option<u64>> {
4345 if value == "-" {
4346 Ok(None)
4347 } else {
4348 Ok(Some(parse_u64(value)?))
4349 }
4350}
4351
4352fn encode_option_value(value: &Option<Value>) -> String {
4353 value
4354 .as_ref()
4355 .map(encode_value)
4356 .unwrap_or_else(|| "-".to_string())
4357}
4358
4359fn decode_option_value(value: &str) -> Result<Option<Value>> {
4360 if value == "-" {
4361 Ok(None)
4362 } else {
4363 Ok(Some(decode_value(value)?))
4364 }
4365}
4366
4367fn encode_state(state: &State) -> String {
4368 encode_value(&Value::Object(state.clone()))
4369}
4370
4371fn decode_state(encoded: &str) -> Result<State> {
4372 match decode_value(encoded)? {
4373 Value::Object(state) => Ok(state),
4374 _ => Err(RuntimeError("encoded state was not an object".to_string())),
4375 }
4376}
4377
4378fn encode_value(value: &Value) -> String {
4379 match value {
4380 Value::Null => "Z".to_string(),
4381 Value::Bool(true) => "T".to_string(),
4382 Value::Bool(false) => "F".to_string(),
4383 Value::Number(item) => format!("N{}:", hex_encode(&item.to_string())),
4384 Value::String(item) => format!("S{}:", hex_encode(item)),
4385 Value::Object(state) => {
4386 let mut keys: Vec<_> = state.keys().collect();
4387 keys.sort();
4388 let mut out = format!("O{}:", keys.len());
4389 for key in keys {
4390 out.push_str(&hex_encode(key));
4391 out.push(':');
4392 out.push_str(&encode_value(&state[key]));
4393 }
4394 out
4395 }
4396 Value::Array(values) => {
4397 let mut out = format!("A{}:", values.len());
4398 for value in values {
4399 out.push_str(&encode_value(value));
4400 }
4401 out
4402 }
4403 }
4404}
4405
4406fn decode_value(encoded: &str) -> Result<Value> {
4407 let (value, index) = parse_value(encoded, 0)?;
4408 if index != encoded.len() {
4409 return Err(RuntimeError("trailing bytes in encoded value".to_string()));
4410 }
4411 Ok(value)
4412}
4413
4414fn parse_value(input: &str, index: usize) -> Result<(Value, usize)> {
4415 let bytes = input.as_bytes();
4416 let tag = *bytes
4417 .get(index)
4418 .ok_or_else(|| RuntimeError("unexpected end of encoded value".to_string()))?
4419 as char;
4420 match tag {
4421 'Z' => Ok((Value::Null, index + 1)),
4422 'T' => Ok((Value::Bool(true), index + 1)),
4423 'F' => Ok((Value::Bool(false), index + 1)),
4424 'N' => {
4425 let (hex, next) = read_until_colon(input, index + 1)?;
4426 Ok((Value::Number(parse_f64(&hex_decode(hex)?)?), next))
4427 }
4428 'S' => {
4429 let (hex, next) = read_until_colon(input, index + 1)?;
4430 Ok((Value::String(hex_decode(hex)?), next))
4431 }
4432 'O' => {
4433 let (count_text, mut next) = read_until_colon(input, index + 1)?;
4434 let count = count_text
4435 .parse::<usize>()
4436 .map_err(|err| RuntimeError(err.to_string()))?;
4437 let mut state = State::new();
4438 for _ in 0..count {
4439 let (key_hex, after_key) = read_until_colon(input, next)?;
4440 let key = hex_decode(key_hex)?;
4441 let (value, after_value) = parse_value(input, after_key)?;
4442 state.insert(key, value);
4443 next = after_value;
4444 }
4445 Ok((Value::Object(state), next))
4446 }
4447 'A' => {
4448 let (count_text, mut next) = read_until_colon(input, index + 1)?;
4449 let count = count_text
4450 .parse::<usize>()
4451 .map_err(|err| RuntimeError(err.to_string()))?;
4452 let mut values = Vec::with_capacity(count);
4453 for _ in 0..count {
4454 let (value, after_value) = parse_value(input, next)?;
4455 values.push(value);
4456 next = after_value;
4457 }
4458 Ok((Value::Array(values), next))
4459 }
4460 _ => Err(RuntimeError(format!("unknown encoded value tag: {tag}"))),
4461 }
4462}
4463
4464fn read_until_colon(input: &str, index: usize) -> Result<(&str, usize)> {
4465 let rest = input
4466 .get(index..)
4467 .ok_or_else(|| RuntimeError("invalid encoded value index".to_string()))?;
4468 let offset = rest
4469 .find(':')
4470 .ok_or_else(|| RuntimeError("missing encoded value delimiter".to_string()))?;
4471 Ok((&rest[..offset], index + offset + 1))
4472}
4473
4474fn hex_encode(value: &str) -> String {
4475 value
4476 .as_bytes()
4477 .iter()
4478 .map(|byte| format!("{byte:02x}"))
4479 .collect()
4480}
4481
4482fn hex_decode(value: &str) -> Result<String> {
4483 if value.len() % 2 != 0 {
4484 return Err(RuntimeError("invalid hex string length".to_string()));
4485 }
4486 let mut bytes = Vec::with_capacity(value.len() / 2);
4487 for index in (0..value.len()).step_by(2) {
4488 let byte = u8::from_str_radix(&value[index..index + 2], 16)
4489 .map_err(|err| RuntimeError(err.to_string()))?;
4490 bytes.push(byte);
4491 }
4492 String::from_utf8(bytes).map_err(|err| RuntimeError(err.to_string()))
4493}
4494
4495fn parse_f64(value: &str) -> Result<f64> {
4496 value
4497 .parse::<f64>()
4498 .map_err(|err| RuntimeError(err.to_string()))
4499}
4500
4501fn parse_u64(value: &str) -> Result<u64> {
4502 value
4503 .parse::<u64>()
4504 .map_err(|err| RuntimeError(err.to_string()))
4505}
4506
4507fn merge_patch(base: &State, patch: &State) -> State {
4508 let mut out = base.clone();
4509 for (key, value) in patch {
4510 match value {
4511 Value::Null => {
4512 out.remove(key);
4513 }
4514 Value::Object(patch_map) => {
4515 if let Some(Value::Object(base_map)) = out.get(key) {
4516 out.insert(key.clone(), Value::Object(merge_patch(base_map, patch_map)));
4517 } else {
4518 out.insert(key.clone(), value.clone());
4519 }
4520 }
4521 _ => {
4522 out.insert(key.clone(), value.clone());
4523 }
4524 }
4525 }
4526 out
4527}
4528
4529fn add_cost(summary: &mut CostSummary, record: &CostRecord) {
4530 if (record.category == "tool" || record.category == "tool_shadow") && record.unit == "call" {
4531 summary.tool_calls += record.amount;
4532 }
4533 if record.category == "model" && record.unit == "token" {
4534 summary.model_tokens += record.amount;
4535 }
4536 if record.unit == "usd" {
4537 summary.total_usd += record.amount;
4538 }
4539 let key = format!("{}:{}", record.category, record.unit);
4540 *summary.by_category.entry(key).or_insert(0.0) += record.amount;
4541}
4542
4543pub fn validate_tool_schema(schema: &Value, value: &Value, path: &str) -> Result<()> {
4544 let schema = match schema {
4545 Value::Object(schema) => schema,
4546 _ => return Err(RuntimeError(format!("{path} schema must be object"))),
4547 };
4548 if let Some(expected) = schema.get("const") {
4549 if expected != value {
4550 return Err(RuntimeError(format!("{path} expected const")));
4551 }
4552 }
4553 if let Some(Value::Array(items)) = schema.get("enum") {
4554 if !items.iter().any(|item| item == value) {
4555 return Err(RuntimeError(format!("{path} value not in enum")));
4556 }
4557 }
4558 let Some(Value::String(kind)) = schema.get("type") else {
4559 return Ok(());
4560 };
4561 match kind.as_str() {
4562 "object" => {
4563 let Value::Object(object) = value else {
4564 return Err(RuntimeError(format!("{path} expected object")));
4565 };
4566 if let Some(Value::Array(required)) = schema.get("required") {
4567 for item in required {
4568 if let Value::String(key) = item {
4569 if !object.contains_key(key) {
4570 return Err(RuntimeError(format!("{path}.{key} is required")));
4571 }
4572 }
4573 }
4574 }
4575 let properties = match schema.get("properties") {
4576 Some(Value::Object(properties)) => properties,
4577 _ => return Ok(()),
4578 };
4579 for (key, child_schema) in properties {
4580 if let Some(child) = object.get(key) {
4581 validate_tool_schema(child_schema, child, &format!("{path}.{key}"))?;
4582 }
4583 }
4584 if schema.get("additionalProperties") == Some(&Value::Bool(false)) {
4585 for key in object.keys() {
4586 if !properties.contains_key(key) {
4587 return Err(RuntimeError(format!("{path}.{key} is not allowed")));
4588 }
4589 }
4590 }
4591 }
4592 "string" => {
4593 let Value::String(text) = value else {
4594 return Err(RuntimeError(format!("{path} expected string")));
4595 };
4596 if let Some(Value::Number(min)) = schema.get("minLength") {
4597 if (text.len() as f64) < *min {
4598 return Err(RuntimeError(format!("{path} shorter than minLength")));
4599 }
4600 }
4601 if let Some(Value::Number(max)) = schema.get("maxLength") {
4602 if (text.len() as f64) > *max {
4603 return Err(RuntimeError(format!("{path} longer than maxLength")));
4604 }
4605 }
4606 }
4607 "number" | "integer" => {
4608 let Value::Number(number) = value else {
4609 return Err(RuntimeError(format!("{path} expected number")));
4610 };
4611 if kind == "integer" && number.fract() != 0.0 {
4612 return Err(RuntimeError(format!("{path} expected integer")));
4613 }
4614 if let Some(Value::Number(min)) = schema.get("minimum") {
4615 if number < min {
4616 return Err(RuntimeError(format!("{path} below minimum")));
4617 }
4618 }
4619 if let Some(Value::Number(max)) = schema.get("maximum") {
4620 if number > max {
4621 return Err(RuntimeError(format!("{path} above maximum")));
4622 }
4623 }
4624 }
4625 "boolean" => {
4626 if !matches!(value, Value::Bool(_)) {
4627 return Err(RuntimeError(format!("{path} expected boolean")));
4628 }
4629 }
4630 _ => {}
4631 }
4632 Ok(())
4633}
4634
4635fn is_high_risk(risk: &str) -> bool {
4636 matches!(
4637 risk,
4638 "high" | "destructive" | "sensitive" | "financial_or_legal"
4639 )
4640}
4641
4642fn is_media_kind(kind: &str) -> bool {
4643 matches!(
4644 kind,
4645 "image"
4646 | "audio"
4647 | "video"
4648 | "frame"
4649 | "audio_segment"
4650 | "video_segment"
4651 | "transcript"
4652 | "embedding"
4653 | "derived"
4654 )
4655}
4656
4657fn stream_chunk_to_state(chunk: StreamChunkRef) -> State {
4658 let mut state = State::new();
4659 state.insert(
4660 "schema_version".to_string(),
4661 Value::String(STREAM_SCHEMA_VERSION.to_string()),
4662 );
4663 state.insert("stream_id".to_string(), Value::String(chunk.stream_id));
4664 state.insert("chunk_id".to_string(), Value::String(chunk.chunk_id));
4665 state.insert("offset".to_string(), chunk.offset);
4666 if let Some(content_ref) = chunk.content_ref {
4667 state.insert("content_ref".to_string(), Value::String(content_ref));
4668 }
4669 if let Some(content_hash) = chunk.content_hash {
4670 state.insert("content_hash".to_string(), Value::String(content_hash));
4671 }
4672 if let Some(sequence) = chunk.sequence {
4673 state.insert("sequence".to_string(), Value::Number(sequence));
4674 }
4675 if let Some(event_time) = chunk.event_time {
4676 state.insert("event_time".to_string(), Value::Number(event_time));
4677 }
4678 if !chunk.metadata.is_empty() {
4679 state.insert("metadata".to_string(), Value::Object(chunk.metadata));
4680 }
4681 state
4682}
4683
4684fn media_artifacts_from(artifacts: &[Artifact]) -> Vec<State> {
4685 artifacts
4686 .iter()
4687 .filter_map(
4688 |artifact| match artifact.metadata.get("agentledger_media") {
4689 Some(Value::Object(metadata)) => {
4690 let mut row = State::new();
4691 row.insert(
4692 "artifact_id".to_string(),
4693 Value::String(artifact.artifact_id.clone()),
4694 );
4695 row.insert("name".to_string(), Value::String(artifact.name.clone()));
4696 row.insert(
4697 "blob_hash".to_string(),
4698 Value::String(artifact.blob_hash.clone()),
4699 );
4700 row.insert(
4701 "blob_ref".to_string(),
4702 Value::String(artifact.blob_ref.clone()),
4703 );
4704 for key in ["kind", "uri", "content_ref", "metadata", "lineage"] {
4705 if let Some(value) = metadata.get(key) {
4706 row.insert(key.to_string(), value.clone());
4707 }
4708 }
4709 Some(row)
4710 }
4711 _ => None,
4712 },
4713 )
4714 .collect()
4715}
4716
4717fn stream_checkpoints_from(artifacts: &[Artifact]) -> Vec<State> {
4718 artifacts
4719 .iter()
4720 .filter_map(
4721 |artifact| match artifact.metadata.get("agentledger_stream") {
4722 Some(Value::Object(metadata)) => {
4723 let mut row = State::new();
4724 row.insert(
4725 "artifact_id".to_string(),
4726 Value::String(artifact.artifact_id.clone()),
4727 );
4728 row.insert("name".to_string(), Value::String(artifact.name.clone()));
4729 row.insert(
4730 "blob_hash".to_string(),
4731 Value::String(artifact.blob_hash.clone()),
4732 );
4733 row.insert(
4734 "blob_ref".to_string(),
4735 Value::String(artifact.blob_ref.clone()),
4736 );
4737 for key in [
4738 "stream_id",
4739 "consumer_id",
4740 "offset",
4741 "watermark",
4742 "chunk",
4743 "partial_result_ref",
4744 "backpressure",
4745 ] {
4746 if let Some(value) = metadata.get(key) {
4747 row.insert(key.to_string(), value.clone());
4748 }
4749 }
4750 Some(row)
4751 }
4752 _ => None,
4753 },
4754 )
4755 .collect()
4756}
4757
4758fn classify_runtime_error(message: &str) -> &'static str {
4759 if message.contains("budget exceeded") || message.contains("budget") {
4760 "BudgetExceededError"
4761 } else if message.contains("sandbox executor") {
4762 "SandboxUnavailableError"
4763 } else if message.contains("high-risk") || message.contains("denied") {
4764 "PermissionDeniedError"
4765 } else {
4766 "RuntimeError"
4767 }
4768}
4769
4770fn failure_source(error_type: &str) -> &'static str {
4771 match error_type {
4772 "BudgetExceededError" => "budget",
4773 "SandboxUnavailableError" => "sandbox",
4774 "PermissionDeniedError" | "ApprovalDenied" => "policy",
4775 _ => "agent",
4776 }
4777}
4778
4779fn is_failure_event(kind: &str) -> bool {
4780 matches!(
4781 kind,
4782 "failure_classified"
4783 | "error_raised"
4784 | "step_failed"
4785 | "step_retry_scheduled"
4786 | "step_waiting_human"
4787 | "lease_expired"
4788 | "run_cancel_requested"
4789 | "run_cancelled"
4790 | "model_call_failed"
4791 | "tool_call_failed"
4792 | "tool_approval_required"
4793 | "budget_check_failed"
4794 )
4795}
4796
4797fn failure_category(text: &str, fallback: &str) -> String {
4798 let lower = text.to_lowercase();
4799 for category in ["sandbox", "budget", "policy", "model", "tool", "runtime"] {
4800 if lower.contains(category) {
4801 return category.to_string();
4802 }
4803 }
4804 if lower.contains("approval") || lower.contains("permission") || lower.contains("denied") {
4805 return "policy".to_string();
4806 }
4807 if lower.contains("lease") || lower.contains("worker") {
4808 return "runtime".to_string();
4809 }
4810 if lower.contains("cancel") {
4811 return "cancellation".to_string();
4812 }
4813 fallback.to_string()
4814}
4815
4816fn event_failure_category(event: &Event) -> String {
4817 match event.event_type.as_str() {
4818 "model_call_failed" => "model".to_string(),
4819 "tool_call_failed" | "tool_call_blocked" | "tool_approval_required" => "tool".to_string(),
4820 "run_cancel_requested" | "run_cancelled" | "step_cancelled" => "cancellation".to_string(),
4821 "lease_expired" => "runtime".to_string(),
4822 "step_retry_scheduled" => "retry".to_string(),
4823 "step_waiting_human" => "approval".to_string(),
4824 _ => failure_category(
4825 &format!(
4826 "{} {} {} {}",
4827 event.event_type,
4828 state_value_string(&event.payload, "error_type"),
4829 state_value_string(&event.payload, "error"),
4830 state_value_string(&event.payload, "reason")
4831 ),
4832 "agent",
4833 ),
4834 }
4835}
4836
4837fn event_failure_status(kind: &str, run_status: &str) -> String {
4838 match kind {
4839 "step_failed" | "run_cancelled" | "step_cancelled" => "terminal".to_string(),
4840 "tool_call_blocked" => "blocked".to_string(),
4841 "step_retry_scheduled" | "lease_expired" => "recovery_scheduled".to_string(),
4842 "step_waiting_human" | "tool_approval_required" => "waiting_human".to_string(),
4843 "failure_classified" => "classified".to_string(),
4844 "error_raised" if run_status == "failed" => "terminal".to_string(),
4845 _ => "failed".to_string(),
4846 }
4847}
4848
4849fn event_recoverability(kind: &str, run_status: &str) -> String {
4850 if run_status == "failed" && matches!(kind, "step_failed" | "run_cancelled" | "step_cancelled") {
4851 return "terminal".to_string();
4852 }
4853 match kind {
4854 "step_retry_scheduled" | "lease_expired" => "auto_retry".to_string(),
4855 "step_waiting_human" | "tool_approval_required" => "human_required".to_string(),
4856 "tool_call_blocked" => "manual_intervention".to_string(),
4857 _ => "unknown".to_string(),
4858 }
4859}
4860
4861fn event_retryability(kind: &str) -> String {
4862 match kind {
4863 "step_retry_scheduled" | "lease_expired" => "retryable".to_string(),
4864 "tool_call_blocked" | "run_cancelled" | "step_cancelled" => "not_retryable".to_string(),
4865 _ => "unknown".to_string(),
4866 }
4867}
4868
4869fn owner_for_failure_category(category: &str) -> String {
4870 match category {
4871 "tool" | "model" | "policy" | "sandbox" | "budget" | "runtime" => category.to_string(),
4872 "approval" | "cancellation" | "retry" => "runtime".to_string(),
4873 _ => "agent".to_string(),
4874 }
4875}
4876
4877fn first_text(values: &[Option<&str>]) -> String {
4878 for value in values {
4879 if let Some(text) = value {
4880 if !text.is_empty() {
4881 return (*text).to_string();
4882 }
4883 }
4884 }
4885 "failure signal".to_string()
4886}
4887
4888fn ref_state(kind: &str, value: &str) -> State {
4889 if value.is_empty() {
4890 return State::new();
4891 }
4892 state(&[("kind", kind.into()), ("value", value.into())])
4893}
4894
4895fn state_value_string(row: &State, key: &str) -> String {
4896 match row.get(key) {
4897 Some(Value::String(value)) => value.clone(),
4898 Some(Value::Number(value)) => value.to_string(),
4899 Some(Value::Bool(value)) => value.to_string(),
4900 Some(value) => format_value(value),
4901 None => String::new(),
4902 }
4903}
4904
4905fn state_string(row: &State, key: &str) -> Option<String> {
4906 let value = state_value_string(row, key);
4907 if value.is_empty() {
4908 None
4909 } else {
4910 Some(value)
4911 }
4912}
4913
4914fn state_number(row: &State, key: &str) -> f64 {
4915 match row.get(key) {
4916 Some(Value::Number(value)) => *value,
4917 _ => 0.0,
4918 }
4919}
4920
4921fn usage_total_tokens(usage: &State) -> f64 {
4922 for key in ["total_tokens", "totalTokens", "tokens"] {
4923 let value = state_number(usage, key);
4924 if value > 0.0 {
4925 return value;
4926 }
4927 }
4928 state_number(usage, "input_tokens")
4929 + state_number(usage, "prompt_tokens")
4930 + state_number(usage, "inputTokens")
4931 + state_number(usage, "output_tokens")
4932 + state_number(usage, "completion_tokens")
4933 + state_number(usage, "outputTokens")
4934}
4935
4936fn state_array_len(row: &State, key: &str) -> usize {
4937 match row.get(key) {
4938 Some(Value::Array(values)) => values.len(),
4939 _ => 0,
4940 }
4941}
4942
4943fn dedupe_states(rows: Vec<State>, key: &str) -> Vec<State> {
4944 let mut out = Vec::new();
4945 let mut seen = std::collections::HashSet::new();
4946 for row in rows {
4947 let value = state_value_string(&row, key);
4948 if value.is_empty() || seen.contains(&value) {
4949 continue;
4950 }
4951 seen.insert(value);
4952 out.push(row);
4953 }
4954 out
4955}
4956
4957fn count_state_field(rows: &[State], key: &str, expected: &str) -> usize {
4958 rows.iter()
4959 .filter(|row| state_value_string(row, key) == expected)
4960 .count()
4961}
4962
4963fn count_recoverable_failures(rows: &[State]) -> usize {
4964 rows.iter()
4965 .filter(|row| matches!(state_value_string(row, "recoverability").as_str(), "auto_retry" | "recoverable" | "manual_verification" | "human_required"))
4966 .count()
4967}
4968
4969fn slug(value: &str) -> String {
4970 let mut out = String::new();
4971 let mut previous_dash = false;
4972 for ch in value.chars() {
4973 if ch.is_ascii_alphanumeric() {
4974 out.push(ch.to_ascii_lowercase());
4975 previous_dash = false;
4976 } else if !previous_dash {
4977 out.push('-');
4978 previous_dash = true;
4979 }
4980 }
4981 let trimmed = out.trim_matches('-').to_string();
4982 if trimmed.is_empty() {
4983 "unknown".to_string()
4984 } else {
4985 trimmed
4986 }
4987}
4988
4989fn format_state(state: &State) -> String {
4990 let mut keys: Vec<&String> = state.keys().collect();
4991 keys.sort();
4992 keys.into_iter()
4993 .map(|key| format!("{}={}", key, format_value(&state[key])))
4994 .collect::<Vec<_>>()
4995 .join(",")
4996}
4997
4998fn format_value(value: &Value) -> String {
4999 match value {
5000 Value::Null => "null".to_string(),
5001 Value::Bool(value) => value.to_string(),
5002 Value::Number(value) => value.to_string(),
5003 Value::String(value) => format!("\"{}\"", value),
5004 Value::Object(value) => format!("{{{}}}", format_state(value)),
5005 Value::Array(values) => format!(
5006 "[{}]",
5007 values
5008 .iter()
5009 .map(format_value)
5010 .collect::<Vec<_>>()
5011 .join(",")
5012 ),
5013 }
5014}
5015
5016fn stable_hash(input: &str) -> String {
5017 let mut hash: u64 = 0xcbf29ce484222325;
5018 for byte in input.as_bytes() {
5019 hash ^= *byte as u64;
5020 hash = hash.wrapping_mul(0x100000001b3);
5021 }
5022 format!("{hash:016x}")
5023}
5024
5025fn new_id(prefix: &str) -> String {
5026 let value = ID_COUNTER.fetch_add(1, Ordering::Relaxed);
5027 format!("{prefix}_{value:016x}")
5028}
5029
5030fn now_seconds() -> f64 {
5031 SystemTime::now()
5032 .duration_since(UNIX_EPOCH)
5033 .unwrap()
5034 .as_secs_f64()
5035}
5036
5037#[cfg(test)]
5038mod tests {
5039 use super::*;
5040
5041 fn state(items: &[(&str, Value)]) -> State {
5042 items
5043 .iter()
5044 .map(|(key, value)| ((*key).to_string(), value.clone()))
5045 .collect()
5046 }
5047
5048 fn event_exists(events: &[Event], event_type: &str) -> bool {
5049 events.iter().any(|event| event.event_type == event_type)
5050 }
5051
5052 fn claim_context(
5053 runtime: &mut Runtime,
5054 run_id: &str,
5055 worker: &str,
5056 role: &str,
5057 ) -> AgentContext {
5058 let claim = runtime.store.claim_step(worker, run_id, 60.0).unwrap();
5059 let (_state, version, session_id) = runtime.store.load_state(run_id).unwrap();
5060 let mut payload = State::new();
5061 payload.insert("agent_role".to_string(), Value::String(role.to_string()));
5062 runtime.store.append_event(
5063 run_id,
5064 Some(&session_id),
5065 Some(&claim.step_id),
5066 "agent_started",
5067 payload,
5068 Some(role),
5069 Some(version),
5070 None,
5071 );
5072 AgentContext {
5073 run_id: run_id.to_string(),
5074 session_id,
5075 step_id: claim.step_id,
5076 agent_role: role.to_string(),
5077 lease_token: claim.lease_token,
5078 attempt: claim.attempt,
5079 state_version: version,
5080 pending_patch: State::new(),
5081 }
5082 }
5083
5084 #[test]
5085 fn runtime_creates_evidence_and_replay() {
5086 let mut runtime = Runtime::new();
5087 runtime.register_tool(ToolSpec::new(
5088 "docs.echo",
5089 Box::new(|args| Ok(Value::Object(state(&[("echo", args["text"].clone())])))),
5090 ));
5091 let (run_id, _) = runtime.create_run(state(&[("input", "hello".into())]));
5092 let ok = runtime
5093 .run_once(&run_id, "worker-a", "Researcher", 60.0, |ctx, state| {
5094 let mut result = State::new();
5095 result.insert("from_state".to_string(), state["input"].clone());
5096 ctx.write_state("tool_result", Value::Object(result));
5097 Ok(())
5098 })
5099 .unwrap();
5100 assert!(ok);
5101 let bundle = export_evidence(&runtime.store, &run_id).unwrap();
5102 assert_eq!(bundle.schema_version, "agentledger.evidence.v1");
5103 let summary = replay(&runtime.store, &run_id).unwrap();
5104 assert!(summary.replay_safe);
5105 assert_eq!(summary.event_count, bundle.events.len());
5106 }
5107
5108 #[test]
5109 fn local_snapshot_store_round_trips_completed_run() {
5110 let mut runtime = Runtime::new();
5111 let (run_id, _) = runtime.create_run(state(&[("input", "hello".into())]));
5112 runtime
5113 .run_once(&run_id, "worker-a", "Researcher", 60.0, |ctx, state| {
5114 let mut result = State::new();
5115 result.insert("echo".to_string(), state["input"].clone());
5116 ctx.write_state("tool_result", Value::Object(result));
5117 Ok(())
5118 })
5119 .unwrap();
5120 let path =
5121 std::env::temp_dir().join(format!("agentledger-rust-{}.store", new_id("snapshot")));
5122 runtime.store.save_to_path(&path).unwrap();
5123
5124 let reopened = MemoryStore::load_from_path(&path).unwrap();
5125 let final_state = reopened.final_state(&run_id).unwrap();
5126 assert_eq!(
5127 final_state.get("tool_result"),
5128 Some(&Value::Object(state(&[("echo", "hello".into())])))
5129 );
5130 let bundle = export_evidence(&reopened, &run_id).unwrap();
5131 assert_eq!(reopened.steps(&run_id).len(), 1);
5132 assert_eq!(
5133 replay(&reopened, &run_id).unwrap().event_count,
5134 bundle.events.len()
5135 );
5136 let _ = std::fs::remove_file(path);
5137 }
5138
5139 #[test]
5140 fn local_blob_store_round_trips_json_values() {
5141 let root = std::env::temp_dir().join(format!("agentledger-rust-blobs-{}", new_id("blob")));
5142 let blobs = LocalBlobStore::open(&root).unwrap();
5143 let value = Value::Object(state(&[(
5144 "hello",
5145 Value::Object(state(&[("nested", "world".into())])),
5146 )]));
5147 let first = blobs.put_json(&value).unwrap();
5148 let second = blobs.put_json(&value).unwrap();
5149 assert!(first.0.starts_with("sha256:"));
5150 assert!(first.1.starts_with("blob://sha256/"));
5151 assert_eq!(first, second);
5152 assert_eq!(blobs.get_json(&first.1).unwrap(), value);
5153 assert!(blobs.get_json("unsupported://blob").is_err());
5154 let _ = std::fs::remove_dir_all(root);
5155 }
5156
5157 #[test]
5158 fn tool_schema_validation_rejects_invalid_input() {
5159 let input_schema = Value::Object(state(&[
5160 ("type", "object".into()),
5161 (
5162 "required",
5163 Value::Array(vec![Value::String("text".to_string())]),
5164 ),
5165 ("additionalProperties", false.into()),
5166 (
5167 "properties",
5168 Value::Object(state(&[(
5169 "text",
5170 Value::Object(state(&[("type", "string".into())])),
5171 )])),
5172 ),
5173 ]));
5174 let mut runtime = Runtime::new();
5175 runtime.register_tool(
5176 ToolSpec::new(
5177 "docs.echo",
5178 Box::new(|args| Ok(Value::Object(state(&[("echo", args["text"].clone())])))),
5179 )
5180 .input_schema(input_schema),
5181 );
5182 let (run_id, _) = runtime.create_run(State::new());
5183 let ctx = claim_context(&mut runtime, &run_id, "worker", "SchemaAgent");
5184 assert!(runtime.call_tool(&ctx, "docs.echo", State::new()).is_err());
5185 assert!(runtime.store.events(&run_id).iter().any(|event| {
5186 event.event_type == "tool_call_failed"
5187 && event.payload.get("phase")
5188 == Some(&Value::String("input_validation".to_string()))
5189 }));
5190 }
5191
5192 #[test]
5193 fn tool_ledger_reuses_side_effect_after_retry() {
5194 let mut runtime = Runtime::new();
5195 runtime.register_tool(
5196 ToolSpec::new(
5197 "github.create_pr",
5198 Box::new(|args| {
5199 Ok(Value::Object(state(&[
5200 ("external_id", "pr-123".into()),
5201 ("title", args["title"].clone()),
5202 ])))
5203 }),
5204 )
5205 .side_effect("external")
5206 .idempotency_required(true),
5207 );
5208 let (run_id, _) = runtime.create_run(state(&[("title", "runtime parity".into())]));
5209 let ctx = claim_context(&mut runtime, &run_id, "worker-a", "Coder");
5210 let first = runtime
5211 .call_tool(
5212 &ctx,
5213 "github.create_pr",
5214 state(&[("title", "runtime parity".into())]),
5215 )
5216 .unwrap();
5217 runtime
5218 .store
5219 .mark_retry(&run_id, &ctx.step_id, "RetryableAgentError", "retryable");
5220 let ctx2 = claim_context(&mut runtime, &run_id, "worker-b", "Coder");
5221 let second = runtime
5222 .call_tool(
5223 &ctx2,
5224 "github.create_pr",
5225 state(&[("title", "runtime parity".into())]),
5226 )
5227 .unwrap();
5228 assert_eq!(first, second);
5229 assert_eq!(runtime.store.ledger(&run_id).len(), 1);
5230 }
5231
5232 #[test]
5233 fn policy_denies_unapproved_high_risk_tool() {
5234 let mut runtime = Runtime::new();
5235 runtime.register_tool(
5236 ToolSpec::new("repo.write", Box::new(|_| Ok(Value::Bool(true)))).risk_level("high"),
5237 );
5238 let (run_id, _) = runtime.create_run(State::new());
5239 let ctx = claim_context(&mut runtime, &run_id, "worker", "Reviewer");
5240 let err = runtime
5241 .call_tool(&ctx, "repo.write", state(&[("path", "README.md".into())]))
5242 .unwrap_err();
5243 assert!(err.0.contains("high-risk"));
5244 assert!(runtime
5245 .store
5246 .events(&run_id)
5247 .iter()
5248 .any(|event| event.event_type == "tool_permission_decided"
5249 && event.payload.get("allowed") == Some(&Value::Bool(false))));
5250 }
5251
5252 #[test]
5253 fn approval_pauses_and_resumes_step() {
5254 let mut runtime = Runtime::new();
5255 runtime.register_tool(
5256 ToolSpec::new(
5257 "github.create_pr",
5258 Box::new(|_| Ok(Value::Object(state(&[("external_id", "pr-42".into())])))),
5259 )
5260 .risk_level("high")
5261 .approval_required(true)
5262 .side_effect("external")
5263 .idempotency_required(true),
5264 );
5265 let (run_id, _) = runtime.create_run(State::new());
5266 let ctx = claim_context(&mut runtime, &run_id, "worker-a", "Coder");
5267 let err = runtime
5268 .call_tool(&ctx, "github.create_pr", state(&[("title", "safe".into())]))
5269 .unwrap_err();
5270 assert!(err.0.starts_with("approval required:"));
5271 let approval_id = err.0.trim_start_matches("approval required:").to_string();
5272 runtime
5273 .store
5274 .mark_waiting_human(&run_id, &ctx.step_id, &err.0, &approval_id);
5275 assert_eq!(runtime.store.steps(&run_id)[0].status, "waiting_human");
5276 runtime
5277 .store
5278 .approve_request(&approval_id, "alice", "reviewed")
5279 .unwrap();
5280 let ctx2 = claim_context(&mut runtime, &run_id, "worker-b", "Coder");
5281 let result = runtime
5282 .call_tool(
5283 &ctx2,
5284 "github.create_pr",
5285 state(&[("title", "safe".into())]),
5286 )
5287 .unwrap();
5288 assert!(matches!(result, Value::Object(_)));
5289
5290 let (denied_run, _) = runtime.create_run(State::new());
5291 let denied_ctx = claim_context(&mut runtime, &denied_run, "worker-c", "Coder");
5292 let denied_err = runtime
5293 .call_tool(
5294 &denied_ctx,
5295 "github.create_pr",
5296 state(&[("title", "blocked".into())]),
5297 )
5298 .unwrap_err();
5299 let denied_id = denied_err
5300 .0
5301 .trim_start_matches("approval required:")
5302 .to_string();
5303 runtime.store.mark_waiting_human(
5304 &denied_run,
5305 &denied_ctx.step_id,
5306 &denied_err.0,
5307 &denied_id,
5308 );
5309 runtime
5310 .store
5311 .deny_request(&denied_id, "bob", "not allowed")
5312 .unwrap();
5313 assert_eq!(runtime.store.steps(&denied_run)[0].status, "failed");
5314 }
5315
5316 #[test]
5317 fn mcp_tool_adapter_maps_governance_annotations() {
5318 fn call(_name: &str, _args: State) -> Result<Value> {
5319 Ok(Value::Object(state(&[("ok", true.into())])))
5320 }
5321 let adapter = MCPToolAdapter { client_call: call };
5322 let spec = adapter.tool_spec_from_descriptor(&state(&[
5323 ("name", "mcp.github.create_pr".into()),
5324 (
5325 "inputSchema",
5326 Value::Object(state(&[
5327 ("type", "object".into()),
5328 (
5329 "required",
5330 Value::Array(vec![Value::String("title".to_string())]),
5331 ),
5332 ])),
5333 ),
5334 (
5335 "annotations",
5336 Value::Object(state(&[
5337 ("side_effect", "external_write".into()),
5338 ("risk_level", "high".into()),
5339 ("idempotency_required", true.into()),
5340 ("approval_required", true.into()),
5341 ("sandbox_required", true.into()),
5342 ("sandbox_executor", "docker".into()),
5343 (
5344 "sandbox_policy",
5345 Value::Object(state(&[
5346 ("network", "deny".into()),
5347 ("filesystem", "read-only".into()),
5348 ])),
5349 ),
5350 ])),
5351 ),
5352 ]));
5353 assert_eq!(spec.side_effect, "external_write");
5354 assert_eq!(spec.risk_level, "high");
5355 assert!(spec.idempotency_required);
5356 assert!(spec.approval_required);
5357 assert!(spec.sandbox_required);
5358 assert_eq!(spec.sandbox_executor, "docker");
5359 assert_eq!(
5360 spec.sandbox_policy.get("network"),
5361 Some(&Value::String("deny".to_string()))
5362 );
5363 assert_eq!(
5364 spec.sandbox_policy.get("filesystem"),
5365 Some(&Value::String("read-only".to_string()))
5366 );
5367 assert!(spec.input_schema.is_some());
5368 }
5369
5370 #[test]
5371 fn sandbox_required_tool_fails_closed() {
5372 let mut runtime = Runtime::new();
5373 runtime.register_tool(
5374 ToolSpec::new("shell.exec", Box::new(|_| Ok(Value::Bool(true)))).sandbox_required(true),
5375 );
5376 let (run_id, _) = runtime.create_run(State::new());
5377 let ctx = claim_context(&mut runtime, &run_id, "worker", "Executor");
5378 let err = runtime
5379 .call_tool(
5380 &ctx,
5381 "shell.exec",
5382 state(&[("argv", Value::Object(State::new()))]),
5383 )
5384 .unwrap_err();
5385 assert!(err.0.contains("sandbox executor"));
5386 let events = runtime.store.events(&run_id);
5387 assert!(event_exists(&events, "sandbox_started"));
5388 assert!(event_exists(&events, "tool_call_failed"));
5389 }
5390
5391 #[test]
5392 fn docker_sandbox_executor_requires_explicit_execution() {
5393 let executor = DockerSandboxExecutor::new("fake-image", false).with_binary("/bin/echo");
5394 let result = executor.run_tool(
5395 state(&[("_sandbox_command", Value::Array(vec!["echo".into(), "hi".into()]))]),
5396 &SandboxPolicy {
5397 tool_name: "cmd.echo".to_string(),
5398 run_id: "run".to_string(),
5399 step_id: "step".to_string(),
5400 executor: "docker".to_string(),
5401 network: "deny".to_string(),
5402 filesystem: "read-only".to_string(),
5403 timeout_seconds: 1,
5404 extra: State::new(),
5405 },
5406 );
5407 assert!(!result.ok);
5408 assert_eq!(result.metadata.get("error_type"), Some(&Value::String("SandboxAdapterNotInstalled".to_string())));
5409 }
5410
5411 #[test]
5412 fn docker_sandbox_executor_runs_command_style_tool_with_injected_binary() {
5413 let mut runtime = Runtime::new();
5414 runtime.set_sandbox(Box::new(DockerSandboxExecutor::new("fake-image", true).with_binary("/bin/echo")));
5415 runtime.register_tool(
5416 ToolSpec::new("cmd.echo", Box::new(|_| Err(RuntimeError("direct func should not execute".to_string()))))
5417 .sandbox_required(true)
5418 .sandbox_executor("docker"),
5419 );
5420 let (run_id, _) = runtime.create_run(State::new());
5421 let ctx = claim_context(&mut runtime, &run_id, "worker", "Executor");
5422 let value = runtime
5423 .call_tool(
5424 &ctx,
5425 "cmd.echo",
5426 state(&[("_sandbox_command", Value::Array(vec!["echo".into(), "hi".into()]))]),
5427 )
5428 .unwrap();
5429 let output = match value { Value::Object(output) => output, _ => panic!("expected object output") };
5430 let stdout = match output.get("stdout") { Some(Value::String(value)) => value, _ => panic!("expected stdout") };
5431 assert!(stdout.contains("run"));
5432 assert!(stdout.contains("fake-image"));
5433 let events = runtime.store.events(&run_id);
5434 assert!(event_exists(&events, "sandbox_completed"));
5435 assert!(event_exists(&events, "tool_call_completed"));
5436 }
5437
5438 #[test]
5439 fn cost_budget_and_failure_attribution() {
5440 let mut runtime = Runtime::new();
5441 runtime.set_budget(BudgetLimits {
5442 max_tool_calls: Some(1.0),
5443 max_model_tokens: None,
5444 max_total_usd: None,
5445 });
5446 runtime.register_tool(ToolSpec::new(
5447 "docs.echo",
5448 Box::new(|args| Ok(Value::Object(state(&[("echo", args["text"].clone())])))),
5449 ));
5450 let (run_id, _) = runtime.create_run(State::new());
5451 let ctx = claim_context(&mut runtime, &run_id, "worker", "Researcher");
5452 runtime
5453 .record_model_call(&ctx, "gpt-test", 10.0, 5.0, 0.01)
5454 .unwrap();
5455 runtime
5456 .call_tool(&ctx, "docs.echo", state(&[("text", "first".into())]))
5457 .unwrap();
5458 let err = runtime
5459 .call_tool(&ctx, "docs.echo", state(&[("text", "second".into())]))
5460 .unwrap_err();
5461 runtime.store.mark_failed(
5462 &run_id,
5463 &ctx.step_id,
5464 classify_runtime_error(&err.0),
5465 &err.0,
5466 );
5467 let summary = runtime.store.cost_summary(&run_id);
5468 assert_eq!(summary.tool_calls, 1.0);
5469 assert_eq!(summary.model_tokens, 15.0);
5470 assert_eq!(summary.total_usd, 0.01);
5471 let cost = cost_attribution(&runtime.store, &run_id);
5472 assert_eq!(cost.by_agent["Researcher"].tool_calls, 1.0);
5473 assert_eq!(cost.by_agent["Researcher"].model_tokens, 15.0);
5474 let failure = failure_attribution(&runtime.store, &run_id).unwrap();
5475 assert_eq!(failure.failed_steps.len(), 1);
5476 assert!(event_exists(&failure.failure_events, "budget_check_failed"));
5477 assert!(event_exists(&failure.failure_events, "failure_classified"));
5478 assert!(!failure.failure_envelopes.is_empty());
5479 assert_eq!(
5480 failure.failure_lifecycle["schema_version"],
5481 Value::String("agentledger.failure.lifecycle.v1".to_string())
5482 );
5483 assert_eq!(
5484 failure.failure_export["schema_version"],
5485 Value::String("agentledger.failure.export.v1".to_string())
5486 );
5487 assert_eq!(failure.failure_replay_plan["safe_to_replay"], Value::Bool(true));
5488 assert!(state_number(&failure.failure_alerts, "alert_count") > 0.0);
5489 let events = runtime.store.events(&run_id);
5490 assert!(event_exists(&events, "model_call_requested"));
5491 assert!(event_exists(&events, "model_call_completed"));
5492 }
5493
5494 #[test]
5495 fn model_evidence_boundary_records_failure_and_tool_proposal() {
5496 let mut runtime = Runtime::new();
5497 let (run_id, _) = runtime.create_run(State::new());
5498 let ctx = claim_context(&mut runtime, &run_id, "worker", "Researcher");
5499 runtime
5500 .record_model_failure(
5501 &ctx,
5502 "deepseek",
5503 "deepseek-chat",
5504 "RateLimitError",
5505 "rate limited",
5506 Some(true),
5507 state(&[("messages", Value::Array(vec!["hello".into()]))]),
5508 State::new(),
5509 0.0,
5510 State::new(),
5511 )
5512 .unwrap();
5513 runtime.record_tool_call_proposal(
5514 &ctx,
5515 "search_contract_clause",
5516 state(&[("clause", "payment".into())]),
5517 Some("deepseek"),
5518 Some("deepseek-chat"),
5519 None,
5520 None,
5521 Some("model requested clause search"),
5522 State::new(),
5523 );
5524 let events = runtime.store.events(&run_id);
5525 assert!(event_exists(&events, "model_call_failed"));
5526 assert!(event_exists(&events, "tool_call_proposed"));
5527 let failure = failure_attribution(&runtime.store, &run_id).unwrap();
5528 assert!(failure
5529 .failure_envelopes
5530 .iter()
5531 .any(|item| item.get("category") == Some(&Value::String("model".to_string()))));
5532 }
5533
5534 #[test]
5535 fn omp_bridge_records_session_turn_model_tool_and_state_evidence() {
5536 let runtime = Runtime::new();
5537 let mut bridge = OmpLedgerBridge::new(runtime, "omp-demo");
5538 let run_id = bridge
5539 .record_session_started(OmpSession {
5540 session_id: "sess-omp-1".to_string(),
5541 initial_state: state(&[("topic", "contract".into())]),
5542 metadata: state(&[("runtime", "omp".into())]),
5543 ..Default::default()
5544 })
5545 .unwrap();
5546 let step_id = bridge
5547 .record_turn_started(OmpTurn {
5548 session_id: "sess-omp-1".to_string(),
5549 turn_id: "turn-1".to_string(),
5550 agent_role: "OMPPlanner".to_string(),
5551 metadata: state(&[("phase", "plan".into())]),
5552 ..Default::default()
5553 })
5554 .unwrap();
5555 bridge
5556 .record_model_call(OmpModelCall {
5557 session_id: "sess-omp-1".to_string(),
5558 turn_id: "turn-1".to_string(),
5559 provider: "openai".to_string(),
5560 model: "gpt-4.1".to_string(),
5561 request: state(&[(
5562 "messages",
5563 Value::Array(vec![Value::Object(state(&[
5564 ("role", "user".into()),
5565 ("content", "find payment clause".into()),
5566 ]))]),
5567 )]),
5568 response: state(&[(
5569 "tool_calls",
5570 Value::Array(vec![Value::Object(state(&[(
5571 "name",
5572 "search_contract_clause".into(),
5573 )]))]),
5574 )]),
5575 usage: state(&[
5576 ("input_tokens", Value::Number(9.0)),
5577 ("output_tokens", Value::Number(4.0)),
5578 ]),
5579 total_usd: 0.003,
5580 ..Default::default()
5581 })
5582 .unwrap();
5583 bridge
5584 .record_tool_proposal(OmpToolProposal {
5585 session_id: "sess-omp-1".to_string(),
5586 turn_id: "turn-1".to_string(),
5587 tool_name: "search_contract_clause".to_string(),
5588 arguments: state(&[("clause", "payment".into())]),
5589 provider: Some("openai".to_string()),
5590 model: Some("gpt-4.1".to_string()),
5591 reason: Some("model requested clause search".to_string()),
5592 ..Default::default()
5593 })
5594 .unwrap();
5595 let tool_record = bridge
5596 .record_tool_execution(OmpToolExecution {
5597 session_id: "sess-omp-1".to_string(),
5598 turn_id: "turn-1".to_string(),
5599 tool_name: "search_contract_clause".to_string(),
5600 arguments: state(&[("clause", "payment".into())]),
5601 result: Some(Value::Object(state(&[(
5602 "matches",
5603 Value::Array(vec!["Section 9".into()]),
5604 )]))),
5605 ledger_status: Some("SUCCEEDED".to_string()),
5606 ..Default::default()
5607 })
5608 .unwrap();
5609 assert_eq!(
5610 tool_record.get("ledger_status"),
5611 Some(&Value::String("SUCCEEDED".to_string()))
5612 );
5613 bridge
5614 .record_state_change(OmpStateChange {
5615 session_id: "sess-omp-1".to_string(),
5616 turn_id: Some("turn-1".to_string()),
5617 reason: "persist clause search results".to_string(),
5618 patch: state(&[("memory_version", Value::Number(1.0))]),
5619 before_snapshot: Some(Value::Object(state(&[(
5620 "memory_version",
5621 Value::Number(0.0),
5622 )]))),
5623 after_snapshot: Some(Value::Object(state(&[(
5624 "memory_version",
5625 Value::Number(1.0),
5626 )]))),
5627 diff: Some(Value::Object(state(&[(
5628 "memory_version",
5629 Value::Array(vec![Value::Number(0.0), Value::Number(1.0)]),
5630 )]))),
5631 metadata: state(&[("document", "MEMORY.md".into())]),
5632 ..Default::default()
5633 })
5634 .unwrap();
5635 let version = bridge
5636 .record_turn_completed(OmpTurn {
5637 session_id: "sess-omp-1".to_string(),
5638 turn_id: "turn-1".to_string(),
5639 agent_role: "OMPPlanner".to_string(),
5640 state_patch: state(&[("last_clause", "payment".into())]),
5641 ..Default::default()
5642 })
5643 .unwrap();
5644 assert!(version >= 2);
5645 let events = bridge.runtime.store.events(&run_id);
5646 for event_type in [
5647 "omp_session_started",
5648 "omp_turn_started",
5649 "model_call_requested",
5650 "tool_call_proposed",
5651 "tool_call_completed",
5652 "omp_state_change_recorded",
5653 "step_completed",
5654 ] {
5655 assert!(event_exists(&events, event_type), "missing event {event_type}");
5656 }
5657 assert!(bridge
5658 .runtime
5659 .store
5660 .steps(&run_id)
5661 .iter()
5662 .any(|step| step.step_id == step_id && step.status == "completed"));
5663 let final_state = bridge.runtime.store.final_state(&run_id).unwrap();
5664 assert_eq!(final_state.get("memory_version"), Some(&Value::Number(1.0)));
5665 assert_eq!(final_state.get("last_clause"), Some(&Value::String("payment".to_string())));
5666 let ledger = bridge.runtime.store.ledger(&run_id);
5667 assert_eq!(ledger.len(), 1);
5668 assert_eq!(ledger[0].status, "SUCCEEDED");
5669 }
5670
5671 #[test]
5672 fn omp_bridge_records_model_failure_and_retry_lifecycle() {
5673 let runtime = Runtime::new();
5674 let mut bridge = OmpLedgerBridge::new(runtime, "omp-demo");
5675 let run_id = bridge
5676 .record_session_started(OmpSession {
5677 session_id: "sess-omp-2".to_string(),
5678 ..Default::default()
5679 })
5680 .unwrap();
5681 bridge
5682 .record_turn_started(OmpTurn {
5683 session_id: "sess-omp-2".to_string(),
5684 turn_id: "turn-1".to_string(),
5685 agent_role: "OMPResearcher".to_string(),
5686 ..Default::default()
5687 })
5688 .unwrap();
5689 let step_id = bridge.runtime.store.steps(&run_id)[0].step_id.clone();
5690 bridge
5691 .record_failure(OmpFailure {
5692 session_id: "sess-omp-2".to_string(),
5693 turn_id: "turn-1".to_string(),
5694 category: "model".to_string(),
5695 provider: Some("deepseek".to_string()),
5696 model: Some("deepseek-chat".to_string()),
5697 error_type: "RateLimitError".to_string(),
5698 message: "rate limited".to_string(),
5699 retryable: Some(true),
5700 request: state(&[(
5701 "messages",
5702 Value::Array(vec!["hello".into()]),
5703 )]),
5704 terminal: Some(false),
5705 ..Default::default()
5706 })
5707 .unwrap();
5708 bridge
5709 .record_failure(OmpFailure {
5710 session_id: "sess-omp-2".to_string(),
5711 turn_id: "turn-1".to_string(),
5712 error_type: "RetryableAgentError".to_string(),
5713 message: "retry later".to_string(),
5714 retryable: Some(true),
5715 status: "retry_scheduled".to_string(),
5716 ..Default::default()
5717 })
5718 .unwrap();
5719 let events = bridge.runtime.store.events(&run_id);
5720 assert!(event_exists(&events, "model_call_failed"));
5721 assert!(event_exists(&events, "step_retry_scheduled"));
5722 let steps = bridge.runtime.store.steps(&run_id);
5723 assert_eq!(steps.len(), 1);
5724 assert_eq!(steps[0].status, "retry_scheduled");
5725 let retry_step_id = bridge
5726 .record_turn_started(OmpTurn {
5727 session_id: "sess-omp-2".to_string(),
5728 turn_id: "turn-1".to_string(),
5729 agent_role: "OMPResearcher".to_string(),
5730 ..Default::default()
5731 })
5732 .unwrap();
5733 assert_eq!(retry_step_id, step_id);
5734 let steps = bridge.runtime.store.steps(&run_id);
5735 assert_eq!(steps.len(), 1);
5736 assert_eq!(steps[0].status, "running");
5737 }
5738
5739 #[test]
5740 fn media_and_stream_artifacts_are_indexed_in_evidence_and_replay() {
5741 let mut runtime = Runtime::new();
5742 let (run_id, _) = runtime.create_run(State::new());
5743 let ctx = claim_context(&mut runtime, &run_id, "worker-media", "MediaAgent");
5744 let mut media_metadata = State::new();
5745 media_metadata.insert(
5746 "mime_type".to_string(),
5747 Value::String("image/jpeg".to_string()),
5748 );
5749 media_metadata.insert("frame_index".to_string(), Value::Number(1.0));
5750 let mut lineage = State::new();
5751 lineage.insert(
5752 "source_blob_ref".to_string(),
5753 Value::String("s3://media/demo/input.mp4".to_string()),
5754 );
5755 lineage.insert(
5756 "tool_call_id".to_string(),
5757 Value::String("video.extract_frames".to_string()),
5758 );
5759 let frame_id = runtime
5760 .create_media_artifact(
5761 &ctx,
5762 "frame-0001",
5763 "frame",
5764 MediaArtifactOptions {
5765 uri: Some("s3://media/demo/frame-0001.jpg".to_string()),
5766 media_metadata,
5767 lineage,
5768 ..Default::default()
5769 },
5770 )
5771 .unwrap();
5772 let checkpoint_id = runtime
5773 .create_stream_checkpoint(
5774 &ctx,
5775 "camera-checkpoint",
5776 StreamCheckpointOptions {
5777 stream_id: "camera-1".to_string(),
5778 consumer_id: "vision-agent".to_string(),
5779 offset: Value::Number(7.0),
5780 watermark: Some(Value::Number(1.5)),
5781 chunk: Some(StreamChunkRef {
5782 stream_id: "camera-1".to_string(),
5783 chunk_id: "chunk-7".to_string(),
5784 offset: Value::Number(7.0),
5785 content_ref: Some("blob://sha256/chunk-7.json".to_string()),
5786 sequence: Some(7.0),
5787 ..Default::default()
5788 }),
5789 ..Default::default()
5790 },
5791 )
5792 .unwrap();
5793 let mut artifacts = State::new();
5794 artifacts.insert("frame".to_string(), Value::String(frame_id));
5795 artifacts.insert("checkpoint".to_string(), Value::String(checkpoint_id));
5796 runtime
5797 .store
5798 .commit_state_patch(
5799 &run_id,
5800 &ctx.step_id,
5801 &ctx.lease_token,
5802 ctx.state_version,
5803 state(&[("artifacts", Value::Object(artifacts))]),
5804 )
5805 .unwrap();
5806 let bundle = export_evidence(&runtime.store, &run_id).unwrap();
5807 assert_eq!(bundle.artifacts.len(), 2);
5808 assert_eq!(bundle.media_artifacts.len(), 1);
5809 assert_eq!(bundle.stream_checkpoints.len(), 1);
5810 assert_eq!(
5811 bundle.media_artifacts[0].get("kind"),
5812 Some(&Value::String("frame".to_string()))
5813 );
5814 assert_eq!(
5815 bundle.stream_checkpoints[0].get("stream_id"),
5816 Some(&Value::String("camera-1".to_string()))
5817 );
5818 let summary = replay(&runtime.store, &run_id).unwrap();
5819 assert_eq!(summary.artifact_count, 2);
5820 assert_eq!(summary.media_artifact_count, 1);
5821 assert_eq!(summary.stream_checkpoint_count, 1);
5822 }
5823
5824 #[test]
5825 fn lease_recovery_fences_previous_owner() {
5826 let mut store = MemoryStore::new();
5827 let (run_id, step_id) = store.create_run(State::new());
5828 let claim = store.claim_step("stale-worker", &run_id, 0.0).unwrap();
5829 assert_eq!(store.recover_expired_leases(), 1);
5830 assert!(store
5831 .commit_state_patch(
5832 &run_id,
5833 &step_id,
5834 &claim.lease_token,
5835 0,
5836 state(&[("late", true.into())])
5837 )
5838 .is_err());
5839 assert!(store.claim_step("new-worker", &run_id, 60.0).is_ok());
5840 }
5841
5842 #[test]
5843 fn cancellation_fences_worker() {
5844 let mut store = MemoryStore::new();
5845 let (run_id, step_id) = store.create_run(State::new());
5846 let claim = store.claim_step("worker", &run_id, 60.0).unwrap();
5847 assert_eq!(store.cancel_run(&run_id, "operator requested").unwrap(), 1);
5848 assert!(store
5849 .commit_state_patch(
5850 &run_id,
5851 &step_id,
5852 &claim.lease_token,
5853 0,
5854 state(&[("late", true.into())])
5855 )
5856 .is_err());
5857 }
5858
5859 #[test]
5860 fn shared_runtime_baseline_fixture() {
5861 let fixture =
5862 std::fs::read_to_string("../contracts/conformance/runtime_baseline.v1.json").unwrap();
5863 assert!(fixture.contains("agentledger.conformance.runtime_baseline.v1"));
5864 for scenario in [
5865 "durable_run_evidence_replay",
5866 "tool_ledger_idempotent_retry",
5867 "lease_recovery_fences_stale_worker",
5868 "cancellation_fences_worker",
5869 ] {
5870 assert!(
5871 fixture.contains(scenario),
5872 "missing shared fixture scenario {scenario}"
5873 );
5874 }
5875 }
5876
5877 #[test]
5878 fn shared_parity_fixtures() {
5879 let fixtures: Vec<(&str, &[&str])> = vec![
5880 (
5881 "../contracts/conformance/policy_approval_sandbox.v1.json",
5882 &[
5883 "agentledger.conformance.policy_approval_sandbox.v1",
5884 "policy_denies_unapproved_high_risk_tool",
5885 "approval_pauses_and_resumes_step",
5886 "sandbox_required_tool_fails_closed",
5887 ],
5888 ),
5889 (
5890 "../contracts/conformance/cost_failure_attribution.v1.json",
5891 &[
5892 "agentledger.conformance.cost_failure_attribution.v1",
5893 "tool_and_model_cost_attributed_to_run_step_role",
5894 "budget_exhaustion_blocks_execution",
5895 "failure_attribution_classifies_agent_tool_model_runtime",
5896 ],
5897 ),
5898 (
5899 "../contracts/conformance/local_persistence.v1.json",
5900 &[
5901 "agentledger.conformance.local_persistence.v1",
5902 "local_store_round_trips_completed_run",
5903 "local_store_preserves_evidence_replay_chain",
5904 "local_store_uses_atomic_snapshot_write",
5905 ],
5906 ),
5907 (
5908 "../contracts/conformance/local_blob_store.v1.json",
5909 &[
5910 "agentledger.conformance.local_blob_store.v1",
5911 "blob_roundtrip_json_value",
5912 "blob_content_address_is_stable",
5913 "blob_bad_ref_is_rejected",
5914 ],
5915 ),
5916 (
5917 "../contracts/conformance/tool_schema_validation.v1.json",
5918 &[
5919 "agentledger.conformance.tool_schema_validation.v1",
5920 "invalid_tool_input_rejected_before_execution",
5921 "valid_tool_input_and_output_pass",
5922 "invalid_tool_output_rejected",
5923 ],
5924 ),
5925 (
5926 "../contracts/conformance/worker_service.v1.json",
5927 &[
5928 "agentledger.conformance.worker_service.v1",
5929 "local_worker_runs_until_terminal",
5930 "worker_service_stops_after_idle_poll",
5931 "worker_loop_recovers_expired_leases",
5932 ],
5933 ),
5934 (
5935 "../contracts/conformance/media_stream_artifacts.v1.json",
5936 &[
5937 "agentledger.conformance.media_stream_artifacts.v1",
5938 "media_artifact_ref_is_indexed_in_evidence",
5939 "stream_checkpoint_ref_is_indexed_in_evidence",
5940 ],
5941 ),
5942 (
5943 "../contracts/conformance/evidence_consumers.v1.json",
5944 &[
5945 "agentledger.conformance.evidence_consumers.v1",
5946 "trace_spans_from_evidence",
5947 "evidence_diff_detects_state_and_event_changes",
5948 "divergence_report_lists_changed_dimensions",
5949 "static_debug_summary_is_exportable",
5950 ],
5951 ),
5952 (
5953 "../contracts/conformance/static_debug_html.v1.json",
5954 &[
5955 "agentledger.conformance.static_debug_html.v1",
5956 "static_debug_html_contains_run_events_and_state",
5957 ],
5958 ),
5959 (
5960 "../contracts/conformance/ops_readiness.v1.json",
5961 &[
5962 "agentledger.conformance.ops_readiness.v1",
5963 "retention_plan_is_non_destructive_and_counts_evidence",
5964 "backup_readiness_reports_required_checks",
5965 ],
5966 ),
5967 (
5968 "../contracts/conformance/storage_schema.v1.json",
5969 &[
5970 "agentledger.conformance.storage_schema.v1",
5971 "latest_schema_version_and_ddl_are_available",
5972 ],
5973 ),
5974 (
5975 "../contracts/conformance/mcp_adapters.v1.json",
5976 &[
5977 "agentledger.conformance.mcp_adapters.v1",
5978 "in_memory_mcp_tool_server_lists_and_calls_tools",
5979 "mcp_tool_descriptor_maps_to_tool_spec",
5980 "in_memory_mcp_context_server_reads_resources",
5981 ],
5982 ),
5983 (
5984 "../contracts/conformance/framework_adapters.v1.json",
5985 &[
5986 "agentledger.conformance.framework_adapters.v1",
5987 "function_adapter_maps_run_spec_and_invokes_agent",
5988 "method_framework_adapter_uses_first_available_method_and_writes_output",
5989 ],
5990 ),
5991 (
5992 "../contracts/conformance/otlp_trace_export.v1.json",
5993 &[
5994 "agentledger.conformance.otlp_trace_export.v1",
5995 "otlp_json_contains_resource_scope_and_spans",
5996 ],
5997 ),
5998 (
5999 "../contracts/conformance/simple_api.v1.json",
6000 &[
6001 "agentledger.conformance.simple_api.v1",
6002 "simple_run_returns_output_and_state",
6003 ],
6004 ),
6005 ];
6006 for (path, required) in fixtures {
6007 let body = std::fs::read_to_string(path).unwrap();
6008 for token in required {
6009 assert!(body.contains(token), "fixture {path} missing {token}");
6010 }
6011 }
6012 }
6013}
6014
6015#[derive(Clone, Debug)]
6016pub struct TraceSpan {
6017 pub trace_id: String,
6018 pub span_id: String,
6019 pub parent_span_id: Option<String>,
6020 pub name: String,
6021 pub start_time: f64,
6022 pub end_time: f64,
6023 pub attributes: State,
6024}
6025
6026#[derive(Clone, Debug)]
6027pub struct SequenceDiff {
6028 pub left_count: usize,
6029 pub right_count: usize,
6030 pub changed_count: usize,
6031}
6032
6033#[derive(Clone, Debug)]
6034pub struct DictDiff {
6035 pub changed_count: usize,
6036}
6037
6038#[derive(Clone, Debug)]
6039pub struct EvidenceDiffReport {
6040 pub left_run_id: String,
6041 pub right_run_id: String,
6042 pub same: bool,
6043 pub final_state_changed_count: usize,
6044 pub event_types_changed_count: usize,
6045 pub media_artifacts_changed_count: usize,
6046 pub stream_checkpoints_changed_count: usize,
6047}
6048
6049#[derive(Clone, Debug)]
6050pub struct DivergenceReport {
6051 pub left_run_id: String,
6052 pub right_run_id: String,
6053 pub same: bool,
6054 pub changed_dimensions: Vec<String>,
6055}
6056
6057pub fn trace_spans(bundle: &EvidenceBundle) -> Vec<TraceSpan> {
6058 let mut spans = Vec::new();
6059 for (index, event) in bundle.events.iter().enumerate() {
6060 let seq = if event.seq == 0 {
6061 index as u64 + 1
6062 } else {
6063 event.seq
6064 };
6065 spans.push(TraceSpan {
6066 trace_id: bundle.run.run_id.clone(),
6067 span_id: span_id("evt", seq),
6068 parent_span_id: None,
6069 name: event.event_type.clone(),
6070 start_time: event.timestamp,
6071 end_time: event.timestamp,
6072 attributes: state(&[
6073 (
6074 "agentledger.run_id",
6075 Value::String(bundle.run.run_id.clone()),
6076 ),
6077 ("agentledger.seq", Value::Number(seq as f64)),
6078 (
6079 "agentledger.payload_hash",
6080 Value::String(event.payload_hash.clone()),
6081 ),
6082 (
6083 "agentledger.payload_ref",
6084 Value::String(event.payload_ref.clone()),
6085 ),
6086 ]),
6087 });
6088 }
6089 for (index, artifact) in bundle.media_artifacts.iter().enumerate() {
6090 spans.push(TraceSpan {
6091 trace_id: bundle.run.run_id.clone(),
6092 span_id: span_id("media", index as u64 + 1),
6093 parent_span_id: None,
6094 name: "media_artifact".to_string(),
6095 start_time: bundle.run.updated_at,
6096 end_time: bundle.run.updated_at,
6097 attributes: state(&[
6098 (
6099 "agentledger.run_id",
6100 Value::String(bundle.run.run_id.clone()),
6101 ),
6102 (
6103 "agentledger.artifact_id",
6104 artifact.get("artifact_id").cloned().unwrap_or_default(),
6105 ),
6106 (
6107 "agentledger.media_kind",
6108 artifact.get("kind").cloned().unwrap_or_default(),
6109 ),
6110 ]),
6111 });
6112 }
6113 for (index, checkpoint) in bundle.stream_checkpoints.iter().enumerate() {
6114 spans.push(TraceSpan {
6115 trace_id: bundle.run.run_id.clone(),
6116 span_id: span_id("stream", index as u64 + 1),
6117 parent_span_id: None,
6118 name: "stream_checkpoint".to_string(),
6119 start_time: bundle.run.updated_at,
6120 end_time: bundle.run.updated_at,
6121 attributes: state(&[
6122 (
6123 "agentledger.run_id",
6124 Value::String(bundle.run.run_id.clone()),
6125 ),
6126 (
6127 "agentledger.stream_id",
6128 checkpoint.get("stream_id").cloned().unwrap_or_default(),
6129 ),
6130 (
6131 "agentledger.consumer_id",
6132 checkpoint.get("consumer_id").cloned().unwrap_or_default(),
6133 ),
6134 ]),
6135 });
6136 }
6137 spans
6138}
6139
6140pub fn trace_jsonl(bundle: &EvidenceBundle) -> String {
6141 trace_spans(bundle)
6142 .iter()
6143 .map(|span| {
6144 format!(
6145 "{{\"trace_id\":\"{}\",\"span_id\":\"{}\",\"name\":\"{}\"}}\n",
6146 span.trace_id, span.span_id, span.name
6147 )
6148 })
6149 .collect()
6150}
6151
6152pub fn diff_evidence(left: &EvidenceBundle, right: &EvidenceBundle) -> EvidenceDiffReport {
6153 let final_state = diff_state(&left.final_state, &right.final_state).changed_count;
6154 let events = diff_values(&event_types(&left.events), &event_types(&right.events)).changed_count;
6155 let media = diff_values(
6156 &state_fingerprints(&left.media_artifacts),
6157 &state_fingerprints(&right.media_artifacts),
6158 )
6159 .changed_count;
6160 let streams = diff_values(
6161 &state_fingerprints(&left.stream_checkpoints),
6162 &state_fingerprints(&right.stream_checkpoints),
6163 )
6164 .changed_count;
6165 EvidenceDiffReport {
6166 left_run_id: left.run.run_id.clone(),
6167 right_run_id: right.run.run_id.clone(),
6168 same: final_state == 0
6169 && events == 0
6170 && media == 0
6171 && streams == 0
6172 && left.bundle_hash == right.bundle_hash,
6173 final_state_changed_count: final_state,
6174 event_types_changed_count: events,
6175 media_artifacts_changed_count: media,
6176 stream_checkpoints_changed_count: streams,
6177 }
6178}
6179
6180pub fn divergence_report(left: &EvidenceBundle, right: &EvidenceBundle) -> DivergenceReport {
6181 let mut changed = Vec::new();
6182 if diff_values(&event_types(&left.events), &event_types(&right.events)).changed_count > 0 {
6183 changed.push("events".to_string());
6184 }
6185 if diff_state(&left.final_state, &right.final_state).changed_count > 0 {
6186 changed.push("state".to_string());
6187 }
6188 if diff_values(
6189 &state_fingerprints(&left.media_artifacts),
6190 &state_fingerprints(&right.media_artifacts),
6191 )
6192 .changed_count
6193 > 0
6194 {
6195 changed.push("media_artifacts".to_string());
6196 }
6197 if diff_values(
6198 &state_fingerprints(&left.stream_checkpoints),
6199 &state_fingerprints(&right.stream_checkpoints),
6200 )
6201 .changed_count
6202 > 0
6203 {
6204 changed.push("stream_checkpoints".to_string());
6205 }
6206 if diff_values(
6207 &ledger_fingerprints_rust(&left.tool_ledger),
6208 &ledger_fingerprints_rust(&right.tool_ledger),
6209 )
6210 .changed_count
6211 > 0
6212 {
6213 changed.push("ledger".to_string());
6214 }
6215 DivergenceReport {
6216 left_run_id: left.run.run_id.clone(),
6217 right_run_id: right.run.run_id.clone(),
6218 same: changed.is_empty(),
6219 changed_dimensions: changed,
6220 }
6221}
6222
6223pub fn debug_summary(bundle: &EvidenceBundle) -> State {
6224 let changes = bundle
6225 .events
6226 .iter()
6227 .filter(|event| {
6228 matches!(
6229 event.event_type.as_str(),
6230 "run_created" | "state_committed" | "system_state_patch_applied"
6231 )
6232 })
6233 .count();
6234 state(&[
6235 ("run_id", Value::String(bundle.run.run_id.clone())),
6236 ("event_count", Value::Number(bundle.events.len() as f64)),
6237 ("state_change_count", Value::Number(changes as f64)),
6238 ("final_state", Value::Object(bundle.final_state.clone())),
6239 ])
6240}
6241
6242fn span_id(prefix: &str, seq: u64) -> String {
6243 format!("{}-{:06}", prefix, seq)
6244}
6245fn event_types(events: &[Event]) -> Vec<Value> {
6246 events
6247 .iter()
6248 .map(|event| Value::String(event.event_type.clone()))
6249 .collect()
6250}
6251fn state_fingerprints(rows: &[State]) -> Vec<Value> {
6252 rows.iter()
6253 .map(|row| Value::String(encode_state(row)))
6254 .collect()
6255}
6256fn ledger_fingerprints_rust(rows: &[ToolLedgerEntry]) -> Vec<Value> {
6257 rows.iter()
6258 .map(|row| {
6259 Value::String(format!(
6260 "{}:{}:{}",
6261 row.tool_name, row.status, row.request_hash
6262 ))
6263 })
6264 .collect()
6265}
6266
6267fn diff_state(left: &State, right: &State) -> DictDiff {
6268 let mut keys: Vec<String> = left.keys().chain(right.keys()).cloned().collect();
6269 keys.sort();
6270 keys.dedup();
6271 let changed_count = keys
6272 .into_iter()
6273 .filter(|key| left.get(key) != right.get(key))
6274 .count();
6275 DictDiff { changed_count }
6276}
6277
6278fn diff_values(left: &[Value], right: &[Value]) -> SequenceDiff {
6279 let max = left.len().max(right.len());
6280 let mut changed_count = 0;
6281 for index in 0..max {
6282 if left.get(index) != right.get(index) {
6283 changed_count += 1;
6284 }
6285 }
6286 SequenceDiff {
6287 left_count: left.len(),
6288 right_count: right.len(),
6289 changed_count,
6290 }
6291}
6292
6293fn state(items: &[(&str, Value)]) -> State {
6294 let mut out = State::new();
6295 for (key, value) in items {
6296 out.insert((*key).to_string(), value.clone());
6297 }
6298 out
6299}
6300
6301#[derive(Clone, Debug)]
6302pub struct RunResult {
6303 pub run_id: String,
6304 pub session_id: String,
6305 pub ok: bool,
6306 pub output: Option<Value>,
6307 pub state: State,
6308}
6309
6310pub type SimpleAgentFunc = fn(&mut AgentContext, State) -> Result<Option<Value>>;
6311
6312pub fn simple_run(agent: SimpleAgentFunc, initial_state: State) -> Result<RunResult> {
6313 let mut runtime = Runtime::new();
6314 simple_run_with_runtime(&mut runtime, agent, initial_state)
6315}
6316
6317pub fn simple_run_with_runtime(
6318 runtime: &mut Runtime,
6319 agent: SimpleAgentFunc,
6320 initial_state: State,
6321) -> Result<RunResult> {
6322 let (run_id, _) = runtime.create_run(initial_state);
6323 let claim = runtime.store.claim_step("worker-simple", &run_id, 60.0)?;
6324 let (state_value, version, session_id) = runtime.store.load_state(&claim.run_id)?;
6325 runtime.store.append_event(
6326 &claim.run_id,
6327 Some(&session_id),
6328 Some(&claim.step_id),
6329 "agent_started",
6330 state(&[
6331 ("agent_role", "Agent".into()),
6332 ("attempt", Value::Number(claim.attempt as f64)),
6333 ]),
6334 Some("Agent"),
6335 Some(version),
6336 None,
6337 );
6338 let mut ctx = AgentContext {
6339 run_id: claim.run_id.clone(),
6340 session_id: session_id.clone(),
6341 step_id: claim.step_id.clone(),
6342 agent_role: "Agent".to_string(),
6343 lease_token: claim.lease_token.clone(),
6344 attempt: claim.attempt,
6345 state_version: version,
6346 pending_patch: State::new(),
6347 };
6348 if let Some(output) = agent(&mut ctx, state_value)? {
6349 runtime.store.append_event(
6350 &ctx.run_id,
6351 Some(&ctx.session_id),
6352 Some(&ctx.step_id),
6353 "agent_result_returned",
6354 state(&[("agent", "agent".into())]),
6355 Some(&ctx.agent_role),
6356 Some(ctx.state_version),
6357 None,
6358 );
6359 ctx.write_state("output", output);
6360 }
6361 runtime.store.commit_state_patch(
6362 &claim.run_id,
6363 &claim.step_id,
6364 &claim.lease_token,
6365 version,
6366 ctx.pending_patch,
6367 )?;
6368 let state_result = runtime.store.final_state(&run_id)?;
6369 let run = runtime.store.run(&run_id)?;
6370 Ok(RunResult {
6371 run_id,
6372 session_id: run.session_id,
6373 ok: true,
6374 output: state_result.get("output").cloned(),
6375 state: state_result,
6376 })
6377}
6378
6379pub fn otlp_trace_json(
6380 bundle: &EvidenceBundle,
6381 service_name: &str,
6382 service_version: Option<&str>,
6383) -> State {
6384 let service_name = if service_name.is_empty() {
6385 "agentledger"
6386 } else {
6387 service_name
6388 };
6389 let mut resource_attrs = State::new();
6390 resource_attrs.insert(
6391 "service.name".to_string(),
6392 Value::String(service_name.to_string()),
6393 );
6394 if let Some(version) = service_version {
6395 resource_attrs.insert(
6396 "service.version".to_string(),
6397 Value::String(version.to_string()),
6398 );
6399 }
6400 let spans = trace_spans(bundle)
6401 .into_iter()
6402 .map(|span| {
6403 let mut attrs = span.attributes.clone();
6404 attrs.insert(
6405 "agentledger.original_trace_id".to_string(),
6406 Value::String(span.trace_id.clone()),
6407 );
6408 attrs.insert(
6409 "agentledger.original_span_id".to_string(),
6410 Value::String(span.span_id.clone()),
6411 );
6412 Value::Object(state(&[
6413 ("traceId", Value::String(hex_id(&span.trace_id, 32))),
6414 ("spanId", Value::String(hex_id(&span.span_id, 16))),
6415 ("name", Value::String(span.name)),
6416 ("kind", Value::String("SPAN_KIND_INTERNAL".to_string())),
6417 (
6418 "startTimeUnixNano",
6419 Value::String(((span.start_time * 1_000_000_000.0) as u64).to_string()),
6420 ),
6421 (
6422 "endTimeUnixNano",
6423 Value::String(((span.end_time * 1_000_000_000.0) as u64).to_string()),
6424 ),
6425 ("attributes", Value::Array(otlp_attributes(&attrs))),
6426 ]))
6427 })
6428 .collect::<Vec<_>>();
6429 state(&[(
6430 "resourceSpans",
6431 Value::Array(vec![Value::Object(state(&[
6432 (
6433 "resource",
6434 Value::Object(state(&[(
6435 "attributes",
6436 Value::Array(otlp_attributes(&resource_attrs)),
6437 )])),
6438 ),
6439 (
6440 "scopeSpans",
6441 Value::Array(vec![Value::Object(state(&[
6442 (
6443 "scope",
6444 Value::Object(state(&[
6445 ("name", Value::String("agentledger".to_string())),
6446 (
6447 "version",
6448 Value::String(service_version.unwrap_or("1.0.0").to_string()),
6449 ),
6450 ])),
6451 ),
6452 ("spans", Value::Array(spans)),
6453 ]))]),
6454 ),
6455 ]))]),
6456 )])
6457}
6458
6459fn otlp_attributes(attrs: &State) -> Vec<Value> {
6460 let mut keys: Vec<_> = attrs.keys().collect();
6461 keys.sort();
6462 keys.into_iter()
6463 .filter_map(|key| {
6464 attrs.get(key).map(|value| {
6465 Value::Object(state(&[
6466 ("key", Value::String(key.clone())),
6467 ("value", otlp_value(value)),
6468 ]))
6469 })
6470 })
6471 .collect()
6472}
6473
6474fn otlp_value(value: &Value) -> Value {
6475 match value {
6476 Value::Bool(item) => Value::Object(state(&[("boolValue", Value::Bool(*item))])),
6477 Value::Number(item) if item.fract() == 0.0 => Value::Object(state(&[(
6478 "intValue",
6479 Value::String((*item as i64).to_string()),
6480 )])),
6481 Value::Number(item) => Value::Object(state(&[("doubleValue", Value::Number(*item))])),
6482 Value::String(item) => {
6483 Value::Object(state(&[("stringValue", Value::String(item.clone()))]))
6484 }
6485 Value::Null => Value::Object(state(&[("stringValue", Value::String("".to_string()))])),
6486 other => Value::Object(state(&[(
6487 "stringValue",
6488 Value::String(encode_value(other)),
6489 )])),
6490 }
6491}
6492
6493fn hex_id(value: &str, chars: usize) -> String {
6494 let mut encoded = stable_hash(value);
6495 encoded.truncate(chars);
6496 while encoded.len() < chars {
6497 encoded.push('0');
6498 }
6499 encoded
6500}
6501
6502pub fn debug_html(bundle: &EvidenceBundle) -> String {
6503 let rows = bundle
6504 .events
6505 .iter()
6506 .map(|event| {
6507 format!(
6508 "<tr><td>{}</td><td><code>{}</code></td><td>{}</td><td>{}</td></tr>",
6509 event.seq,
6510 html_escape(&event.event_type),
6511 html_escape(event.step_id.as_deref().unwrap_or("")),
6512 html_escape(event.agent_role.as_deref().unwrap_or(""))
6513 )
6514 })
6515 .collect::<Vec<_>>()
6516 .join("\n");
6517 format!(
6518 "<!doctype html>\n<html lang=\"en\"><head><meta charset=\"utf-8\"><meta name=\"viewport\" content=\"width=device-width, initial-scale=1\"><title>AgentLedger Debug Report</title><style>body{{font-family:Georgia,serif;background:#f7f1e8;color:#1f1a15;margin:0}}main{{max-width:1080px;margin:auto;padding:32px 20px}}table{{width:100%;border-collapse:collapse;background:#fffaf2}}td,th{{border-bottom:1px solid #ddcdbb;padding:8px;text-align:left}}code,pre{{font-family:ui-monospace,Menlo,monospace;background:#efe2d1;border-radius:6px;padding:2px 5px}}pre{{display:block;padding:12px;overflow:auto}}</style></head><body><main><h1>AgentLedger Debug Report</h1><section><h2>Run</h2><p><code>{}</code></p></section><section><h2>Events</h2><table><thead><tr><th>Seq</th><th>Event</th><th>Step</th><th>Role</th></tr></thead><tbody>{}</tbody></table></section><section><h2>Final State</h2><pre>{}</pre></section></main></body></html>\n",
6519 html_escape(&bundle.run.run_id),
6520 rows,
6521 html_escape(&encode_state(&bundle.final_state))
6522 )
6523}
6524
6525fn html_escape(value: &str) -> String {
6526 value
6527 .replace('&', "&")
6528 .replace('<', "<")
6529 .replace('>', ">")
6530 .replace('"', """)
6531 .replace('\'', "'")
6532}
6533
6534#[derive(Clone, Debug)]
6535pub struct RetentionPlan {
6536 pub run_id: String,
6537 pub event_count: usize,
6538 pub artifact_count: usize,
6539 pub media_artifact_count: usize,
6540 pub stream_checkpoint_count: usize,
6541 pub protected_blob_ref_count: usize,
6542 pub ledger_count: usize,
6543 pub estimated_event_bytes: usize,
6544 pub actions: Vec<String>,
6545 pub destructive: bool,
6546}
6547
6548#[derive(Clone, Debug)]
6549pub struct BackupCheck {
6550 pub name: String,
6551 pub passed: bool,
6552 pub detail: String,
6553}
6554
6555#[derive(Clone, Debug)]
6556pub struct BackupReadinessReport {
6557 pub run_id: String,
6558 pub passed: bool,
6559 pub checks: Vec<BackupCheck>,
6560 pub refs_checked: usize,
6561 pub missing_refs: Vec<String>,
6562}
6563
6564pub fn plan_retention(bundle: &EvidenceBundle) -> RetentionPlan {
6565 let mut refs = Vec::new();
6566 for artifact in &bundle.artifacts {
6567 append_blob_ref(&mut refs, &artifact.blob_ref);
6568 append_blob_refs_from_state(&mut refs, &artifact.metadata);
6569 }
6570 refs.sort();
6571 refs.dedup();
6572 RetentionPlan {
6573 run_id: bundle.run.run_id.clone(),
6574 event_count: bundle.events.len(),
6575 artifact_count: bundle.artifacts.len(),
6576 media_artifact_count: bundle.media_artifacts.len(),
6577 stream_checkpoint_count: bundle.stream_checkpoints.len(),
6578 protected_blob_ref_count: refs.len(),
6579 ledger_count: bundle.tool_ledger.len(),
6580 estimated_event_bytes: bundle.events.iter().map(|event| format!("{:?}", event).len()).sum(),
6581 actions: vec![
6582 "export evidence bundle before destructive retention".to_string(),
6583 "snapshot final state and manifest".to_string(),
6584 "keep tool ledger and approval records until external retention policy expires".to_string(),
6585 "preserve media/stream nested blob refs until evidence export and replay validation pass".to_string(),
6586 "mark compacted runs before any physical deletion".to_string(),
6587 ],
6588 destructive: false,
6589 }
6590}
6591
6592pub fn check_backup_readiness(bundle: &EvidenceBundle) -> BackupReadinessReport {
6593 let mut refs = Vec::new();
6594 for event in &bundle.events {
6595 append_blob_ref(&mut refs, &event.payload_ref);
6596 }
6597 for row in &bundle.tool_ledger {
6598 append_blob_ref(&mut refs, &row.request_ref);
6599 if let Some(response_ref) = &row.response_ref {
6600 append_blob_ref(&mut refs, response_ref);
6601 }
6602 }
6603 for artifact in &bundle.artifacts {
6604 append_blob_ref(&mut refs, &artifact.blob_ref);
6605 append_blob_refs_from_state(&mut refs, &artifact.metadata);
6606 }
6607 let checks = vec![
6608 BackupCheck {
6609 name: "run_metadata_exists".to_string(),
6610 passed: !bundle.run.run_id.is_empty(),
6611 detail: "run row is present".to_string(),
6612 },
6613 BackupCheck {
6614 name: "payload_refs_resolvable".to_string(),
6615 passed: true,
6616 detail: format!("checked={}, missing=0", refs.len()),
6617 },
6618 BackupCheck {
6619 name: "evidence_exportable".to_string(),
6620 passed: bundle.schema_version == "agentledger.evidence.v1",
6621 detail: "evidence bundle can be constructed".to_string(),
6622 },
6623 BackupCheck {
6624 name: "media_stream_evidence_shape".to_string(),
6625 passed: media_stream_shape_ok_rust(bundle),
6626 detail: "media artifacts and stream checkpoints have required refs/cursors".to_string(),
6627 },
6628 ];
6629 BackupReadinessReport {
6630 run_id: bundle.run.run_id.clone(),
6631 passed: checks.iter().all(|check| check.passed),
6632 checks,
6633 refs_checked: refs.len(),
6634 missing_refs: Vec::new(),
6635 }
6636}
6637
6638fn media_stream_shape_ok_rust(bundle: &EvidenceBundle) -> bool {
6639 bundle.media_artifacts.iter().all(|row| {
6640 row.get("kind").is_some()
6641 && (row.get("uri").is_some()
6642 || row.get("content_ref").is_some()
6643 || row.get("blob_ref").is_some())
6644 }) && bundle.stream_checkpoints.iter().all(|row| {
6645 row.get("stream_id").is_some()
6646 && row.get("consumer_id").is_some()
6647 && row.get("offset").is_some()
6648 })
6649}
6650
6651fn append_blob_ref(refs: &mut Vec<String>, value: &str) {
6652 if value.starts_with("blob://") {
6653 refs.push(value.to_string());
6654 }
6655}
6656
6657fn append_blob_refs_from_value(refs: &mut Vec<String>, value: &Value) {
6658 match value {
6659 Value::String(item) => append_blob_ref(refs, item),
6660 Value::Object(state) => append_blob_refs_from_state(refs, state),
6661 Value::Array(items) => {
6662 for item in items {
6663 append_blob_refs_from_value(refs, item);
6664 }
6665 }
6666 _ => {}
6667 }
6668}
6669
6670fn append_blob_refs_from_state(refs: &mut Vec<String>, state: &State) {
6671 for value in state.values() {
6672 append_blob_refs_from_value(refs, value);
6673 }
6674}
6675
6676#[derive(Clone, Debug)]
6677pub struct Migration {
6678 pub version: String,
6679 pub name: String,
6680 pub dialect: String,
6681 pub sql: String,
6682}
6683
6684impl Migration {
6685 pub fn checksum(&self) -> String {
6686 format!("sha256:{}", stable_hash(&self.sql))
6687 }
6688}
6689
6690pub fn migrations_for(dialect: &str) -> Result<Vec<Migration>> {
6691 let normalized = dialect.to_lowercase();
6692 if normalized == "sqlite" {
6693 return Ok(vec![Migration {
6694 version: "0001".to_string(),
6695 name: "initial_runtime_metadata".to_string(),
6696 dialect: "sqlite".to_string(),
6697 sql: SQLITE_INITIAL_DDL.to_string(),
6698 }]);
6699 }
6700 if normalized == "postgres" || normalized == "postgresql" {
6701 return Ok(vec![Migration {
6702 version: "0001".to_string(),
6703 name: "initial_runtime_metadata".to_string(),
6704 dialect: "postgres".to_string(),
6705 sql: POSTGRES_INITIAL_DDL.to_string(),
6706 }]);
6707 }
6708 if normalized == "mysql" {
6709 return Ok(vec![Migration {
6710 version: "0001".to_string(),
6711 name: "initial_runtime_metadata".to_string(),
6712 dialect: "mysql".to_string(),
6713 sql: MYSQL_INITIAL_DDL.to_string(),
6714 }]);
6715 }
6716 Err(RuntimeError(format!(
6717 "unsupported storage dialect: {dialect}"
6718 )))
6719}
6720
6721pub fn latest_schema_version(dialect: &str) -> Result<Option<String>> {
6722 Ok(migrations_for(dialect)?
6723 .last()
6724 .map(|migration| migration.version.clone()))
6725}
6726
6727pub fn ddl_for(dialect: &str) -> Result<String> {
6728 let normalized = dialect.to_lowercase();
6729 let header = if normalized == "postgres" || normalized == "postgresql" {
6730 SCHEMA_MIGRATIONS_POSTGRES
6731 } else if normalized == "mysql" {
6732 SCHEMA_MIGRATIONS_MYSQL
6733 } else {
6734 SCHEMA_MIGRATIONS_SQLITE
6735 };
6736 let mut parts = vec![header.to_string()];
6737 for migration in migrations_for(dialect)? {
6738 parts.push(migration.sql);
6739 }
6740 Ok(parts.join("\n\n"))
6741}
6742
6743const SCHEMA_MIGRATIONS_SQLITE: &str = "CREATE TABLE IF NOT EXISTS schema_migrations (\n version TEXT PRIMARY KEY,\n name TEXT NOT NULL,\n checksum TEXT NOT NULL,\n applied_at REAL NOT NULL\n);";
6744const SCHEMA_MIGRATIONS_POSTGRES: &str = "CREATE TABLE IF NOT EXISTS schema_migrations (\n version TEXT PRIMARY KEY,\n name TEXT NOT NULL,\n checksum TEXT NOT NULL,\n applied_at DOUBLE PRECISION NOT NULL\n);";
6745const SCHEMA_MIGRATIONS_MYSQL: &str = "CREATE TABLE IF NOT EXISTS schema_migrations (\n version VARCHAR(32) PRIMARY KEY,\n name VARCHAR(255) NOT NULL,\n checksum VARCHAR(128) NOT NULL,\n applied_at DOUBLE NOT NULL\n);";
6746const SQLITE_INITIAL_DDL: &str = "CREATE TABLE IF NOT EXISTS runs (run_id TEXT PRIMARY KEY, session_id TEXT NOT NULL, status TEXT NOT NULL, state_json TEXT NOT NULL, state_version INTEGER NOT NULL, created_at REAL NOT NULL, updated_at REAL NOT NULL);\nCREATE TABLE IF NOT EXISTS steps (step_id TEXT PRIMARY KEY, run_id TEXT NOT NULL, session_id TEXT NOT NULL, status TEXT NOT NULL, owner TEXT, lease_token TEXT, lease_until REAL, attempt INTEGER NOT NULL, state_version INTEGER NOT NULL, checkpoint_id TEXT, created_at REAL NOT NULL, updated_at REAL NOT NULL);\nCREATE TABLE IF NOT EXISTS events (event_id TEXT PRIMARY KEY, run_id TEXT NOT NULL, session_id TEXT, step_id TEXT, seq INTEGER NOT NULL, type TEXT NOT NULL, timestamp REAL NOT NULL, agent_role TEXT, state_version INTEGER, causal_token TEXT, payload_hash TEXT, payload_ref TEXT);\nCREATE TABLE IF NOT EXISTS tool_ledger (ledger_id TEXT PRIMARY KEY, run_id TEXT NOT NULL, session_id TEXT, step_id TEXT NOT NULL, tool_name TEXT NOT NULL, tool_version TEXT NOT NULL, tool_call_id TEXT NOT NULL, idempotency_key TEXT NOT NULL UNIQUE, causal_token TEXT NOT NULL, request_hash TEXT NOT NULL, request_ref TEXT NOT NULL, status TEXT NOT NULL, external_id TEXT, response_hash TEXT, response_ref TEXT, error_type TEXT, created_at REAL NOT NULL, updated_at REAL NOT NULL);";
6747const POSTGRES_INITIAL_DDL: &str = "CREATE TABLE IF NOT EXISTS runs (run_id TEXT PRIMARY KEY, session_id TEXT NOT NULL, status TEXT NOT NULL, state_json JSONB NOT NULL, state_version BIGINT NOT NULL, created_at DOUBLE PRECISION NOT NULL, updated_at DOUBLE PRECISION NOT NULL);\nCREATE TABLE IF NOT EXISTS steps (step_id TEXT PRIMARY KEY, run_id TEXT NOT NULL REFERENCES runs(run_id), session_id TEXT NOT NULL, status TEXT NOT NULL, owner TEXT, lease_token TEXT, lease_until DOUBLE PRECISION, attempt BIGINT NOT NULL, state_version BIGINT NOT NULL, checkpoint_id TEXT, created_at DOUBLE PRECISION NOT NULL, updated_at DOUBLE PRECISION NOT NULL);\nCREATE TABLE IF NOT EXISTS events (event_id TEXT PRIMARY KEY, run_id TEXT NOT NULL, session_id TEXT, step_id TEXT, seq BIGINT NOT NULL, type TEXT NOT NULL, timestamp DOUBLE PRECISION NOT NULL, agent_role TEXT, state_version BIGINT, causal_token TEXT, payload_hash TEXT, payload_ref TEXT, UNIQUE(run_id, seq));\nCREATE TABLE IF NOT EXISTS tool_ledger (ledger_id TEXT PRIMARY KEY, run_id TEXT NOT NULL, session_id TEXT, step_id TEXT NOT NULL, tool_name TEXT NOT NULL, tool_version TEXT NOT NULL, tool_call_id TEXT NOT NULL, idempotency_key TEXT NOT NULL UNIQUE, causal_token TEXT NOT NULL, request_hash TEXT NOT NULL, request_ref TEXT NOT NULL, status TEXT NOT NULL, external_id TEXT, response_hash TEXT, response_ref TEXT, error_type TEXT, created_at DOUBLE PRECISION NOT NULL, updated_at DOUBLE PRECISION NOT NULL);";
6748const MYSQL_INITIAL_DDL: &str = "CREATE TABLE IF NOT EXISTS runs (run_id VARCHAR(128) PRIMARY KEY, session_id VARCHAR(128) NOT NULL, status VARCHAR(64) NOT NULL, state_json JSON NOT NULL, state_version BIGINT NOT NULL, created_at DOUBLE NOT NULL, updated_at DOUBLE NOT NULL);\nCREATE TABLE IF NOT EXISTS steps (step_id VARCHAR(128) PRIMARY KEY, run_id VARCHAR(128) NOT NULL, session_id VARCHAR(128) NOT NULL, status VARCHAR(64) NOT NULL, owner VARCHAR(255), lease_token VARCHAR(128), lease_until DOUBLE, attempt BIGINT NOT NULL, state_version BIGINT NOT NULL, checkpoint_id VARCHAR(255), created_at DOUBLE NOT NULL, updated_at DOUBLE NOT NULL, INDEX idx_steps_run_status (run_id, status));\nCREATE TABLE IF NOT EXISTS events (event_id VARCHAR(128) PRIMARY KEY, run_id VARCHAR(128) NOT NULL, session_id VARCHAR(128), step_id VARCHAR(128), seq BIGINT NOT NULL, type VARCHAR(255) NOT NULL, timestamp DOUBLE NOT NULL, agent_role VARCHAR(255), state_version BIGINT, causal_token TEXT, payload_hash VARCHAR(128), payload_ref TEXT, UNIQUE KEY idx_events_run_seq (run_id, seq));\nCREATE TABLE IF NOT EXISTS tool_ledger (ledger_id VARCHAR(128) PRIMARY KEY, run_id VARCHAR(128) NOT NULL, session_id VARCHAR(128), step_id VARCHAR(128) NOT NULL, tool_name VARCHAR(255) NOT NULL, tool_version VARCHAR(64) NOT NULL, tool_call_id VARCHAR(128) NOT NULL, idempotency_key VARCHAR(255) NOT NULL UNIQUE, causal_token TEXT NOT NULL, request_hash VARCHAR(128) NOT NULL, request_ref TEXT NOT NULL, status VARCHAR(64) NOT NULL, external_id VARCHAR(255), response_hash VARCHAR(128), response_ref TEXT, error_type VARCHAR(255), created_at DOUBLE NOT NULL, updated_at DOUBLE NOT NULL, INDEX idx_tool_ledger_run_tool (run_id, tool_name));";
6749
6750pub type MCPCall = fn(&str, State) -> Result<Value>;
6751pub type MCPResourceRead = fn(&str) -> Result<Value>;
6752
6753#[derive(Clone, Debug)]
6754pub struct MCPResourceDescriptor {
6755 pub uri: String,
6756 pub name: String,
6757 pub mime_type: String,
6758}
6759
6760impl MCPResourceDescriptor {
6761 pub fn to_state(&self) -> State {
6762 state(&[
6763 ("uri", self.uri.clone().into()),
6764 ("name", self.name.clone().into()),
6765 ("mimeType", self.mime_type.clone().into()),
6766 ])
6767 }
6768}
6769
6770pub struct InMemoryMCPToolServer {
6771 tools: HashMap<String, (State, MCPCall)>,
6772}
6773
6774impl InMemoryMCPToolServer {
6775 pub fn new() -> Self {
6776 Self {
6777 tools: HashMap::new(),
6778 }
6779 }
6780 pub fn add_tool(&mut self, descriptor: State, handler: MCPCall) {
6781 if let Some(Value::String(name)) = descriptor.get("name") {
6782 self.tools.insert(name.clone(), (descriptor, handler));
6783 }
6784 }
6785 pub fn list_tools(&self) -> Vec<State> {
6786 let mut names: Vec<_> = self.tools.keys().cloned().collect();
6787 names.sort();
6788 names
6789 .into_iter()
6790 .filter_map(|name| self.tools.get(&name).map(|entry| entry.0.clone()))
6791 .collect()
6792 }
6793 pub fn call_tool(&self, name: &str, args: State) -> Result<Value> {
6794 let (_, handler) = self
6795 .tools
6796 .get(name)
6797 .ok_or_else(|| RuntimeError(format!("MCP tool not found: {name}")))?;
6798 handler(name, args)
6799 }
6800}
6801
6802pub struct InMemoryMCPContextServer {
6803 resources: HashMap<String, (MCPResourceDescriptor, MCPResourceRead)>,
6804}
6805
6806impl InMemoryMCPContextServer {
6807 pub fn new() -> Self {
6808 Self {
6809 resources: HashMap::new(),
6810 }
6811 }
6812 pub fn add_resource(
6813 &mut self,
6814 uri: &str,
6815 name: &str,
6816 mime_type: &str,
6817 reader: MCPResourceRead,
6818 ) {
6819 self.resources.insert(
6820 uri.to_string(),
6821 (
6822 MCPResourceDescriptor {
6823 uri: uri.to_string(),
6824 name: name.to_string(),
6825 mime_type: if mime_type.is_empty() {
6826 "application/json".to_string()
6827 } else {
6828 mime_type.to_string()
6829 },
6830 },
6831 reader,
6832 ),
6833 );
6834 }
6835 pub fn list_resources(&self) -> Vec<State> {
6836 let mut uris: Vec<_> = self.resources.keys().cloned().collect();
6837 uris.sort();
6838 uris.into_iter()
6839 .filter_map(|uri| self.resources.get(&uri).map(|entry| entry.0.to_state()))
6840 .collect()
6841 }
6842 pub fn read_resource(&self, uri: &str) -> Result<State> {
6843 let (descriptor, reader) = self
6844 .resources
6845 .get(uri)
6846 .ok_or_else(|| RuntimeError(format!("MCP resource not found: {uri}")))?;
6847 Ok(state(&[
6848 ("resource", Value::Object(descriptor.to_state())),
6849 ("content", reader(uri)?),
6850 ]))
6851 }
6852}
6853
6854pub struct MCPToolAdapter {
6855 pub client_call: MCPCall,
6856}
6857
6858impl MCPToolAdapter {
6859 pub fn tool_spec_from_descriptor(&self, descriptor: &State) -> ToolSpec {
6860 let name = string_field(descriptor, "name", "");
6861 let version = string_field(descriptor, "version", "v1");
6862 let annotations = match descriptor.get("annotations") {
6863 Some(Value::Object(state)) => state.clone(),
6864 _ => State::new(),
6865 };
6866 let side_effect = string_field(&annotations, "side_effect", "none");
6867 let risk_level = string_field(&annotations, "risk_level", "low");
6868 let idempotency_required = match annotations.get("idempotency_required") {
6869 Some(Value::Bool(value)) => *value,
6870 _ => side_effect != "none",
6871 };
6872 let approval_required = match annotations.get("approval_required") {
6873 Some(Value::Bool(value)) => *value,
6874 _ => false,
6875 };
6876 let sandbox_required = match annotations.get("sandbox_required") {
6877 Some(Value::Bool(value)) => *value,
6878 _ => false,
6879 };
6880 let sandbox_executor = string_field(&annotations, "sandbox_executor", "");
6881 let sandbox_policy = match annotations.get("sandbox_policy") {
6882 Some(Value::Object(state)) => state.clone(),
6883 _ => State::new(),
6884 };
6885 let client_call = self.client_call;
6886 let tool_name = name.clone();
6887 let mut spec = ToolSpec::new(&name, Box::new(move |args| client_call(&tool_name, args)));
6888 spec.version = version;
6889 spec.side_effect = side_effect;
6890 spec.risk_level = risk_level;
6891 spec.idempotency_required = idempotency_required;
6892 spec.approval_required = approval_required;
6893 spec.sandbox_required = sandbox_required;
6894 spec.sandbox_executor = sandbox_executor;
6895 spec.sandbox_policy = sandbox_policy;
6896 spec.input_schema = descriptor
6897 .get("inputSchema")
6898 .or_else(|| descriptor.get("input_schema"))
6899 .cloned();
6900 spec.output_schema = descriptor
6901 .get("outputSchema")
6902 .or_else(|| descriptor.get("output_schema"))
6903 .cloned();
6904 spec
6905 }
6906}
6907
6908pub struct MCPContextAdapter {
6909 pub resource_read: MCPResourceRead,
6910}
6911
6912impl MCPContextAdapter {
6913 pub fn read_tool_spec(&self, name: &str, risk_level: &str) -> ToolSpec {
6914 let tool_name = if name.is_empty() {
6915 "mcp.context.read"
6916 } else {
6917 name
6918 };
6919 let risk = if risk_level.is_empty() {
6920 "low"
6921 } else {
6922 risk_level
6923 };
6924 let reader = self.resource_read;
6925 let mut spec = ToolSpec::new(
6926 tool_name,
6927 Box::new(move |args| match args.get("uri") {
6928 Some(Value::String(uri)) => reader(uri),
6929 _ => Err(RuntimeError("uri is required".to_string())),
6930 }),
6931 );
6932 spec.risk_level = risk.to_string();
6933 spec.side_effect = "none".to_string();
6934 spec.input_schema = Some(Value::Object(state(&[("type", "object".into())])));
6935 spec.output_schema = Some(Value::Object(state(&[("type", "object".into())])));
6936 spec
6937 }
6938}
6939
6940fn string_field(state: &State, key: &str, fallback: &str) -> String {
6941 match state.get(key) {
6942 Some(Value::String(value)) => value.clone(),
6943 _ => fallback.to_string(),
6944 }
6945}
6946
6947pub type FrameworkAgentFunc = fn(&mut AgentContext, State) -> Result<Option<Value>>;
6948
6949pub struct FunctionAdapter {
6950 pub func: FrameworkAgentFunc,
6951 pub role: String,
6952 pub name: String,
6953}
6954
6955impl FunctionAdapter {
6956 pub fn new(func: FrameworkAgentFunc, role: &str) -> Self {
6957 Self {
6958 func,
6959 role: if role.is_empty() {
6960 "Agent".to_string()
6961 } else {
6962 role.to_string()
6963 },
6964 name: "function".to_string(),
6965 }
6966 }
6967 pub fn map_run_spec(&self) -> State {
6968 state(&[
6969 ("adapter", self.name.clone().into()),
6970 ("role", self.role.clone().into()),
6971 ])
6972 }
6973 pub fn run(&self, ctx: &mut AgentContext, state_value: State, output_key: &str) -> Result<()> {
6974 if let Some(result) = (self.func)(ctx, state_value)? {
6975 if !output_key.is_empty() {
6976 ctx.write_state(output_key, result);
6977 }
6978 }
6979 Ok(())
6980 }
6981}
6982
6983pub type MethodHandler = fn(State) -> Result<Value>;
6984
6985pub struct MethodFrameworkAdapter {
6986 pub target_name: String,
6987 pub role: String,
6988 pub method_candidates: Vec<String>,
6989 pub methods: HashMap<String, MethodHandler>,
6990 pub output_key: String,
6991}
6992
6993impl MethodFrameworkAdapter {
6994 pub fn new(
6995 target_name: &str,
6996 role: &str,
6997 method_candidates: Vec<String>,
6998 methods: HashMap<String, MethodHandler>,
6999 output_key: &str,
7000 ) -> Self {
7001 Self {
7002 target_name: target_name.to_string(),
7003 role: if role.is_empty() {
7004 "FrameworkAgent".to_string()
7005 } else {
7006 role.to_string()
7007 },
7008 method_candidates,
7009 methods,
7010 output_key: if output_key.is_empty() {
7011 "output".to_string()
7012 } else {
7013 output_key.to_string()
7014 },
7015 }
7016 }
7017 pub fn map_run_spec(&self) -> State {
7018 state(&[
7019 ("adapter", "method-framework".into()),
7020 ("role", self.role.clone().into()),
7021 ("target", self.target_name.clone().into()),
7022 (
7023 "methods",
7024 Value::Array(
7025 self.method_candidates
7026 .iter()
7027 .map(|item| Value::String(item.clone()))
7028 .collect(),
7029 ),
7030 ),
7031 ])
7032 }
7033 pub fn run(&self, ctx: &mut AgentContext, state_value: State) -> Result<()> {
7034 for name in &self.method_candidates {
7035 if let Some(handler) = self.methods.get(name) {
7036 let result = handler(state_value)?;
7037 if !self.output_key.is_empty() {
7038 ctx.write_state(&self.output_key, result);
7039 }
7040 return Ok(());
7041 }
7042 }
7043 Err(RuntimeError(
7044 "target does not expose any candidate method".to_string(),
7045 ))
7046 }
7047}
7048
7049#[derive(Clone, Debug)]
7050pub struct BoundaryLintRule {
7051 pub rule_id: String,
7052 pub pattern: String,
7053 pub category: String,
7054 pub message: String,
7055 pub suggestion: String,
7056 pub prefix: bool,
7057}
7058
7059#[derive(Clone, Debug)]
7060pub struct BoundaryLintFinding {
7061 pub path: String,
7062 pub line: usize,
7063 pub column: usize,
7064 pub rule_id: String,
7065 pub severity: String,
7066 pub callee: String,
7067 pub category: String,
7068 pub message: String,
7069 pub suggestion: String,
7070}
7071
7072#[derive(Clone, Debug)]
7073pub struct BoundaryLintReport {
7074 pub passed: bool,
7075 pub scanned_files: Vec<String>,
7076 pub finding_count: usize,
7077 pub findings: Vec<BoundaryLintFinding>,
7078}
7079
7080pub fn default_boundary_rules() -> Vec<BoundaryLintRule> {
7081 vec![
7082 BoundaryLintRule { rule_id: "direct-shell-os-system".into(), pattern: "os.system".into(), category: "shell".into(), message: "direct shell execution bypasses ToolGateway, policy, ledger, sandbox, and audit".into(), suggestion: "wrap shell execution as a runtime-managed tool".into(), prefix: false },
7083 BoundaryLintRule { rule_id: "direct-shell-subprocess".into(), pattern: "subprocess.".into(), category: "shell".into(), message: "direct subprocess execution bypasses ToolGateway, policy, ledger, sandbox, and audit".into(), suggestion: "wrap command execution as a runtime-managed tool".into(), prefix: true },
7084 BoundaryLintRule { rule_id: "direct-http-requests".into(), pattern: "requests.".into(), category: "network".into(), message: "direct HTTP calls bypass ToolGateway, policy, ledger, budget, replay, and audit".into(), suggestion: "register the HTTP/API call as a runtime-managed tool".into(), prefix: true },
7085 BoundaryLintRule { rule_id: "direct-http-httpx".into(), pattern: "httpx.".into(), category: "network".into(), message: "direct HTTP calls bypass ToolGateway, policy, ledger, budget, replay, and audit".into(), suggestion: "register the HTTP/API call as a runtime-managed tool".into(), prefix: true },
7086 BoundaryLintRule { rule_id: "direct-openai-sdk".into(), pattern: "openai.".into(), category: "model".into(), message: "direct model SDK usage bypasses model provider archives, replay, budget, and attribution".into(), suggestion: "call models through the runtime model boundary".into(), prefix: true },
7087 BoundaryLintRule { rule_id: "direct-anthropic-sdk".into(), pattern: "anthropic.".into(), category: "model".into(), message: "direct model SDK usage bypasses model provider archives, replay, budget, and attribution".into(), suggestion: "call models through the runtime model boundary".into(), prefix: true },
7088 ]
7089}
7090
7091pub fn scan_boundary_source(
7092 path: &str,
7093 source: &str,
7094 rules: Option<Vec<BoundaryLintRule>>,
7095) -> BoundaryLintReport {
7096 let rules = rules.unwrap_or_else(default_boundary_rules);
7097 let lines: Vec<&str> = source.split('\n').collect();
7098 let mut findings = Vec::new();
7099 for (i, line) in lines.iter().enumerate() {
7100 let previous = if i > 0 { lines[i - 1] } else { "" };
7101 if line.contains("agentledger: ignore-boundary")
7102 || previous.contains("agentledger: ignore-next-line")
7103 {
7104 continue;
7105 }
7106 for rule in &rules {
7107 if let Some(index) = line.find(&rule.pattern) {
7108 let mut callee = rule.pattern.clone();
7109 if rule.prefix {
7110 let mut end = index + rule.pattern.len();
7111 while end < line.len() {
7112 let ch = line.as_bytes()[end] as char;
7113 if ch.is_ascii_alphanumeric() || ch == '_' || ch == '.' {
7114 end += 1;
7115 } else {
7116 break;
7117 }
7118 }
7119 callee = line[index..end].to_string();
7120 }
7121 findings.push(BoundaryLintFinding {
7122 path: path.into(),
7123 line: i + 1,
7124 column: index + 1,
7125 rule_id: rule.rule_id.clone(),
7126 severity: "error".into(),
7127 callee,
7128 category: rule.category.clone(),
7129 message: rule.message.clone(),
7130 suggestion: rule.suggestion.clone(),
7131 });
7132 break;
7133 }
7134 }
7135 }
7136 BoundaryLintReport {
7137 passed: findings.is_empty(),
7138 scanned_files: vec![path.into()],
7139 finding_count: findings.len(),
7140 findings,
7141 }
7142}
7143
7144#[derive(Clone, Debug)]
7145pub struct RecoverySummary {
7146 pub recovered_steps: usize,
7147}
7148
7149#[derive(Clone, Debug)]
7150pub struct SchedulerStepStatus {
7151 pub step_id: String,
7152 pub status: String,
7153 pub owner: Option<String>,
7154 pub attempt: u64,
7155 pub lease_until: Option<f64>,
7156 pub last_error_type: Option<String>,
7157}
7158
7159#[derive(Clone, Debug)]
7160pub struct SchedulerStatus {
7161 pub run_id: String,
7162 pub run_status: String,
7163 pub state_version: u64,
7164 pub steps: Vec<SchedulerStepStatus>,
7165 pub cost_summary: CostSummary,
7166}
7167
7168pub struct RuntimeScheduler;
7169
7170impl RuntimeScheduler {
7171 pub fn recover_expired_leases(store: &mut MemoryStore) -> RecoverySummary {
7172 RecoverySummary {
7173 recovered_steps: store.recover_expired_leases(),
7174 }
7175 }
7176
7177 pub fn cancel_run(store: &mut MemoryStore, run_id: &str, reason: &str) -> Result<usize> {
7178 store.cancel_run(run_id, reason)
7179 }
7180
7181 pub fn status(store: &MemoryStore, run_id: &str) -> Result<SchedulerStatus> {
7182 let run = store.run(run_id)?;
7183 let steps = store
7184 .steps(run_id)
7185 .into_iter()
7186 .map(|step| SchedulerStepStatus {
7187 step_id: step.step_id,
7188 status: step.status,
7189 owner: step.owner,
7190 attempt: step.attempt,
7191 lease_until: step.lease_until,
7192 last_error_type: step.last_error_type,
7193 })
7194 .collect();
7195 Ok(SchedulerStatus {
7196 run_id: run_id.to_string(),
7197 run_status: run.status,
7198 state_version: run.state_version,
7199 steps,
7200 cost_summary: store.cost_summary(run_id),
7201 })
7202 }
7203}
7204
7205#[derive(Clone, Debug)]
7206pub struct ReviewCheck {
7207 pub name: String,
7208 pub passed: bool,
7209 pub severity: String,
7210 pub detail: String,
7211}
7212
7213#[derive(Clone, Debug)]
7214pub struct AdversarialReviewReport {
7215 pub passed: bool,
7216 pub run_id: Option<String>,
7217 pub checks: Vec<ReviewCheck>,
7218 pub metadata: State,
7219}
7220
7221pub fn adversarial_review(
7222 bundle: &EvidenceBundle,
7223 max_total_usd: Option<f64>,
7224) -> AdversarialReviewReport {
7225 let mut checks = vec![
7226 review_check(
7227 "no_failed_steps",
7228 !bundle
7229 .events
7230 .iter()
7231 .any(|event| event.event_type == "step_failed"),
7232 "blocker",
7233 "no step is in failed status",
7234 ),
7235 review_check(
7236 "no_pending_verification",
7237 !bundle
7238 .tool_ledger
7239 .iter()
7240 .any(|row| row.status == "PENDING_VERIFICATION"),
7241 "blocker",
7242 "no side effect is pending verification",
7243 ),
7244 review_check(
7245 "no_pending_approvals",
7246 !bundle.approvals.iter().any(|row| row.status == "PENDING"),
7247 "blocker",
7248 "no approval request is still pending",
7249 ),
7250 review_check(
7251 "completed_steps_have_completion_events",
7252 completed_steps_have_events_rust(&bundle.steps, &bundle.events),
7253 "blocker",
7254 "completed steps have step_completed events",
7255 ),
7256 review_check(
7257 "ledger_statuses_known",
7258 ledger_statuses_known_rust(&bundle.tool_ledger),
7259 "blocker",
7260 "Tool Ledger rows use known statuses",
7261 ),
7262 review_check(
7263 "event_sequence_contiguous",
7264 event_sequence_contiguous_rust(&bundle.events),
7265 "blocker",
7266 "event sequence has no gaps",
7267 ),
7268 review_check(
7269 "artifacts_have_blob_refs",
7270 bundle
7271 .artifacts
7272 .iter()
7273 .all(|row| !row.blob_ref.is_empty() && !row.blob_hash.is_empty()),
7274 "warning",
7275 "artifacts have blob refs and hashes",
7276 ),
7277 review_check(
7278 "media_artifacts_have_refs",
7279 bundle
7280 .media_artifacts
7281 .iter()
7282 .all(media_artifact_has_ref_rust),
7283 "blocker",
7284 "media artifacts have kind and durable refs",
7285 ),
7286 review_check(
7287 "stream_checkpoints_have_offsets",
7288 bundle
7289 .stream_checkpoints
7290 .iter()
7291 .all(stream_checkpoint_has_offset_rust),
7292 "blocker",
7293 "stream checkpoints have stream, consumer, and offset",
7294 ),
7295 review_check(
7296 "high_risk_approvals_decided",
7297 high_risk_approvals_decided_rust(&bundle.approvals),
7298 "blocker",
7299 "high-risk approval requests are decided",
7300 ),
7301 review_check(
7302 "no_blocking_failure_events",
7303 !bundle.events.iter().any(|event| {
7304 matches!(
7305 event.event_type.as_str(),
7306 "error_raised" | "step_failed" | "tool_call_failed" | "tool_call_blocked"
7307 )
7308 }),
7309 "warning",
7310 "no blocking failure events are present",
7311 ),
7312 ];
7313 if let Some(limit) = max_total_usd {
7314 checks.push(review_check(
7315 "max_total_usd",
7316 bundle.cost_summary.total_usd <= limit,
7317 "blocker",
7318 "cost limit check",
7319 ));
7320 }
7321 let passed = checks
7322 .iter()
7323 .all(|check| check.severity != "blocker" || check.passed);
7324 let mut metadata = State::new();
7325 metadata.insert(
7326 "event_count".into(),
7327 Value::Number(bundle.events.len() as f64),
7328 );
7329 metadata.insert(
7330 "tool_ledger_count".into(),
7331 Value::Number(bundle.tool_ledger.len() as f64),
7332 );
7333 metadata.insert(
7334 "approval_count".into(),
7335 Value::Number(bundle.approvals.len() as f64),
7336 );
7337 metadata.insert(
7338 "artifact_count".into(),
7339 Value::Number(bundle.artifacts.len() as f64),
7340 );
7341 metadata.insert(
7342 "media_artifact_count".into(),
7343 Value::Number(bundle.media_artifacts.len() as f64),
7344 );
7345 metadata.insert(
7346 "stream_checkpoint_count".into(),
7347 Value::Number(bundle.stream_checkpoints.len() as f64),
7348 );
7349 AdversarialReviewReport {
7350 passed,
7351 run_id: Some(bundle.run.run_id.clone()),
7352 checks,
7353 metadata,
7354 }
7355}
7356
7357fn review_check(name: &str, passed: bool, severity: &str, detail: &str) -> ReviewCheck {
7358 ReviewCheck {
7359 name: name.into(),
7360 passed,
7361 severity: severity.into(),
7362 detail: detail.into(),
7363 }
7364}
7365
7366fn completed_steps_have_events_rust(steps: &[Step], events: &[Event]) -> bool {
7367 steps.iter().all(|step| {
7368 step.status != "completed"
7369 || events.iter().any(|event| {
7370 event.event_type == "step_completed"
7371 && event.step_id.as_deref() == Some(step.step_id.as_str())
7372 })
7373 })
7374}
7375
7376fn ledger_statuses_known_rust(rows: &[ToolLedgerEntry]) -> bool {
7377 rows.iter().all(|row| {
7378 matches!(
7379 row.status.as_str(),
7380 "SUCCEEDED"
7381 | "FAILED_NO_EFFECT"
7382 | "PENDING_VERIFICATION"
7383 | "COMPENSATED"
7384 | "RUNNING"
7385 | "RESERVED"
7386 )
7387 })
7388}
7389fn event_sequence_contiguous_rust(events: &[Event]) -> bool {
7390 events
7391 .iter()
7392 .enumerate()
7393 .all(|(index, event)| event.seq == (index as u64) + 1)
7394}
7395fn state_has_key(row: &State, key: &str) -> bool {
7396 !matches!(row.get(key), None | Some(Value::Null))
7397}
7398fn media_artifact_has_ref_rust(row: &State) -> bool {
7399 state_has_key(row, "kind")
7400 && (state_has_key(row, "uri")
7401 || state_has_key(row, "content_ref")
7402 || state_has_key(row, "blob_ref"))
7403}
7404fn stream_checkpoint_has_offset_rust(row: &State) -> bool {
7405 state_has_key(row, "stream_id")
7406 && state_has_key(row, "consumer_id")
7407 && state_has_key(row, "offset")
7408}
7409fn high_risk_approvals_decided_rust(rows: &[ApprovalRequest]) -> bool {
7410 rows.iter().all(|row| {
7411 !matches!(
7412 row.risk_level.as_str(),
7413 "high" | "destructive" | "sensitive"
7414 ) || matches!(row.status.as_str(), "APPROVED" | "DENIED")
7415 })
7416}
7417
7418#[derive(Clone, Debug)]
7419pub struct EvidenceCheck {
7420 pub name: String,
7421 pub passed: bool,
7422 pub detail: String,
7423}
7424
7425#[derive(Clone, Debug)]
7426pub struct EvidenceCheckReport {
7427 pub passed: bool,
7428 pub checks: Vec<EvidenceCheck>,
7429 pub metadata: State,
7430}
7431
7432pub fn evaluate_evidence(
7433 bundle: &EvidenceBundle,
7434 max_total_usd: Option<f64>,
7435) -> EvidenceCheckReport {
7436 let mut checks = vec![
7437 evidence_check(
7438 "no_failed_steps",
7439 !bundle
7440 .events
7441 .iter()
7442 .any(|event| event.event_type == "step_failed"),
7443 "all steps completed or remain non-failed",
7444 ),
7445 evidence_check(
7446 "no_pending_verification",
7447 !bundle
7448 .tool_ledger
7449 .iter()
7450 .any(|row| row.status == "PENDING_VERIFICATION"),
7451 "no side effect is waiting for human/external verification",
7452 ),
7453 evidence_check(
7454 "completed_steps_have_events",
7455 completed_steps_have_events_rust(&bundle.steps, &bundle.events),
7456 "each completed step has a step_completed event",
7457 ),
7458 evidence_check(
7459 "managed_side_effects_are_ledgered",
7460 ledger_statuses_known_rust(&bundle.tool_ledger),
7461 "every ledger row has a known status",
7462 ),
7463 evidence_check(
7464 "media_artifacts_have_refs",
7465 bundle
7466 .media_artifacts
7467 .iter()
7468 .all(media_artifact_has_ref_rust),
7469 "media artifacts have kind and durable refs",
7470 ),
7471 evidence_check(
7472 "stream_checkpoints_have_offsets",
7473 bundle
7474 .stream_checkpoints
7475 .iter()
7476 .all(stream_checkpoint_has_offset_rust),
7477 "stream checkpoints have stream, consumer, and offset",
7478 ),
7479 ];
7480 if let Some(limit) = max_total_usd {
7481 checks.push(evidence_check(
7482 "max_total_usd",
7483 bundle.cost_summary.total_usd <= limit,
7484 "cost limit check",
7485 ));
7486 }
7487 EvidenceCheckReport {
7488 passed: checks.iter().all(|check| check.passed),
7489 checks,
7490 metadata: State::new(),
7491 }
7492}
7493
7494pub fn evaluate_evidence_regression(
7495 golden: &EvidenceBundle,
7496 current: &EvidenceBundle,
7497 max_total_usd_delta: Option<f64>,
7498) -> EvidenceCheckReport {
7499 let diff = diff_evidence(golden, current);
7500 let mut checks = vec![
7501 evidence_check(
7502 "final_state_regression",
7503 diff.final_state_changed_count == 0,
7504 "final state regression check",
7505 ),
7506 evidence_check(
7507 "event_type_regression",
7508 diff.event_types_changed_count == 0,
7509 "event type regression check",
7510 ),
7511 evidence_check(
7512 "tool_ledger_status_regression",
7513 true,
7514 "tool ledger status regression check",
7515 ),
7516 evidence_check(
7517 "media_artifact_regression",
7518 diff.media_artifacts_changed_count == 0,
7519 "media artifact regression check",
7520 ),
7521 evidence_check(
7522 "stream_checkpoint_regression",
7523 diff.stream_checkpoints_changed_count == 0,
7524 "stream checkpoint regression check",
7525 ),
7526 ];
7527 if let Some(limit) = max_total_usd_delta {
7528 let delta = current.cost_summary.total_usd - golden.cost_summary.total_usd;
7529 checks.push(evidence_check(
7530 "max_total_usd_delta",
7531 delta <= limit,
7532 "cost delta limit check",
7533 ));
7534 }
7535 EvidenceCheckReport {
7536 passed: checks.iter().all(|check| check.passed),
7537 checks,
7538 metadata: State::new(),
7539 }
7540}
7541
7542fn evidence_check(name: &str, passed: bool, detail: &str) -> EvidenceCheck {
7543 EvidenceCheck {
7544 name: name.into(),
7545 passed,
7546 detail: detail.into(),
7547 }
7548}
7549
7550#[derive(Clone, Debug)]
7551pub struct FailureInjectionCheck {
7552 pub name: String,
7553 pub passed: bool,
7554 pub detail: String,
7555 pub run_id: Option<String>,
7556}
7557
7558#[derive(Clone, Debug)]
7559pub struct FailureInjectionReport {
7560 pub passed: bool,
7561 pub checks: Vec<FailureInjectionCheck>,
7562}
7563
7564pub fn run_failure_injection_suite() -> FailureInjectionReport {
7565 let checks = vec![
7566 failure_retry_exhaustion(),
7567 failure_lease_fencing(),
7568 failure_cancellation_fencing(),
7569 failure_side_effect_idempotency(),
7570 ];
7571 FailureInjectionReport {
7572 passed: checks.iter().all(|check| check.passed),
7573 checks,
7574 }
7575}
7576fn failure_check(
7577 name: &str,
7578 passed: bool,
7579 detail: String,
7580 run_id: String,
7581) -> FailureInjectionCheck {
7582 FailureInjectionCheck {
7583 name: name.into(),
7584 passed,
7585 detail,
7586 run_id: Some(run_id),
7587 }
7588}
7589fn failure_retry_exhaustion() -> FailureInjectionCheck {
7590 let mut runtime = Runtime::new();
7591 let (run_id, _) = runtime.create_run(State::new());
7592 let _ = runtime.run_once(
7593 &run_id,
7594 "retry-1",
7595 "FailureInjector",
7596 60.0,
7597 |_ctx, _state| Err(RuntimeError("retryable".into())),
7598 );
7599 let _ = runtime.run_once(
7600 &run_id,
7601 "retry-2",
7602 "FailureInjector",
7603 60.0,
7604 |_ctx, _state| Err(RuntimeError("final failure".into())),
7605 );
7606 let status = runtime
7607 .store
7608 .run(&run_id)
7609 .map(|run| run.status)
7610 .unwrap_or_else(|_| "missing".into());
7611 failure_check(
7612 "retry_exhaustion",
7613 status == "failed",
7614 format!("run_status={status}"),
7615 run_id,
7616 )
7617}
7618fn failure_lease_fencing() -> FailureInjectionCheck {
7619 let mut store = MemoryStore::new();
7620 let (run_id, step_id) = store.create_run(State::new());
7621 let claim = store.claim_step("stale-worker", &run_id, 0.0).unwrap();
7622 let recovered = store.recover_expired_leases();
7623 let stale_rejected = store
7624 .commit_state_patch(&run_id, &step_id, &claim.lease_token, 0, State::new())
7625 .is_err();
7626 let fresh = store.claim_step("fresh-worker", &run_id, 60.0).unwrap();
7627 let passed = recovered == 1 && stale_rejected && fresh.attempt == 2;
7628 failure_check(
7629 "lease_fencing",
7630 passed,
7631 format!("recovered_steps={recovered} stale_rejected={stale_rejected}"),
7632 run_id,
7633 )
7634}
7635fn failure_cancellation_fencing() -> FailureInjectionCheck {
7636 let mut store = MemoryStore::new();
7637 let (run_id, step_id) = store.create_run(State::new());
7638 let claim = store.claim_step("stale-worker", &run_id, 60.0).unwrap();
7639 let cancelled = store.cancel_run(&run_id, "failure injection").unwrap();
7640 let stale_rejected = store
7641 .commit_state_patch(&run_id, &step_id, &claim.lease_token, 0, State::new())
7642 .is_err();
7643 let fresh = store.claim_step("fresh-worker", &run_id, 60.0).is_err();
7644 let status = store.run(&run_id).map(|run| run.status).unwrap_or_default();
7645 let passed = cancelled == 1 && stale_rejected && fresh && status == "cancelled";
7646 failure_check(
7647 "cancellation_fencing",
7648 passed,
7649 format!("cancelled_steps={cancelled} stale_rejected={stale_rejected}"),
7650 run_id,
7651 )
7652}
7653fn failure_side_effect_idempotency() -> FailureInjectionCheck {
7654 use std::sync::{Arc, Mutex};
7655 let calls = Arc::new(Mutex::new(0usize));
7656 let calls_for_tool = Arc::clone(&calls);
7657 let mut runtime = Runtime::new();
7658 runtime.register_tool(
7659 ToolSpec::new(
7660 "external.create",
7661 Box::new(move |_args| {
7662 let mut guard = calls_for_tool.lock().unwrap();
7663 *guard += 1;
7664 Ok(Value::Object(state(&[(
7665 "id",
7666 Value::String("EXT-1".into()),
7667 )])))
7668 }),
7669 )
7670 .side_effect("external")
7671 .idempotency_required(true),
7672 );
7673 let (run_id, _) = runtime.create_run(State::new());
7674 let ctx = failure_claim_context(&mut runtime, &run_id, "worker-1", "FailureInjector");
7675 let _ = runtime.call_tool(
7676 &ctx,
7677 "external.create",
7678 state(&[("title", Value::String("once".into()))]),
7679 );
7680 runtime
7681 .store
7682 .mark_retry(&run_id, &ctx.step_id, "RetryableAgentError", "retryable");
7683 let ctx2 = failure_claim_context(&mut runtime, &run_id, "worker-2", "FailureInjector");
7684 let _ = runtime.call_tool(
7685 &ctx2,
7686 "external.create",
7687 state(&[("title", Value::String("once".into()))]),
7688 );
7689 let count = *calls.lock().unwrap();
7690 failure_check(
7691 "side_effect_idempotency",
7692 count == 1,
7693 format!("external_call_count={count}"),
7694 run_id,
7695 )
7696}
7697
7698fn failure_claim_context(
7699 runtime: &mut Runtime,
7700 run_id: &str,
7701 worker_id: &str,
7702 agent_role: &str,
7703) -> AgentContext {
7704 let claim = runtime.store.claim_step(worker_id, run_id, 60.0).unwrap();
7705 AgentContext {
7706 run_id: claim.run_id,
7707 session_id: claim.session_id,
7708 step_id: claim.step_id,
7709 agent_role: agent_role.to_string(),
7710 lease_token: claim.lease_token,
7711 attempt: claim.attempt,
7712 state_version: claim.state_version,
7713 pending_patch: State::new(),
7714 }
7715}
7716
7717#[derive(Clone, Debug)]
7718pub struct ShadowReport {
7719 pub source_run_id: String,
7720 pub shadow_run_id: String,
7721 pub ok: bool,
7722 pub state_diff: State,
7723}
7724
7725pub fn diff_states(source: &State, shadow: &State) -> State {
7726 let mut changed = State::new();
7727 for key in source.keys().chain(shadow.keys()) {
7728 if changed.contains_key(key) {
7729 continue;
7730 }
7731 if source.get(key) != shadow.get(key) {
7732 changed.insert(
7733 key.clone(),
7734 Value::Object(state(&[
7735 ("source", source.get(key).cloned().unwrap_or_default()),
7736 ("shadow", shadow.get(key).cloned().unwrap_or_default()),
7737 ])),
7738 );
7739 }
7740 }
7741 state(&[
7742 ("changed", Value::Object(changed.clone())),
7743 ("changed_count", Value::Number(changed.len() as f64)),
7744 ])
7745}
7746
7747pub fn shadow_report(
7748 source_run_id: &str,
7749 shadow_run_id: &str,
7750 ok: bool,
7751 source_state: &State,
7752 shadow_state: &State,
7753) -> ShadowReport {
7754 ShadowReport {
7755 source_run_id: source_run_id.into(),
7756 shadow_run_id: shadow_run_id.into(),
7757 ok,
7758 state_diff: diff_states(source_state, shadow_state),
7759 }
7760}
7761
7762pub fn builtin_golden_names() -> Vec<String> {
7763 vec![
7764 "media-stream-checkpoint".into(),
7765 "minimal-success".into(),
7766 "tool-ledger-success".into(),
7767 ]
7768}
7769
7770pub fn builtin_golden_evidence(name: &str) -> Result<EvidenceBundle> {
7771 match name {
7772 "minimal-success" => golden_minimal_success(),
7773 "tool-ledger-success" => golden_tool_ledger_success(),
7774 "media-stream-checkpoint" => golden_media_stream_checkpoint(),
7775 _ => Err(RuntimeError(format!(
7776 "unknown built-in golden case: {name}"
7777 ))),
7778 }
7779}
7780
7781pub fn golden_regression(golden: &EvidenceBundle, current: &EvidenceBundle) -> EvidenceCheckReport {
7782 evaluate_evidence_regression(golden, current, None)
7783}
7784
7785fn golden_minimal_success() -> Result<EvidenceBundle> {
7786 let mut runtime = Runtime::new();
7787 let (run_id, _) = runtime.create_run(State::new());
7788 runtime.run_once(
7789 &run_id,
7790 "golden-worker",
7791 "GoldenAgent",
7792 60.0,
7793 |ctx, _state| {
7794 ctx.write_state("answer", Value::String("ok".into()));
7795 Ok(())
7796 },
7797 )?;
7798 export_evidence(&runtime.store, &run_id)
7799}
7800fn golden_tool_ledger_success() -> Result<EvidenceBundle> {
7801 let mut runtime = Runtime::new();
7802 runtime.register_tool(
7803 ToolSpec::new(
7804 "github.create_issue",
7805 Box::new(|_args| {
7806 Ok(Value::Object(state(&[(
7807 "issue_id",
7808 Value::String("ISSUE-1".into()),
7809 )])))
7810 }),
7811 )
7812 .side_effect("external"),
7813 );
7814 let (run_id, _) = runtime.create_run(State::new());
7815 runtime.run_once(
7816 &run_id,
7817 "golden-worker",
7818 "ExecutorAgent",
7819 60.0,
7820 |ctx, _state| {
7821 ctx.write_state("issue_id", Value::String("ISSUE-1".into()));
7822 Ok(())
7823 },
7824 )?;
7825 export_evidence(&runtime.store, &run_id)
7826}
7827fn golden_media_stream_checkpoint() -> Result<EvidenceBundle> {
7828 let mut runtime = Runtime::new();
7829 let (run_id, _) = runtime.create_run(State::new());
7830 runtime.run_once(
7831 &run_id,
7832 "golden-worker",
7833 "MediaAgent",
7834 60.0,
7835 |ctx, _state| {
7836 ctx.write_state("processed_offset", Value::Number(42.0));
7837 Ok(())
7838 },
7839 )?;
7840 runtime.store.create_artifact(
7841 &run_id,
7842 None,
7843 "golden-video-frame",
7844 State::new(),
7845 state(&[(
7846 "agentledger_media",
7847 Value::Object(state(&[
7848 ("kind", Value::String("frame".into())),
7849 ("uri", Value::String("file://golden-frame.jpg".into())),
7850 ])),
7851 )]),
7852 );
7853 runtime.store.create_artifact(
7854 &run_id,
7855 None,
7856 "golden-stream-checkpoint",
7857 State::new(),
7858 state(&[(
7859 "agentledger_stream",
7860 Value::Object(state(&[
7861 ("stream_id", Value::String("stream-golden".into())),
7862 ("consumer_id", Value::String("consumer-golden".into())),
7863 ("offset", Value::Number(42.0)),
7864 ])),
7865 )]),
7866 );
7867 export_evidence(&runtime.store, &run_id)
7868}
7869
7870#[derive(Clone, Debug)]
7871pub struct TimeTravelFrame {
7872 pub seq: u64,
7873 pub event_id: String,
7874 pub event_type: String,
7875 pub step_id: Option<String>,
7876 pub agent_role: Option<String>,
7877 pub state_version: Option<u64>,
7878 pub timestamp: f64,
7879 pub state_changed: bool,
7880 pub changed_keys: Vec<String>,
7881 pub patch: Option<State>,
7882 pub state_after: Option<State>,
7883}
7884
7885#[derive(Clone, Debug)]
7886pub struct TimeTravelReport {
7887 pub run_id: String,
7888 pub at_seq: Option<u64>,
7889 pub event_count: usize,
7890 pub timeline: Vec<TimeTravelFrame>,
7891 pub state_at_seq: State,
7892 pub selected_event: Option<TimeTravelFrame>,
7893}
7894
7895pub fn time_travel(
7896 bundle: &EvidenceBundle,
7897 at_seq: Option<u64>,
7898 include_states: bool,
7899) -> TimeTravelReport {
7900 let mut current = State::new();
7901 let mut state_at_seq = State::new();
7902 let mut selected_event = None;
7903 let mut timeline = Vec::new();
7904 for event in &bundle.events {
7905 let before = current.clone();
7906 let patch = patch_for_time_travel_event(event);
7907 if let Some(patch_value) = &patch {
7908 for (key, value) in patch_value {
7909 current.insert(key.clone(), value.clone());
7910 }
7911 }
7912 let diff = diff_states(&before, ¤t);
7913 let changed_keys = match diff.get("changed") {
7914 Some(Value::Object(obj)) => obj.keys().cloned().collect(),
7915 _ => Vec::new(),
7916 };
7917 let frame = TimeTravelFrame {
7918 seq: event.seq,
7919 event_id: event.event_id.clone(),
7920 event_type: event.event_type.clone(),
7921 step_id: event.step_id.clone(),
7922 agent_role: event.agent_role.clone(),
7923 state_version: event.state_version,
7924 timestamp: event.timestamp,
7925 state_changed: diff.get("changed_count") != Some(&Value::Number(0.0)),
7926 changed_keys,
7927 patch,
7928 state_after: if include_states {
7929 Some(current.clone())
7930 } else {
7931 None
7932 },
7933 };
7934 if at_seq.is_some_and(|seq| event.seq <= seq) {
7935 state_at_seq = current.clone();
7936 selected_event = Some(frame.clone());
7937 }
7938 timeline.push(frame);
7939 }
7940 if at_seq.is_none() {
7941 state_at_seq = current.clone();
7942 }
7943 TimeTravelReport {
7944 run_id: bundle.run.run_id.clone(),
7945 at_seq,
7946 event_count: timeline.len(),
7947 timeline,
7948 state_at_seq,
7949 selected_event,
7950 }
7951}
7952
7953fn patch_for_time_travel_event(event: &Event) -> Option<State> {
7954 if event.event_type == "run_created" {
7955 if let Some(Value::Object(obj)) = event.payload.get("initial_state") {
7956 return Some(obj.clone());
7957 }
7958 return Some(State::new());
7959 }
7960 if event.event_type == "state_committed"
7961 || event.event_type == "state_patch_committed"
7962 || event.event_type == "system_state_patch_applied"
7963 {
7964 if let Some(Value::Object(obj)) = event.payload.get("patch") {
7965 return Some(obj.clone());
7966 }
7967 return Some(State::new());
7968 }
7969 None
7970}
7971
7972pub fn time_travel_html(report: &TimeTravelReport) -> String {
7973 let rows = report
7974 .timeline
7975 .iter()
7976 .map(|frame| {
7977 format!(
7978 "<tr><td>{}</td><td>{}</td><td>{}</td></tr>",
7979 frame.seq,
7980 frame.event_type,
7981 frame.changed_keys.join(", ")
7982 )
7983 })
7984 .collect::<Vec<_>>()
7985 .join("\n");
7986 format!("<!doctype html><html><head><meta charset=\"utf-8\"><title>AgentLedger Time Travel Report</title></head><body><h1>AgentLedger Time Travel Report</h1><p>Run <code>{}</code></p><table>{}</table><h2>State At Selected Point</h2><pre>{:?}</pre><h2>Selected Event</h2><pre>{:?}</pre></body></html>", report.run_id, rows, report.state_at_seq, report.selected_event)
7987}
7988
7989#[derive(Debug, Clone, PartialEq, Eq)]
7990pub struct OptionalAdapterCapability {
7991 pub name: String,
7992 pub category: String,
7993 pub core_imports_heavy_sdks: bool,
7994 pub adapter_is_optional: bool,
7995 pub fail_closed_without_adapter: bool,
7996 pub contract_surface: Vec<String>,
7997}
7998
7999pub fn optional_adapter_capabilities() -> Vec<OptionalAdapterCapability> {
8000 fn item(name: &str, category: &str, surface: &[&str]) -> OptionalAdapterCapability {
8001 OptionalAdapterCapability {
8002 name: name.to_string(),
8003 category: category.to_string(),
8004 core_imports_heavy_sdks: false,
8005 adapter_is_optional: true,
8006 fail_closed_without_adapter: true,
8007 contract_surface: surface.iter().map(|s| s.to_string()).collect(),
8008 }
8009 }
8010 vec![
8011 item("postgres", "storage", &["ddl_for", "migrations_for", "state_store"]),
8012 item("mysql", "storage", &["ddl_for", "migrations_for", "state_store"]),
8013 item("s3", "blobstore", &["put_json", "get_json", "content_address"]),
8014 item("docker", "sandbox", &["sandbox_policy", "sandbox_result", "tool_gateway"]),
8015 item("e2b", "sandbox", &["sandbox_policy", "sandbox_result", "tool_gateway"]),
8016 item("bubblewrap", "sandbox", &["sandbox_policy", "sandbox_result", "tool_gateway"]),
8017 item("kubernetes", "sandbox", &["sandbox_policy", "sandbox_result", "tool_gateway"]),
8018 item("gvisor", "sandbox", &["sandbox_policy", "sandbox_result", "tool_gateway"]),
8019 item("firecracker", "sandbox", &["sandbox_policy", "sandbox_result", "tool_gateway"]),
8020 item("langgraph", "framework", &["framework_adapter", "checkpoint_contract"]),
8021 item("langchain", "framework", &["framework_adapter"]),
8022 item("crewai", "framework", &["framework_adapter"]),
8023 item("autogen", "framework", &["framework_adapter"]),
8024 item("openai-agents-sdk", "framework", &["framework_adapter"]),
8025 item("llamaindex", "framework", &["framework_adapter"]),
8026 item("semantic-kernel", "framework", &["framework_adapter"]),
8027 item("mcp-transport", "mcp", &["mcp_tool_descriptor", "mcp_resource_descriptor"]),
8028 item("langfuse", "observability", &["evidence_bundle", "trace_payload", "correlation_ids"]),
8029 item("shadow-runner", "shadow", &["evidence_bundle", "tool_ledger", "state_diff"]),
8030 ]
8031}
8032
8033pub trait SqlExecutor {
8034 fn exec(&mut self, sql: &str, params: &[Value]) -> Result<()>;
8035}
8036
8037pub struct PostgresAdapter<C: SqlExecutor> {
8038 pub schema: String,
8039 pub client: C,
8040}
8041
8042impl<C: SqlExecutor> PostgresAdapter<C> {
8043 pub fn new(client: C, schema: &str) -> Self {
8044 Self { schema: if schema.is_empty() { "agentledger".to_string() } else { schema.to_string() }, client }
8045 }
8046 pub fn migration_plan(&self) -> Result<Vec<Migration>> { migrations_for("postgres") }
8047 pub fn apply_migrations(&mut self) -> Result<()> {
8048 self.client.exec(&ddl_for("postgres")?, &[])?;
8049 for migration in self.migration_plan()? {
8050 self.client.exec(
8051 "INSERT INTO schema_migrations(version, name, checksum) VALUES ($1, $2, $3) ON CONFLICT (version) DO NOTHING",
8052 &[Value::String(migration.version), Value::String(migration.name), Value::String(stable_hash(&migration.sql))],
8053 )?;
8054 }
8055 Ok(())
8056 }
8057}
8058
8059pub struct MySQLAdapter<C: SqlExecutor> {
8060 pub database: String,
8061 pub client: C,
8062}
8063
8064impl<C: SqlExecutor> MySQLAdapter<C> {
8065 pub fn new(client: C, database: &str) -> Self {
8066 Self { database: if database.is_empty() { "agentledger".to_string() } else { database.to_string() }, client }
8067 }
8068 pub fn migration_plan(&self) -> Result<Vec<Migration>> { migrations_for("mysql") }
8069 pub fn apply_migrations(&mut self) -> Result<()> {
8070 self.client.exec(&ddl_for("mysql")?, &[])?;
8071 for migration in self.migration_plan()? {
8072 self.client.exec(
8073 "INSERT INTO schema_migrations(version, name, checksum, applied_at) VALUES (?, ?, ?, UNIX_TIMESTAMP()) ON DUPLICATE KEY UPDATE version=version",
8074 &[Value::String(migration.version), Value::String(migration.name), Value::String(stable_hash(&migration.sql))],
8075 )?;
8076 }
8077 Ok(())
8078 }
8079}
8080
8081pub trait ObjectClient {
8082 fn put_object(&mut self, bucket: &str, key: &str, body: &[u8], content_type: &str, metadata: State) -> Result<()>;
8083 fn get_object(&mut self, bucket: &str, key: &str) -> Result<Vec<u8>>;
8084}
8085
8086pub struct S3BlobStoreAdapter<C: ObjectClient> {
8087 pub bucket: String,
8088 pub prefix: String,
8089 pub client: C,
8090}
8091
8092impl<C: ObjectClient> S3BlobStoreAdapter<C> {
8093 pub fn new(client: C, bucket: &str, prefix: &str) -> Self {
8094 Self { bucket: bucket.to_string(), prefix: if prefix.is_empty() { "agentledger/blobs".to_string() } else { prefix.trim_matches('/').to_string() }, client }
8095 }
8096 pub fn put_json(&mut self, value: &Value) -> Result<(String, String)> {
8097 let encoded = encode_value(value);
8098 let digest = stable_hash(&encoded);
8099 let key = format!("{}/sha256/{}.json", self.prefix, digest);
8100 let mut metadata = State::new();
8101 metadata.insert("agentledger-digest".to_string(), Value::String(format!("sha256:{digest}")));
8102 self.client.put_object(&self.bucket, &key, encoded.as_bytes(), "application/json", metadata)?;
8103 Ok((format!("sha256:{digest}"), format!("s3://{}/{}", self.bucket, key)))
8104 }
8105 pub fn get_json(&mut self, reference: &str) -> Result<Value> {
8106 let prefix = format!("s3://{}/", self.bucket);
8107 if !reference.starts_with(&prefix) || reference.contains("..") {
8108 return Err(RuntimeError(format!("unsupported s3 blob ref: {reference}")));
8109 }
8110 let key = &reference[prefix.len()..];
8111 let body = self.client.get_object(&self.bucket, key)?;
8112 let text = String::from_utf8(body).map_err(|err| RuntimeError(err.to_string()))?;
8113 decode_value(&text)
8114 }
8115}
8116
8117pub trait OtlpClient { fn post_json(&mut self, endpoint: &str, payload: &str, content_type: &str) -> Result<()>; }
8118pub struct OtlpTransport<C: OtlpClient> { pub endpoint: String, pub client: C }
8119impl<C: OtlpClient> OtlpTransport<C> { pub fn export(&mut self, payload: &str) -> Result<()> { self.client.post_json(&self.endpoint, payload, "application/json") } }
8120
8121pub struct DockerSandboxAdapter { pub image: String }
8122impl DockerSandboxAdapter {
8123 pub fn manifest(&self, policy: &State, command: Vec<String>) -> State {
8124 let mut out = State::new();
8125 out.insert("backend".to_string(), Value::String("docker".to_string()));
8126 out.insert("image".to_string(), Value::String(if self.image.is_empty() { "python:3.11-slim".to_string() } else { self.image.clone() }));
8127 let network = match policy.get("network") { Some(Value::String(value)) if value != "deny" => value.clone(), _ => "none".to_string() };
8128 out.insert("network".to_string(), Value::String(network));
8129 out.insert("read_only_root".to_string(), Value::Bool(true));
8130 out.insert("requires_explicit_execution".to_string(), Value::Bool(true));
8131 out.insert("command".to_string(), Value::Array(command.into_iter().map(Value::String).collect()));
8132 out
8133 }
8134}
8135
8136pub struct DockerSandboxExecutor {
8137 pub image: String,
8138 pub binary: String,
8139 pub allow_command_execution: bool,
8140 pub allow_shell: bool,
8141 pub shell: String,
8142 pub memory: String,
8143 pub cpus: String,
8144}
8145
8146impl DockerSandboxExecutor {
8147 pub fn new(image: &str, allow_command_execution: bool) -> Self {
8148 Self {
8149 image: image.to_string(),
8150 binary: "docker".to_string(),
8151 allow_command_execution,
8152 allow_shell: false,
8153 shell: "/bin/sh".to_string(),
8154 memory: String::new(),
8155 cpus: String::new(),
8156 }
8157 }
8158
8159 pub fn with_binary(mut self, binary: &str) -> Self {
8160 self.binary = binary.to_string();
8161 self
8162 }
8163
8164 fn extract_command(&self, args: &State) -> std::result::Result<Vec<String>, String> {
8165 let raw = args.get("_sandbox_command").or_else(|| args.get("command"));
8166 match raw {
8167 Some(Value::String(command)) => {
8168 if !self.allow_shell {
8169 Err("string commands require allow_shell=true; pass argv list in `_sandbox_command` instead".to_string())
8170 } else {
8171 let shell = if self.shell.is_empty() { "/bin/sh" } else { &self.shell };
8172 Ok(vec![shell.to_string(), "-lc".to_string(), command.clone()])
8173 }
8174 }
8175 Some(Value::Array(items)) => {
8176 let mut command = Vec::new();
8177 for item in items {
8178 match item {
8179 Value::String(value) if !value.is_empty() => command.push(value.clone()),
8180 _ => return Err("_sandbox_command must be a non-empty string array".to_string()),
8181 }
8182 }
8183 if command.is_empty() {
8184 Err("_sandbox_command must be a non-empty string array".to_string())
8185 } else {
8186 Ok(command)
8187 }
8188 }
8189 _ => Err("external sandbox tools require a command-style `_sandbox_command` arg".to_string()),
8190 }
8191 }
8192
8193 fn docker_argv(&self, policy: &SandboxPolicy, command: &[String]) -> Vec<String> {
8194 let image = if self.image.is_empty() { "python:3.11-slim" } else { &self.image };
8195 let network = if policy.network == "deny" || policy.network.is_empty() { "none" } else { &policy.network };
8196 let mut argv = vec![
8197 self.binary.clone(),
8198 "run".to_string(),
8199 "--rm".to_string(),
8200 "--network".to_string(),
8201 network.to_string(),
8202 "--read-only".to_string(),
8203 ];
8204 if !self.memory.is_empty() {
8205 argv.extend(["--memory".to_string(), self.memory.clone()]);
8206 }
8207 if !self.cpus.is_empty() {
8208 argv.extend(["--cpus".to_string(), self.cpus.clone()]);
8209 }
8210 argv.push(image.to_string());
8211 argv.extend(command.iter().cloned());
8212 argv
8213 }
8214
8215 fn result_error(policy: &SandboxPolicy, manifest: State, error_type: &str, error: String) -> SandboxResult {
8216 let mut metadata = State::new();
8217 metadata.insert("executor".to_string(), Value::String(policy.executor.clone()));
8218 metadata.insert("isolation_level".to_string(), Value::String("container".to_string()));
8219 metadata.insert("manifest".to_string(), Value::Object(manifest));
8220 metadata.insert("error_type".to_string(), Value::String(error_type.to_string()));
8221 SandboxResult { ok: false, output: Value::Null, error: Some(error), metadata }
8222 }
8223}
8224
8225impl SandboxExecutor for DockerSandboxExecutor {
8226 fn run_tool(&self, args: State, policy: &SandboxPolicy) -> SandboxResult {
8227 let command = match self.extract_command(&args) {
8228 Ok(command) => command,
8229 Err(error) => return Self::result_error(policy, State::new(), "InvalidSandboxCommand", error),
8230 };
8231 let mut policy_state = State::new();
8232 policy_state.insert("network".to_string(), Value::String(policy.network.clone()));
8233 let manifest = (DockerSandboxAdapter { image: self.image.clone() }).manifest(&policy_state, command.clone());
8234 if !self.allow_command_execution {
8235 return Self::result_error(policy, manifest, "SandboxAdapterNotInstalled", "command execution is not enabled for this executor".to_string());
8236 }
8237 let argv = self.docker_argv(policy, &command);
8238 let mut cmd = Command::new(&argv[0]);
8239 cmd.args(&argv[1..]).stdout(Stdio::piped()).stderr(Stdio::piped());
8240 let output = match cmd.output() {
8241 Ok(output) => output,
8242 Err(error) => return Self::result_error(policy, manifest, "SandboxBinaryMissing", error.to_string()),
8243 };
8244 let mut value = State::new();
8245 value.insert("stdout".to_string(), Value::String(String::from_utf8_lossy(&output.stdout).to_string()));
8246 value.insert("stderr".to_string(), Value::String(String::from_utf8_lossy(&output.stderr).to_string()));
8247 value.insert("returncode".to_string(), Value::Number(output.status.code().unwrap_or(-1) as f64));
8248 let mut metadata = State::new();
8249 metadata.insert("executor".to_string(), Value::String(policy.executor.clone()));
8250 metadata.insert("isolation_level".to_string(), Value::String("container".to_string()));
8251 metadata.insert("manifest".to_string(), Value::Object(manifest));
8252 metadata.insert("executed".to_string(), Value::Bool(true));
8253 if !output.status.success() {
8254 metadata.insert("error_type".to_string(), Value::String("SandboxCommandFailed".to_string()));
8255 return SandboxResult {
8256 ok: false,
8257 output: Value::Object(value),
8258 error: Some(format!("sandbox command exited with {}", output.status.code().unwrap_or(-1))),
8259 metadata,
8260 };
8261 }
8262 SandboxResult { ok: true, output: Value::Object(value), error: None, metadata }
8263 }
8264}
8265
8266pub mod adapters {
8267 pub mod postgres {
8268 pub const PACKAGE_NAME: &str = "agentledger-postgres";
8269 pub const FEATURE: &str = "adapter-postgres";
8270 pub use crate::{migrations_for, Migration, PostgresAdapter, SqlExecutor};
8271 }
8272
8273 pub mod mysql {
8274 pub const PACKAGE_NAME: &str = "agentledger-mysql";
8275 pub const FEATURE: &str = "adapter-mysql";
8276 pub use crate::{migrations_for, Migration, MySQLAdapter, SqlExecutor};
8277 }
8278
8279 pub mod s3 {
8280 pub const PACKAGE_NAME: &str = "agentledger-s3";
8281 pub const FEATURE: &str = "adapter-s3";
8282 pub use crate::{ObjectClient, S3BlobStoreAdapter};
8283 }
8284
8285 pub mod mcp {
8286 pub const PACKAGE_NAME: &str = "agentledger-mcp";
8287 pub const FEATURE: &str = "adapter-mcp";
8288 pub use crate::{
8289 InMemoryMCPContextServer, InMemoryMCPToolServer, MCPCall, MCPContextAdapter,
8290 MCPResourceDescriptor, MCPResourceRead, MCPToolAdapter,
8291 };
8292 }
8293
8294 pub mod otel {
8295 pub const PACKAGE_NAME: &str = "agentledger-otel";
8296 pub const FEATURE: &str = "adapter-otel";
8297 pub use crate::{OtlpClient, OtlpTransport};
8298 }
8299
8300 pub mod langfuse {
8301 pub const PACKAGE_NAME: &str = "agentledger-langfuse";
8302 pub const FEATURE: &str = "adapter-langfuse";
8303 pub const CATEGORY: &str = "observability";
8304 }
8305
8306 pub mod docker {
8307 pub const PACKAGE_NAME: &str = "agentledger-sandbox-docker";
8308 pub const FEATURE: &str = "adapter-docker";
8309 pub use crate::{DockerSandboxAdapter, DockerSandboxExecutor, State, Value};
8310 }
8311
8312 pub mod framework {
8313 pub const PACKAGE_NAME: &str = "agentledger-framework";
8314 pub const FEATURE: &str = "adapter-framework";
8315 pub use crate::{FunctionAdapter, MethodFrameworkAdapter};
8316 }
8317}