1use std::collections::HashMap;
2use std::fmt;
3use std::fs;
4use std::path::{Path, PathBuf};
5use std::process::{Command, Stdio};
6use std::sync::atomic::{AtomicU64, Ordering};
7use std::time::{SystemTime, UNIX_EPOCH};
8
9static ID_COUNTER: AtomicU64 = AtomicU64::new(1);
10
11pub type State = HashMap<String, Value>;
12
13pub const MODEL_EVIDENCE_SCHEMA_VERSION: &str = "agentledger.model.evidence.v1";
14
15#[derive(Clone, Debug, PartialEq)]
16pub enum Value {
17 Null,
18 Bool(bool),
19 Number(f64),
20 String(String),
21 Object(State),
22 Array(Vec<Value>),
23}
24
25impl Default for Value {
26 fn default() -> Self {
27 Value::Null
28 }
29}
30
31impl From<&str> for Value {
32 fn from(value: &str) -> Self {
33 Value::String(value.to_string())
34 }
35}
36
37impl From<String> for Value {
38 fn from(value: String) -> Self {
39 Value::String(value)
40 }
41}
42
43impl From<bool> for Value {
44 fn from(value: bool) -> Self {
45 Value::Bool(value)
46 }
47}
48
49impl From<i64> for Value {
50 fn from(value: i64) -> Self {
51 Value::Number(value as f64)
52 }
53}
54
55impl From<f64> for Value {
56 fn from(value: f64) -> Self {
57 Value::Number(value)
58 }
59}
60
61#[derive(Clone, Debug)]
62pub struct Run {
63 pub run_id: String,
64 pub session_id: String,
65 pub status: String,
66 pub state: State,
67 pub state_version: u64,
68 pub created_at: f64,
69 pub updated_at: f64,
70}
71
72#[derive(Clone, Debug)]
73pub struct Step {
74 pub step_id: String,
75 pub run_id: String,
76 pub session_id: String,
77 pub status: String,
78 pub owner: Option<String>,
79 pub lease_token: Option<String>,
80 pub lease_until: Option<f64>,
81 pub attempt: u64,
82 pub state_version: u64,
83 pub checkpoint_id: Option<String>,
84 pub last_error_type: Option<String>,
85 pub last_error: Option<String>,
86 pub cancelled_at: Option<f64>,
87 pub created_at: f64,
88 pub updated_at: f64,
89}
90
91#[derive(Clone, Debug)]
92pub struct StepClaim {
93 pub run_id: String,
94 pub session_id: String,
95 pub step_id: String,
96 pub attempt: u64,
97 pub lease_token: String,
98 pub state_version: u64,
99 pub lease_until: f64,
100}
101
102#[derive(Clone, Debug)]
103pub struct Event {
104 pub event_id: String,
105 pub run_id: String,
106 pub session_id: Option<String>,
107 pub step_id: Option<String>,
108 pub seq: u64,
109 pub event_type: String,
110 pub timestamp: f64,
111 pub agent_role: Option<String>,
112 pub state_version: Option<u64>,
113 pub causal_token: Option<String>,
114 pub payload_hash: String,
115 pub payload_ref: String,
116 pub payload: State,
117}
118
119#[derive(Clone, Debug)]
120pub struct ToolLedgerEntry {
121 pub ledger_id: String,
122 pub run_id: String,
123 pub session_id: String,
124 pub step_id: String,
125 pub tool_name: String,
126 pub tool_version: String,
127 pub tool_call_id: String,
128 pub idempotency_key: String,
129 pub causal_token: String,
130 pub request_hash: String,
131 pub request_ref: String,
132 pub status: String,
133 pub external_id: Option<String>,
134 pub response_hash: Option<String>,
135 pub response_ref: Option<String>,
136 pub error_type: Option<String>,
137 pub response: Option<Value>,
138 pub created_at: f64,
139 pub updated_at: f64,
140}
141
142#[derive(Clone, Debug)]
143pub struct ApprovalRequest {
144 pub approval_id: String,
145 pub approval_key: String,
146 pub run_id: String,
147 pub session_id: String,
148 pub step_id: String,
149 pub tool_name: String,
150 pub risk_level: String,
151 pub status: String,
152 pub reason: String,
153 pub request_hash: String,
154 pub request_ref: String,
155 pub requested_by: String,
156 pub approved_by: Option<String>,
157 pub decision_reason: Option<String>,
158 pub created_at: f64,
159 pub updated_at: f64,
160}
161
162#[derive(Clone, Debug)]
163pub struct CostRecord {
164 pub cost_id: String,
165 pub run_id: String,
166 pub session_id: String,
167 pub step_id: String,
168 pub category: String,
169 pub name: String,
170 pub amount: f64,
171 pub unit: String,
172 pub metadata: State,
173 pub created_at: f64,
174}
175
176#[derive(Clone, Debug, Default, PartialEq)]
177pub struct CostSummary {
178 pub tool_calls: f64,
179 pub model_tokens: f64,
180 pub total_usd: f64,
181 pub by_category: HashMap<String, f64>,
182}
183
184pub const MEDIA_SCHEMA_VERSION: &str = "agentledger.media.v0";
185pub const STREAM_SCHEMA_VERSION: &str = "agentledger.stream.v0";
186
187#[derive(Clone, Debug)]
188pub struct LocalBlobStore {
189 root: PathBuf,
190}
191
192impl LocalBlobStore {
193 pub fn open(root: impl AsRef<Path>) -> Result<Self> {
194 fs::create_dir_all(root.as_ref()).map_err(|err| RuntimeError(err.to_string()))?;
195 Ok(Self {
196 root: root.as_ref().to_path_buf(),
197 })
198 }
199
200 pub fn put_json(&self, value: &Value) -> Result<(String, String)> {
201 let encoded = encode_value(value);
202 let digest = stable_hash(&encoded);
203 let dir = self.root.join("sha256");
204 fs::create_dir_all(&dir).map_err(|err| RuntimeError(err.to_string()))?;
205 let path = dir.join(format!("{digest}.json"));
206 if !path.exists() {
207 let tmp = path.with_extension("json.tmp");
208 fs::write(&tmp, encoded).map_err(|err| RuntimeError(err.to_string()))?;
209 fs::rename(&tmp, &path).map_err(|err| RuntimeError(err.to_string()))?;
210 }
211 Ok((
212 format!("sha256:{digest}"),
213 format!("blob://sha256/{digest}.json"),
214 ))
215 }
216
217 pub fn get_json(&self, reference: &str) -> Result<Value> {
218 let prefix = "blob://sha256/";
219 if !reference.starts_with(prefix) {
220 return Err(RuntimeError(format!("unsupported blob ref: {reference}")));
221 }
222 let name = &reference[prefix.len()..];
223 if !name.ends_with(".json")
224 || name.contains("..")
225 || name.contains('/')
226 || name.contains('\\')
227 {
228 return Err(RuntimeError(format!("unsupported blob ref: {reference}")));
229 }
230 let body = fs::read_to_string(self.root.join("sha256").join(name))
231 .map_err(|err| RuntimeError(err.to_string()))?;
232 decode_value(&body)
233 }
234}
235
236#[derive(Clone, Debug)]
237pub struct Artifact {
238 pub artifact_id: String,
239 pub run_id: String,
240 pub step_id: Option<String>,
241 pub name: String,
242 pub blob_hash: String,
243 pub blob_ref: String,
244 pub metadata: State,
245 pub created_at: f64,
246}
247
248#[derive(Clone, Debug, Default)]
249pub struct MediaArtifactOptions {
250 pub uri: Option<String>,
251 pub content_ref: Option<String>,
252 pub media_metadata: State,
253 pub lineage: State,
254 pub derived_outputs: State,
255 pub metadata: State,
256}
257
258#[derive(Clone, Debug, Default)]
259pub struct StreamChunkRef {
260 pub stream_id: String,
261 pub chunk_id: String,
262 pub offset: Value,
263 pub content_ref: Option<String>,
264 pub content_hash: Option<String>,
265 pub sequence: Option<f64>,
266 pub event_time: Option<f64>,
267 pub metadata: State,
268}
269
270#[derive(Clone, Debug, Default)]
271pub struct StreamCheckpointOptions {
272 pub stream_id: String,
273 pub consumer_id: String,
274 pub offset: Value,
275 pub watermark: Option<Value>,
276 pub chunk: Option<StreamChunkRef>,
277 pub partial_result_ref: Option<String>,
278 pub backpressure: State,
279 pub metadata: State,
280}
281
282#[derive(Clone, Debug, Default)]
283pub struct BudgetLimits {
284 pub max_tool_calls: Option<f64>,
285 pub max_model_tokens: Option<f64>,
286 pub max_total_usd: Option<f64>,
287}
288
289#[derive(Debug, Clone)]
290pub struct RuntimeError(pub String);
291
292impl fmt::Display for RuntimeError {
293 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
294 write!(f, "{}", self.0)
295 }
296}
297
298impl std::error::Error for RuntimeError {}
299
300pub type Result<T> = std::result::Result<T, RuntimeError>;
301pub type ToolFunc = Box<dyn Fn(State) -> Result<Value> + Send + Sync>;
302pub type AgentFunc = fn(&mut AgentContext, State) -> Result<()>;
303
304pub struct SandboxPolicy {
305 pub tool_name: String,
306 pub run_id: String,
307 pub step_id: String,
308 pub executor: String,
309 pub network: String,
310 pub filesystem: String,
311 pub timeout_seconds: u64,
312 pub extra: State,
313}
314
315pub struct SandboxResult {
316 pub ok: bool,
317 pub output: Value,
318 pub error: Option<String>,
319 pub metadata: State,
320}
321
322pub trait SandboxExecutor {
323 fn run_tool(&self, args: State, policy: &SandboxPolicy) -> SandboxResult;
324}
325
326pub struct DisabledSandboxExecutor;
327
328impl SandboxExecutor for DisabledSandboxExecutor {
329 fn run_tool(&self, _args: State, policy: &SandboxPolicy) -> SandboxResult {
330 let mut metadata = State::new();
331 metadata.insert("executor".to_string(), Value::String(policy.executor.clone()));
332 metadata.insert("isolation_level".to_string(), Value::String("none".to_string()));
333 metadata.insert("fail_closed".to_string(), Value::Bool(true));
334 SandboxResult {
335 ok: false,
336 output: Value::Null,
337 error: Some(format!("sandbox executor \"{}\" is disabled", policy.executor)),
338 metadata,
339 }
340 }
341}
342
343pub struct ToolSpec {
344 pub name: String,
345 pub version: String,
346 pub side_effect: String,
347 pub risk_level: String,
348 pub idempotency_required: bool,
349 pub approval_required: bool,
350 pub sandbox_required: bool,
351 pub sandbox_executor: String,
352 pub sandbox_policy: State,
353 pub input_schema: Option<Value>,
354 pub output_schema: Option<Value>,
355 pub func: ToolFunc,
356}
357
358impl ToolSpec {
359 pub fn new(name: &str, func: ToolFunc) -> Self {
360 Self {
361 name: name.to_string(),
362 version: "v1".to_string(),
363 side_effect: "none".to_string(),
364 risk_level: "low".to_string(),
365 idempotency_required: false,
366 approval_required: false,
367 sandbox_required: false,
368 sandbox_executor: String::new(),
369 sandbox_policy: State::new(),
370 input_schema: None,
371 output_schema: None,
372 func,
373 }
374 }
375
376 pub fn side_effect(mut self, side_effect: &str) -> Self {
377 self.side_effect = side_effect.to_string();
378 self
379 }
380
381 pub fn risk_level(mut self, risk_level: &str) -> Self {
382 self.risk_level = risk_level.to_string();
383 self
384 }
385
386 pub fn idempotency_required(mut self, required: bool) -> Self {
387 self.idempotency_required = required;
388 self
389 }
390
391 pub fn approval_required(mut self, required: bool) -> Self {
392 self.approval_required = required;
393 self
394 }
395
396 pub fn sandbox_required(mut self, required: bool) -> Self {
397 self.sandbox_required = required;
398 self
399 }
400
401 pub fn sandbox_executor(mut self, executor: &str) -> Self {
402 self.sandbox_executor = executor.to_string();
403 self
404 }
405
406 pub fn sandbox_policy(mut self, policy: State) -> Self {
407 self.sandbox_policy = policy;
408 self
409 }
410
411 pub fn input_schema(mut self, schema: Value) -> Self {
412 self.input_schema = Some(schema);
413 self
414 }
415
416 pub fn output_schema(mut self, schema: Value) -> Self {
417 self.output_schema = Some(schema);
418 self
419 }
420}
421
422#[derive(Default)]
423pub struct MemoryStore {
424 runs: HashMap<String, Run>,
425 steps: HashMap<String, Step>,
426 events: HashMap<String, Vec<Event>>,
427 tool_ledger: HashMap<String, ToolLedgerEntry>,
428 approval_requests: HashMap<String, ApprovalRequest>,
429 cost_records: HashMap<String, Vec<CostRecord>>,
430 artifacts: HashMap<String, Vec<Artifact>>,
431}
432
433impl MemoryStore {
434 pub fn new() -> Self {
435 Self::default()
436 }
437
438 pub fn save_to_path(&self, path: impl AsRef<Path>) -> Result<()> {
439 fs::write(path, encode_store(self)).map_err(|err| RuntimeError(err.to_string()))
440 }
441
442 pub fn load_from_path(path: impl AsRef<Path>) -> Result<Self> {
443 let body = fs::read_to_string(path).map_err(|err| RuntimeError(err.to_string()))?;
444 decode_store(&body)
445 }
446
447 pub fn create_run(&mut self, initial_state: State) -> (String, String) {
448 let run_id = new_id("run");
449 let session_id = new_id("sess");
450 let step_id = new_id("step");
451 let now = now_seconds();
452 self.runs.insert(
453 run_id.clone(),
454 Run {
455 run_id: run_id.clone(),
456 session_id: session_id.clone(),
457 status: "pending".to_string(),
458 state: initial_state.clone(),
459 state_version: 0,
460 created_at: now,
461 updated_at: now,
462 },
463 );
464 self.steps.insert(
465 step_id.clone(),
466 Step {
467 step_id: step_id.clone(),
468 run_id: run_id.clone(),
469 session_id: session_id.clone(),
470 status: "pending".to_string(),
471 owner: None,
472 lease_token: None,
473 lease_until: None,
474 attempt: 0,
475 state_version: 0,
476 checkpoint_id: None,
477 last_error_type: None,
478 last_error: None,
479 cancelled_at: None,
480 created_at: now,
481 updated_at: now,
482 },
483 );
484 let mut payload = State::new();
485 payload.insert("initial_state".to_string(), Value::Object(initial_state));
486 self.append_event(
487 &run_id,
488 Some(&session_id),
489 None,
490 "run_created",
491 payload,
492 None,
493 None,
494 None,
495 );
496 let mut step_payload = State::new();
497 step_payload.insert("step_id".to_string(), Value::String(step_id.clone()));
498 self.append_event(
499 &run_id,
500 Some(&session_id),
501 Some(&step_id),
502 "step_created",
503 step_payload,
504 None,
505 None,
506 None,
507 );
508 (run_id, step_id)
509 }
510
511 pub fn create_external_step(&mut self, run_id: &str) -> Result<Step> {
512 let run = self
513 .runs
514 .get(run_id)
515 .ok_or_else(|| RuntimeError(format!("run not found: {run_id}")))?
516 .clone();
517 let step_id = new_id("step");
518 let now = now_seconds();
519 let step = Step {
520 step_id: step_id.clone(),
521 run_id: run_id.to_string(),
522 session_id: run.session_id.clone(),
523 status: "pending".to_string(),
524 owner: None,
525 lease_token: None,
526 lease_until: None,
527 attempt: 0,
528 state_version: run.state_version,
529 checkpoint_id: None,
530 last_error_type: None,
531 last_error: None,
532 cancelled_at: None,
533 created_at: now,
534 updated_at: now,
535 };
536 self.steps.insert(step_id.clone(), step.clone());
537 if let Some(run) = self.runs.get_mut(run_id) {
538 run.status = "pending".to_string();
539 run.updated_at = now;
540 }
541 let mut payload = State::new();
542 payload.insert("step_id".to_string(), Value::String(step_id));
543 payload.insert("external".to_string(), Value::Bool(true));
544 self.append_event(
545 run_id,
546 Some(&run.session_id),
547 Some(&step.step_id),
548 "step_created",
549 payload,
550 None,
551 Some(run.state_version),
552 None,
553 );
554 Ok(step)
555 }
556
557 pub fn claim_step(
558 &mut self,
559 worker_id: &str,
560 run_id: &str,
561 lease_seconds: f64,
562 ) -> Result<StepClaim> {
563 let mut candidates: Vec<Step> = self
564 .steps
565 .values()
566 .filter(|step| {
567 (run_id.is_empty() || step.run_id == run_id)
568 && (step.status == "pending" || step.status == "retry_scheduled")
569 })
570 .cloned()
571 .collect();
572 candidates.sort_by(|a, b| {
573 a.created_at
574 .partial_cmp(&b.created_at)
575 .unwrap()
576 .then(a.step_id.cmp(&b.step_id))
577 });
578 let candidate = candidates
579 .first()
580 .ok_or_else(|| RuntimeError("agentledger: no runnable step".to_string()))?;
581 let now = now_seconds();
582 let step = self.steps.get_mut(&candidate.step_id).unwrap();
583 step.status = "running".to_string();
584 step.owner = Some(worker_id.to_string());
585 step.lease_token = Some(new_id("lease"));
586 step.lease_until = Some(now + lease_seconds);
587 step.attempt += 1;
588 step.updated_at = now;
589 let claim = StepClaim {
590 run_id: step.run_id.clone(),
591 session_id: step.session_id.clone(),
592 step_id: step.step_id.clone(),
593 attempt: step.attempt,
594 lease_token: step.lease_token.clone().unwrap(),
595 state_version: step.state_version,
596 lease_until: step.lease_until.unwrap(),
597 };
598 let run = self.runs.get_mut(&claim.run_id).unwrap();
599 run.status = "running".to_string();
600 run.updated_at = now;
601 let mut payload = State::new();
602 payload.insert(
603 "worker_id".to_string(),
604 Value::String(worker_id.to_string()),
605 );
606 payload.insert(
607 "lease_token".to_string(),
608 Value::String(claim.lease_token.clone()),
609 );
610 payload.insert("attempt".to_string(), Value::Number(claim.attempt as f64));
611 self.append_event(
612 &claim.run_id,
613 Some(&claim.session_id),
614 Some(&claim.step_id),
615 "step_claimed",
616 payload,
617 None,
618 None,
619 None,
620 );
621 Ok(claim)
622 }
623
624 pub fn load_state(&self, run_id: &str) -> Result<(State, u64, String)> {
625 let run = self
626 .runs
627 .get(run_id)
628 .ok_or_else(|| RuntimeError(format!("run not found: {run_id}")))?;
629 Ok((run.state.clone(), run.state_version, run.session_id.clone()))
630 }
631
632 pub fn recover_expired_leases(&mut self) -> usize {
633 let now = now_seconds();
634 let ids: Vec<String> = self.steps.keys().cloned().collect();
635 let mut recovered = 0;
636 for step_id in ids {
637 let should_recover = self
638 .steps
639 .get(&step_id)
640 .map(|step| {
641 step.status == "running" && step.lease_until.is_some_and(|until| until <= now)
642 })
643 .unwrap_or(false);
644 if !should_recover {
645 continue;
646 }
647 let (run_id, session_id, previous_owner, attempt) = {
648 let step = self.steps.get_mut(&step_id).unwrap();
649 let previous_owner = step.owner.clone().unwrap_or_default();
650 step.status = "retry_scheduled".to_string();
651 step.owner = None;
652 step.lease_token = None;
653 step.lease_until = None;
654 step.updated_at = now;
655 (
656 step.run_id.clone(),
657 step.session_id.clone(),
658 previous_owner,
659 step.attempt,
660 )
661 };
662 let run = self.runs.get_mut(&run_id).unwrap();
663 run.status = "retry_scheduled".to_string();
664 run.updated_at = now;
665 recovered += 1;
666 let mut payload = State::new();
667 payload.insert("previous_owner".to_string(), Value::String(previous_owner));
668 payload.insert("attempt".to_string(), Value::Number(attempt as f64));
669 self.append_event(
670 &run_id,
671 Some(&session_id),
672 Some(&step_id),
673 "lease_expired",
674 payload,
675 None,
676 None,
677 None,
678 );
679 let mut retry_payload = State::new();
680 retry_payload.insert("step_id".to_string(), Value::String(step_id.clone()));
681 retry_payload.insert(
682 "reason".to_string(),
683 Value::String("lease_expired".to_string()),
684 );
685 self.append_event(
686 &run_id,
687 Some(&session_id),
688 Some(&step_id),
689 "step_retry_scheduled",
690 retry_payload,
691 None,
692 None,
693 None,
694 );
695 }
696 recovered
697 }
698
699 pub fn cancel_run(&mut self, run_id: &str, reason: &str) -> Result<usize> {
700 let session_id = self
701 .runs
702 .get(run_id)
703 .ok_or_else(|| RuntimeError(format!("run not found: {run_id}")))?
704 .session_id
705 .clone();
706 let mut payload = State::new();
707 payload.insert("reason".to_string(), Value::String(reason.to_string()));
708 self.append_event(
709 run_id,
710 Some(&session_id),
711 None,
712 "run_cancel_requested",
713 payload,
714 None,
715 None,
716 None,
717 );
718 let ids: Vec<String> = self.steps.keys().cloned().collect();
719 let now = now_seconds();
720 let mut cancelled = 0;
721 for step_id in ids {
722 let should_cancel = self
723 .steps
724 .get(&step_id)
725 .map(|step| {
726 step.run_id == run_id
727 && !["completed", "failed", "cancelled"].contains(&step.status.as_str())
728 })
729 .unwrap_or(false);
730 if !should_cancel {
731 continue;
732 }
733 let step_session = {
734 let step = self.steps.get_mut(&step_id).unwrap();
735 step.status = "cancelled".to_string();
736 step.owner = None;
737 step.lease_token = None;
738 step.lease_until = None;
739 step.cancelled_at = Some(now);
740 step.updated_at = now;
741 step.session_id.clone()
742 };
743 cancelled += 1;
744 let mut step_payload = State::new();
745 step_payload.insert("reason".to_string(), Value::String(reason.to_string()));
746 self.append_event(
747 run_id,
748 Some(&step_session),
749 Some(&step_id),
750 "step_cancelled",
751 step_payload,
752 None,
753 None,
754 None,
755 );
756 }
757 let run = self.runs.get_mut(run_id).unwrap();
758 run.status = "cancelled".to_string();
759 run.updated_at = now;
760 let mut done_payload = State::new();
761 done_payload.insert("reason".to_string(), Value::String(reason.to_string()));
762 done_payload.insert(
763 "cancelled_steps".to_string(),
764 Value::Number(cancelled as f64),
765 );
766 self.append_event(
767 run_id,
768 Some(&session_id),
769 None,
770 "run_cancelled",
771 done_payload,
772 None,
773 None,
774 None,
775 );
776 Ok(cancelled)
777 }
778
779 pub fn commit_state_patch(
780 &mut self,
781 run_id: &str,
782 step_id: &str,
783 lease_token: &str,
784 base_version: u64,
785 patch: State,
786 ) -> Result<u64> {
787 self.validate_lease(step_id, lease_token)?;
788 let run = self
789 .runs
790 .get(run_id)
791 .ok_or_else(|| RuntimeError(format!("run not found: {run_id}")))?;
792 if run.state_version != base_version {
793 return Err(RuntimeError(format!(
794 "state version conflict: expected {base_version}, got {}",
795 run.state_version
796 )));
797 }
798 let now = now_seconds();
799 let new_version = base_version + 1;
800 let session_id = self.steps.get(step_id).unwrap().session_id.clone();
801 let run = self.runs.get_mut(run_id).unwrap();
802 run.state = merge_patch(&run.state, &patch);
803 run.state_version = new_version;
804 run.status = "completed".to_string();
805 run.updated_at = now;
806 let step = self.steps.get_mut(step_id).unwrap();
807 step.status = "completed".to_string();
808 step.state_version = new_version;
809 step.checkpoint_id = Some(format!("ckpt:{run_id}:{step_id}:{}", step.attempt));
810 step.updated_at = now;
811 let mut payload = State::new();
812 payload.insert("patch".to_string(), Value::Object(patch));
813 payload.insert(
814 "state_version".to_string(),
815 Value::Number(new_version as f64),
816 );
817 self.append_event(
818 run_id,
819 Some(&session_id),
820 Some(step_id),
821 "state_patch_committed",
822 payload,
823 None,
824 Some(new_version),
825 None,
826 );
827 let mut complete_payload = State::new();
828 complete_payload.insert("step_id".to_string(), Value::String(step_id.to_string()));
829 self.append_event(
830 run_id,
831 Some(&session_id),
832 Some(step_id),
833 "step_completed",
834 complete_payload,
835 None,
836 Some(new_version),
837 None,
838 );
839 Ok(new_version)
840 }
841
842 pub fn apply_system_state_patch(
843 &mut self,
844 run_id: &str,
845 patch: State,
846 reason: &str,
847 ) -> Result<u64> {
848 let run = self
849 .runs
850 .get(run_id)
851 .ok_or_else(|| RuntimeError(format!("run not found: {run_id}")))?
852 .clone();
853 let new_version = run.state_version + 1;
854 let now = now_seconds();
855 if let Some(item) = self.runs.get_mut(run_id) {
856 item.state = merge_patch(&item.state, &patch);
857 item.state_version = new_version;
858 item.updated_at = now;
859 }
860 let mut payload = State::new();
861 payload.insert("patch".to_string(), Value::Object(patch));
862 payload.insert("reason".to_string(), Value::String(reason.to_string()));
863 payload.insert(
864 "state_version".to_string(),
865 Value::Number(new_version as f64),
866 );
867 self.append_event(
868 run_id,
869 Some(&run.session_id),
870 None,
871 "system_state_patch_applied",
872 payload,
873 None,
874 Some(new_version),
875 None,
876 );
877 Ok(new_version)
878 }
879
880 pub fn mark_retry(&mut self, run_id: &str, step_id: &str, error_type: &str, message: &str) {
881 let now = now_seconds();
882 let session_id = {
883 let step = self.steps.get_mut(step_id).unwrap();
884 step.status = "retry_scheduled".to_string();
885 step.owner = None;
886 step.lease_token = None;
887 step.lease_until = None;
888 step.last_error_type = Some(error_type.to_string());
889 step.last_error = Some(message.to_string());
890 step.updated_at = now;
891 step.session_id.clone()
892 };
893 let run = self.runs.get_mut(run_id).unwrap();
894 run.status = "retry_scheduled".to_string();
895 run.updated_at = now;
896 let mut classified = State::new();
897 classified.insert("error".to_string(), Value::String(message.to_string()));
898 classified.insert(
899 "error_type".to_string(),
900 Value::String(error_type.to_string()),
901 );
902 classified.insert("retryable".to_string(), Value::Bool(true));
903 classified.insert("source".to_string(), Value::String("agent".to_string()));
904 self.append_event(
905 run_id,
906 Some(&session_id),
907 Some(step_id),
908 "failure_classified",
909 classified,
910 None,
911 None,
912 None,
913 );
914 let mut payload = State::new();
915 payload.insert("error".to_string(), Value::String(message.to_string()));
916 payload.insert(
917 "error_type".to_string(),
918 Value::String(error_type.to_string()),
919 );
920 self.append_event(
921 run_id,
922 Some(&session_id),
923 Some(step_id),
924 "error_raised",
925 payload,
926 None,
927 None,
928 None,
929 );
930 let mut retry_payload = State::new();
931 retry_payload.insert("step_id".to_string(), Value::String(step_id.to_string()));
932 self.append_event(
933 run_id,
934 Some(&session_id),
935 Some(step_id),
936 "step_retry_scheduled",
937 retry_payload,
938 None,
939 None,
940 None,
941 );
942 }
943
944 pub fn mark_waiting_human(
945 &mut self,
946 run_id: &str,
947 step_id: &str,
948 reason: &str,
949 approval_id: &str,
950 ) {
951 let now = now_seconds();
952 let session_id = {
953 let step = self.steps.get_mut(step_id).unwrap();
954 step.status = "waiting_human".to_string();
955 step.owner = None;
956 step.lease_token = None;
957 step.lease_until = None;
958 step.last_error_type = Some("ApprovalRequired".to_string());
959 step.last_error = Some(reason.to_string());
960 step.updated_at = now;
961 step.session_id.clone()
962 };
963 let run = self.runs.get_mut(run_id).unwrap();
964 run.status = "waiting_human".to_string();
965 run.updated_at = now;
966 let mut payload = State::new();
967 payload.insert("reason".to_string(), Value::String(reason.to_string()));
968 payload.insert(
969 "approval_id".to_string(),
970 Value::String(approval_id.to_string()),
971 );
972 self.append_event(
973 run_id,
974 Some(&session_id),
975 Some(step_id),
976 "step_waiting_human",
977 payload,
978 None,
979 None,
980 None,
981 );
982 }
983
984 pub fn mark_failed(&mut self, run_id: &str, step_id: &str, error_type: &str, message: &str) {
985 let now = now_seconds();
986 let session_id = {
987 let step = self.steps.get_mut(step_id).unwrap();
988 step.status = "failed".to_string();
989 step.owner = None;
990 step.lease_token = None;
991 step.lease_until = None;
992 step.last_error_type = Some(error_type.to_string());
993 step.last_error = Some(message.to_string());
994 step.updated_at = now;
995 step.session_id.clone()
996 };
997 let run = self.runs.get_mut(run_id).unwrap();
998 run.status = "failed".to_string();
999 run.updated_at = now;
1000 let mut classified = State::new();
1001 classified.insert("error".to_string(), Value::String(message.to_string()));
1002 classified.insert(
1003 "error_type".to_string(),
1004 Value::String(error_type.to_string()),
1005 );
1006 classified.insert("retryable".to_string(), Value::Bool(false));
1007 classified.insert(
1008 "source".to_string(),
1009 Value::String(failure_source(error_type).to_string()),
1010 );
1011 self.append_event(
1012 run_id,
1013 Some(&session_id),
1014 Some(step_id),
1015 "failure_classified",
1016 classified,
1017 None,
1018 None,
1019 None,
1020 );
1021 let mut error = State::new();
1022 error.insert("error".to_string(), Value::String(message.to_string()));
1023 error.insert(
1024 "error_type".to_string(),
1025 Value::String(error_type.to_string()),
1026 );
1027 self.append_event(
1028 run_id,
1029 Some(&session_id),
1030 Some(step_id),
1031 "error_raised",
1032 error,
1033 None,
1034 None,
1035 None,
1036 );
1037 let mut payload = State::new();
1038 payload.insert("step_id".to_string(), Value::String(step_id.to_string()));
1039 payload.insert(
1040 "error_type".to_string(),
1041 Value::String(error_type.to_string()),
1042 );
1043 self.append_event(
1044 run_id,
1045 Some(&session_id),
1046 Some(step_id),
1047 "step_failed",
1048 payload,
1049 None,
1050 None,
1051 None,
1052 );
1053 }
1054
1055 pub fn append_event(
1056 &mut self,
1057 run_id: &str,
1058 session_id: Option<&str>,
1059 step_id: Option<&str>,
1060 event_type: &str,
1061 payload: State,
1062 agent_role: Option<&str>,
1063 state_version: Option<u64>,
1064 causal_token: Option<&str>,
1065 ) -> Event {
1066 let events = self.events.entry(run_id.to_string()).or_default();
1067 let payload_ref = format_state(&payload);
1068 let event = Event {
1069 event_id: new_id("evt"),
1070 run_id: run_id.to_string(),
1071 session_id: session_id.map(str::to_string),
1072 step_id: step_id.map(str::to_string),
1073 seq: events.len() as u64 + 1,
1074 event_type: event_type.to_string(),
1075 timestamp: now_seconds(),
1076 agent_role: agent_role.map(str::to_string),
1077 state_version,
1078 causal_token: causal_token.map(str::to_string),
1079 payload_hash: stable_hash(&payload_ref),
1080 payload_ref,
1081 payload,
1082 };
1083 events.push(event.clone());
1084 event
1085 }
1086
1087 pub fn reserve_ledger(&mut self, key: &str, entry: ToolLedgerEntry) -> Option<ToolLedgerEntry> {
1088 if let Some(existing) = self.tool_ledger.get(key) {
1089 return Some(existing.clone());
1090 }
1091 self.tool_ledger.insert(key.to_string(), entry);
1092 None
1093 }
1094
1095 pub fn update_ledger(
1096 &mut self,
1097 key: &str,
1098 status: &str,
1099 external_id: Option<String>,
1100 response: Option<Value>,
1101 error_type: Option<String>,
1102 ) {
1103 if let Some(entry) = self.tool_ledger.get_mut(key) {
1104 entry.status = status.to_string();
1105 entry.external_id = external_id;
1106 entry.response = response.clone();
1107 entry.response_ref = response.as_ref().map(format_value);
1108 entry.response_hash = entry.response_ref.as_ref().map(|value| stable_hash(value));
1109 entry.error_type = error_type;
1110 entry.updated_at = now_seconds();
1111 }
1112 }
1113
1114 pub fn request_approval(
1115 &mut self,
1116 approval_key: &str,
1117 run_id: &str,
1118 session_id: &str,
1119 step_id: &str,
1120 tool_name: &str,
1121 risk_level: &str,
1122 reason: &str,
1123 request_hash: &str,
1124 request_ref: &str,
1125 requested_by: &str,
1126 ) -> ApprovalRequest {
1127 if let Some(existing) = self.approval_requests.get(approval_key) {
1128 return existing.clone();
1129 }
1130 let now = now_seconds();
1131 let approval = ApprovalRequest {
1132 approval_id: new_id("approval"),
1133 approval_key: approval_key.to_string(),
1134 run_id: run_id.to_string(),
1135 session_id: session_id.to_string(),
1136 step_id: step_id.to_string(),
1137 tool_name: tool_name.to_string(),
1138 risk_level: risk_level.to_string(),
1139 status: "PENDING".to_string(),
1140 reason: reason.to_string(),
1141 request_hash: request_hash.to_string(),
1142 request_ref: request_ref.to_string(),
1143 requested_by: requested_by.to_string(),
1144 approved_by: None,
1145 decision_reason: None,
1146 created_at: now,
1147 updated_at: now,
1148 };
1149 self.approval_requests
1150 .insert(approval_key.to_string(), approval.clone());
1151 approval
1152 }
1153
1154 pub fn approval_for_key(&self, approval_key: &str) -> Option<ApprovalRequest> {
1155 self.approval_requests.get(approval_key).cloned()
1156 }
1157
1158 pub fn approval_requests(&self, run_id: &str) -> Vec<ApprovalRequest> {
1159 self.approval_requests
1160 .values()
1161 .filter(|item| run_id.is_empty() || item.run_id == run_id)
1162 .cloned()
1163 .collect()
1164 }
1165
1166 pub fn approve_request(
1167 &mut self,
1168 approval_id: &str,
1169 approver: &str,
1170 reason: &str,
1171 ) -> Result<ApprovalRequest> {
1172 self.decide_approval(approval_id, "APPROVED", approver, reason)
1173 }
1174
1175 pub fn deny_request(
1176 &mut self,
1177 approval_id: &str,
1178 approver: &str,
1179 reason: &str,
1180 ) -> Result<ApprovalRequest> {
1181 self.decide_approval(approval_id, "DENIED", approver, reason)
1182 }
1183
1184 fn decide_approval(
1185 &mut self,
1186 approval_id: &str,
1187 status: &str,
1188 approver: &str,
1189 reason: &str,
1190 ) -> Result<ApprovalRequest> {
1191 let key = self
1192 .approval_requests
1193 .iter()
1194 .find(|(_, item)| item.approval_id == approval_id)
1195 .map(|(key, _)| key.clone())
1196 .ok_or_else(|| RuntimeError(format!("approval not found: {approval_id}")))?;
1197 let mut approval = self.approval_requests.get(&key).unwrap().clone();
1198 approval.status = status.to_string();
1199 approval.approved_by = Some(approver.to_string());
1200 approval.decision_reason = Some(reason.to_string());
1201 approval.updated_at = now_seconds();
1202 self.approval_requests.insert(key, approval.clone());
1203 let mut payload = State::new();
1204 payload.insert(
1205 "approval_id".to_string(),
1206 Value::String(approval_id.to_string()),
1207 );
1208 payload.insert(
1209 "tool".to_string(),
1210 Value::String(approval.tool_name.clone()),
1211 );
1212 payload.insert("status".to_string(), Value::String(status.to_string()));
1213 self.append_event(
1214 &approval.run_id,
1215 Some(&approval.session_id),
1216 Some(&approval.step_id),
1217 "tool_approval_decided",
1218 payload,
1219 None,
1220 None,
1221 None,
1222 );
1223 if self
1224 .steps
1225 .get(&approval.step_id)
1226 .map(|step| step.status.as_str())
1227 == Some("waiting_human")
1228 {
1229 if status == "APPROVED" {
1230 let step = self.steps.get_mut(&approval.step_id).unwrap();
1231 step.status = "pending".to_string();
1232 step.owner = None;
1233 step.lease_token = None;
1234 step.lease_until = None;
1235 step.updated_at = now_seconds();
1236 let run = self.runs.get_mut(&approval.run_id).unwrap();
1237 run.status = "pending".to_string();
1238 let mut retry = State::new();
1239 retry.insert(
1240 "step_id".to_string(),
1241 Value::String(approval.step_id.clone()),
1242 );
1243 retry.insert(
1244 "reason".to_string(),
1245 Value::String("approval_granted".to_string()),
1246 );
1247 self.append_event(
1248 &approval.run_id,
1249 Some(&approval.session_id),
1250 Some(&approval.step_id),
1251 "step_retry_scheduled",
1252 retry,
1253 None,
1254 None,
1255 None,
1256 );
1257 } else if status == "DENIED" {
1258 self.mark_failed(
1259 &approval.run_id,
1260 &approval.step_id,
1261 "ApprovalDenied",
1262 reason,
1263 );
1264 }
1265 }
1266 Ok(approval)
1267 }
1268
1269 pub fn record_cost(
1270 &mut self,
1271 run_id: &str,
1272 session_id: &str,
1273 step_id: &str,
1274 category: &str,
1275 name: &str,
1276 amount: f64,
1277 unit: &str,
1278 metadata: State,
1279 ) -> String {
1280 let cost_id = new_id("cost");
1281 let record = CostRecord {
1282 cost_id: cost_id.clone(),
1283 run_id: run_id.to_string(),
1284 session_id: session_id.to_string(),
1285 step_id: step_id.to_string(),
1286 category: category.to_string(),
1287 name: name.to_string(),
1288 amount,
1289 unit: unit.to_string(),
1290 metadata,
1291 created_at: now_seconds(),
1292 };
1293 self.cost_records
1294 .entry(run_id.to_string())
1295 .or_default()
1296 .push(record);
1297 let mut payload = State::new();
1298 payload.insert("cost_id".to_string(), Value::String(cost_id.clone()));
1299 payload.insert("category".to_string(), Value::String(category.to_string()));
1300 payload.insert("name".to_string(), Value::String(name.to_string()));
1301 payload.insert("amount".to_string(), Value::Number(amount));
1302 payload.insert("unit".to_string(), Value::String(unit.to_string()));
1303 self.append_event(
1304 run_id,
1305 Some(session_id),
1306 Some(step_id),
1307 "cost_recorded",
1308 payload,
1309 None,
1310 None,
1311 None,
1312 );
1313 cost_id
1314 }
1315
1316 pub fn cost_records(&self, run_id: &str) -> Vec<CostRecord> {
1317 self.cost_records.get(run_id).cloned().unwrap_or_default()
1318 }
1319
1320 pub fn cost_summary(&self, run_id: &str) -> CostSummary {
1321 let mut summary = CostSummary::default();
1322 for record in self.cost_records(run_id) {
1323 add_cost(&mut summary, &record);
1324 }
1325 summary
1326 }
1327
1328 pub fn create_artifact(
1329 &mut self,
1330 run_id: &str,
1331 step_id: Option<&str>,
1332 name: &str,
1333 content: State,
1334 metadata: State,
1335 ) -> Artifact {
1336 let blob_ref = format_state(&content);
1337 let artifact = Artifact {
1338 artifact_id: new_id("art"),
1339 run_id: run_id.to_string(),
1340 step_id: step_id.map(str::to_string),
1341 name: name.to_string(),
1342 blob_hash: format!("sha256:{}", stable_hash(&blob_ref)),
1343 blob_ref,
1344 metadata,
1345 created_at: now_seconds(),
1346 };
1347 self.artifacts
1348 .entry(run_id.to_string())
1349 .or_default()
1350 .push(artifact.clone());
1351 artifact
1352 }
1353
1354 pub fn artifacts(&self, run_id: &str) -> Vec<Artifact> {
1355 let mut items = self.artifacts.get(run_id).cloned().unwrap_or_default();
1356 items.sort_by(|a, b| {
1357 a.created_at
1358 .partial_cmp(&b.created_at)
1359 .unwrap()
1360 .then(a.artifact_id.cmp(&b.artifact_id))
1361 });
1362 items
1363 }
1364
1365 pub fn validate_lease(&self, step_id: &str, lease_token: &str) -> Result<()> {
1366 let step = self
1367 .steps
1368 .get(step_id)
1369 .ok_or_else(|| RuntimeError(format!("step not found: {step_id}")))?;
1370 if step.status != "running" || step.lease_token.as_deref() != Some(lease_token) {
1371 return Err(RuntimeError("invalid or stale lease token".to_string()));
1372 }
1373 if step.lease_until.is_some_and(|until| until <= now_seconds()) {
1374 return Err(RuntimeError("lease expired".to_string()));
1375 }
1376 Ok(())
1377 }
1378
1379 pub fn final_state(&self, run_id: &str) -> Result<State> {
1380 Ok(self.load_state(run_id)?.0)
1381 }
1382
1383 pub fn run(&self, run_id: &str) -> Result<Run> {
1384 self.runs
1385 .get(run_id)
1386 .cloned()
1387 .ok_or_else(|| RuntimeError(format!("run not found: {run_id}")))
1388 }
1389
1390 pub fn steps(&self, run_id: &str) -> Vec<Step> {
1391 self.steps
1392 .values()
1393 .filter(|step| step.run_id == run_id)
1394 .cloned()
1395 .collect()
1396 }
1397
1398 pub fn events(&self, run_id: &str) -> Vec<Event> {
1399 self.events.get(run_id).cloned().unwrap_or_default()
1400 }
1401
1402 pub fn ledger(&self, run_id: &str) -> Vec<ToolLedgerEntry> {
1403 self.tool_ledger
1404 .values()
1405 .filter(|entry| entry.run_id == run_id)
1406 .cloned()
1407 .collect()
1408 }
1409}
1410
1411pub struct ToolRegistry {
1412 tools: HashMap<String, ToolSpec>,
1413}
1414
1415impl ToolRegistry {
1416 pub fn new() -> Self {
1417 Self {
1418 tools: HashMap::new(),
1419 }
1420 }
1421
1422 pub fn register(&mut self, spec: ToolSpec) {
1423 self.tools.insert(spec.name.clone(), spec);
1424 }
1425
1426 pub fn get(&self, name: &str) -> Result<&ToolSpec> {
1427 self.tools
1428 .get(name)
1429 .ok_or_else(|| RuntimeError(format!("tool not registered: {name}")))
1430 }
1431}
1432
1433impl Default for ToolRegistry {
1434 fn default() -> Self {
1435 Self::new()
1436 }
1437}
1438
1439pub struct Runtime {
1440 pub store: MemoryStore,
1441 pub registry: ToolRegistry,
1442 pub budget: BudgetLimits,
1443 pub sandbox: Box<dyn SandboxExecutor>,
1444}
1445
1446impl Runtime {
1447 pub fn new() -> Self {
1448 Self {
1449 store: MemoryStore::new(),
1450 registry: ToolRegistry::new(),
1451 budget: BudgetLimits::default(),
1452 sandbox: Box::new(DisabledSandboxExecutor),
1453 }
1454 }
1455
1456 pub fn register_tool(&mut self, spec: ToolSpec) {
1457 self.registry.register(spec);
1458 }
1459
1460 pub fn set_budget(&mut self, budget: BudgetLimits) {
1461 self.budget = budget;
1462 }
1463
1464 pub fn set_sandbox(&mut self, sandbox: Box<dyn SandboxExecutor>) {
1465 self.sandbox = sandbox;
1466 }
1467
1468 pub fn create_run(&mut self, initial_state: State) -> (String, String) {
1469 self.store.create_run(initial_state)
1470 }
1471
1472 pub fn run_once(
1473 &mut self,
1474 run_id: &str,
1475 worker_id: &str,
1476 agent_role: &str,
1477 lease_seconds: f64,
1478 agent: AgentFunc,
1479 ) -> Result<bool> {
1480 let claim = match self.store.claim_step(worker_id, run_id, lease_seconds) {
1481 Ok(claim) => claim,
1482 Err(err) if err.0.contains("no runnable step") => return Ok(false),
1483 Err(err) => return Err(err),
1484 };
1485 let (state, version, session_id) = self.store.load_state(&claim.run_id)?;
1486 let mut payload = State::new();
1487 payload.insert(
1488 "agent_role".to_string(),
1489 Value::String(agent_role.to_string()),
1490 );
1491 payload.insert("attempt".to_string(), Value::Number(claim.attempt as f64));
1492 self.store.append_event(
1493 &claim.run_id,
1494 Some(&session_id),
1495 Some(&claim.step_id),
1496 "agent_started",
1497 payload,
1498 Some(agent_role),
1499 Some(version),
1500 None,
1501 );
1502 let mut ctx = AgentContext {
1503 run_id: claim.run_id.clone(),
1504 session_id,
1505 step_id: claim.step_id.clone(),
1506 agent_role: agent_role.to_string(),
1507 lease_token: claim.lease_token.clone(),
1508 attempt: claim.attempt,
1509 state_version: version,
1510 pending_patch: State::new(),
1511 };
1512 match agent(&mut ctx, state) {
1513 Ok(()) => {
1514 self.store.commit_state_patch(
1515 &claim.run_id,
1516 &claim.step_id,
1517 &claim.lease_token,
1518 version,
1519 ctx.pending_patch,
1520 )?;
1521 Ok(true)
1522 }
1523 Err(err) if err.0 == "retryable" => {
1524 self.store.mark_retry(
1525 &claim.run_id,
1526 &claim.step_id,
1527 "RetryableAgentError",
1528 "retryable",
1529 );
1530 Ok(false)
1531 }
1532 Err(err) if err.0.starts_with("approval required:") => {
1533 let approval_id = err.0.trim_start_matches("approval required:");
1534 self.store
1535 .mark_waiting_human(&claim.run_id, &claim.step_id, &err.0, approval_id);
1536 Ok(false)
1537 }
1538 Err(err) => {
1539 self.store.mark_failed(
1540 &claim.run_id,
1541 &claim.step_id,
1542 classify_runtime_error(&err.0),
1543 &err.0,
1544 );
1545 Err(err)
1546 }
1547 }
1548 }
1549
1550 pub fn call_tool(&mut self, ctx: &AgentContext, tool_name: &str, args: State) -> Result<Value> {
1551 let (
1552 version,
1553 side_effect,
1554 risk_level,
1555 idempotency_required,
1556 approval_required,
1557 sandbox_required,
1558 sandbox_executor,
1559 sandbox_policy,
1560 input_schema,
1561 output_schema,
1562 ) = {
1563 let spec = self.registry.get(tool_name)?;
1564 (
1565 spec.version.clone(),
1566 spec.side_effect.clone(),
1567 spec.risk_level.clone(),
1568 spec.idempotency_required,
1569 spec.approval_required,
1570 spec.sandbox_required,
1571 spec.sandbox_executor.clone(),
1572 spec.sandbox_policy.clone(),
1573 spec.input_schema.clone(),
1574 spec.output_schema.clone(),
1575 )
1576 };
1577 let mut request = State::new();
1578 request.insert("tool".to_string(), Value::String(tool_name.to_string()));
1579 request.insert("args".to_string(), Value::Object(args.clone()));
1580 let request_ref = format_state(&request);
1581 let request_hash = stable_hash(&request_ref);
1582 let causal_token = format!(
1583 "{}:{}:{}:{}",
1584 ctx.run_id, ctx.step_id, ctx.attempt, ctx.state_version
1585 );
1586 let idempotency_key = format!(
1587 "{}:{}:{}:{}",
1588 ctx.run_id, ctx.step_id, tool_name, request_hash
1589 );
1590 let managed = side_effect != "none" || idempotency_required;
1591 self.store.append_event(
1592 &ctx.run_id,
1593 Some(&ctx.session_id),
1594 Some(&ctx.step_id),
1595 "tool_call_requested",
1596 request,
1597 Some(&ctx.agent_role),
1598 Some(ctx.state_version),
1599 Some(&causal_token),
1600 );
1601 if let Some(schema) = input_schema.as_ref() {
1602 if let Err(err) = validate_tool_schema(schema, &Value::Object(args.clone()), "$arg") {
1603 let mut failed = State::new();
1604 failed.insert("tool".to_string(), Value::String(tool_name.to_string()));
1605 failed.insert("error".to_string(), Value::String(err.0.clone()));
1606 failed.insert(
1607 "phase".to_string(),
1608 Value::String("input_validation".to_string()),
1609 );
1610 self.store.append_event(
1611 &ctx.run_id,
1612 Some(&ctx.session_id),
1613 Some(&ctx.step_id),
1614 "tool_call_failed",
1615 failed,
1616 Some(&ctx.agent_role),
1617 Some(ctx.state_version),
1618 Some(&causal_token),
1619 );
1620 return Err(err);
1621 }
1622 }
1623
1624 let mut allowed = !is_high_risk(&risk_level);
1625 let mut reason = if allowed {
1626 "default allow for low/medium risk in local runtime".to_string()
1627 } else {
1628 "high-risk tool denied by default".to_string()
1629 };
1630 if let Some(approval) = self.store.approval_for_key(&idempotency_key) {
1631 if approval.status == "DENIED" {
1632 reason = format!("approval denied for tool {tool_name}");
1633 self.record_permission(ctx, tool_name, false, &reason, &causal_token);
1634 return Err(RuntimeError(reason));
1635 }
1636 if approval.status == "APPROVED" {
1637 allowed = true;
1638 reason = format!(
1639 "approved by {}",
1640 approval
1641 .approved_by
1642 .unwrap_or_else(|| "operator".to_string())
1643 );
1644 }
1645 } else if approval_required {
1646 let approval = self.store.request_approval(
1647 &idempotency_key,
1648 &ctx.run_id,
1649 &ctx.session_id,
1650 &ctx.step_id,
1651 tool_name,
1652 &risk_level,
1653 "tool requires approval",
1654 &request_hash,
1655 &request_ref,
1656 &ctx.agent_role,
1657 );
1658 self.record_permission(ctx, tool_name, false, "approval required", &causal_token);
1659 let mut payload = State::new();
1660 payload.insert("tool".to_string(), Value::String(tool_name.to_string()));
1661 payload.insert(
1662 "approval_id".to_string(),
1663 Value::String(approval.approval_id.clone()),
1664 );
1665 payload.insert(
1666 "approval_key".to_string(),
1667 Value::String(idempotency_key.clone()),
1668 );
1669 payload.insert("risk_level".to_string(), Value::String(risk_level));
1670 self.store.append_event(
1671 &ctx.run_id,
1672 Some(&ctx.session_id),
1673 Some(&ctx.step_id),
1674 "tool_approval_required",
1675 payload,
1676 Some(&ctx.agent_role),
1677 Some(ctx.state_version),
1678 Some(&causal_token),
1679 );
1680 return Err(RuntimeError(format!(
1681 "approval required:{}",
1682 approval.approval_id
1683 )));
1684 }
1685 self.record_permission(ctx, tool_name, allowed, &reason, &causal_token);
1686 if !allowed {
1687 return Err(RuntimeError(reason));
1688 }
1689 if let Some(max) = self.budget.max_tool_calls {
1690 let used = self.store.cost_summary(&ctx.run_id).tool_calls;
1691 if used >= max {
1692 let message = format!("tool call budget exceeded: {used}/{max}");
1693 let mut payload = State::new();
1694 payload.insert("category".to_string(), Value::String("tool".to_string()));
1695 payload.insert("tool".to_string(), Value::String(tool_name.to_string()));
1696 payload.insert("error".to_string(), Value::String(message.clone()));
1697 self.store.append_event(
1698 &ctx.run_id,
1699 Some(&ctx.session_id),
1700 Some(&ctx.step_id),
1701 "budget_check_failed",
1702 payload,
1703 Some(&ctx.agent_role),
1704 Some(ctx.state_version),
1705 Some(&causal_token),
1706 );
1707 return Err(RuntimeError(message));
1708 }
1709 }
1710 if managed {
1711 let entry = ToolLedgerEntry {
1712 ledger_id: new_id("ledger"),
1713 run_id: ctx.run_id.clone(),
1714 session_id: ctx.session_id.clone(),
1715 step_id: ctx.step_id.clone(),
1716 tool_name: tool_name.to_string(),
1717 tool_version: version,
1718 tool_call_id: new_id("toolcall"),
1719 idempotency_key: idempotency_key.clone(),
1720 causal_token: causal_token.clone(),
1721 request_hash: request_hash.clone(),
1722 request_ref: request_ref.clone(),
1723 status: "RESERVED".to_string(),
1724 external_id: None,
1725 response_hash: None,
1726 response_ref: None,
1727 error_type: None,
1728 response: None,
1729 created_at: now_seconds(),
1730 updated_at: now_seconds(),
1731 };
1732 if let Some(existing) = self.store.reserve_ledger(&idempotency_key, entry) {
1733 if existing.status == "SUCCEEDED" {
1734 let mut payload = State::new();
1735 payload.insert("tool".to_string(), Value::String(tool_name.to_string()));
1736 payload.insert("replayed_from_ledger".to_string(), Value::Bool(true));
1737 self.store.append_event(
1738 &ctx.run_id,
1739 Some(&ctx.session_id),
1740 Some(&ctx.step_id),
1741 "tool_call_completed",
1742 payload,
1743 Some(&ctx.agent_role),
1744 Some(ctx.state_version),
1745 Some(&causal_token),
1746 );
1747 self.store.record_cost(
1748 &ctx.run_id,
1749 &ctx.session_id,
1750 &ctx.step_id,
1751 "tool",
1752 tool_name,
1753 1.0,
1754 "call",
1755 State::new(),
1756 );
1757 return existing
1758 .response
1759 .ok_or_else(|| RuntimeError("missing ledger response".to_string()));
1760 }
1761 if existing.status == "PENDING_VERIFICATION" {
1762 return Err(RuntimeError(
1763 "tool side effect pending verification".to_string(),
1764 ));
1765 }
1766 if existing.status == "FAILED_NO_EFFECT" {
1767 self.store
1768 .update_ledger(&idempotency_key, "RUNNING", None, None, None);
1769 } else {
1770 return Err(RuntimeError(
1771 "tool side effect already in progress".to_string(),
1772 ));
1773 }
1774 } else {
1775 self.store
1776 .update_ledger(&idempotency_key, "RUNNING", None, None, None);
1777 }
1778 }
1779 if sandbox_required {
1780 let executor = if sandbox_executor.is_empty() { "default".to_string() } else { sandbox_executor.clone() };
1781 let network = match sandbox_policy.get("network") {
1782 Some(Value::String(value)) => value.clone(),
1783 _ => "deny".to_string(),
1784 };
1785 let filesystem = match sandbox_policy.get("filesystem") {
1786 Some(Value::String(value)) => value.clone(),
1787 _ => "read-only".to_string(),
1788 };
1789 let timeout_seconds = match sandbox_policy.get("timeout_seconds") {
1790 Some(Value::Number(value)) if *value > 0.0 => *value as u64,
1791 _ => 30,
1792 };
1793 let policy = SandboxPolicy {
1794 tool_name: tool_name.to_string(),
1795 run_id: ctx.run_id.clone(),
1796 step_id: ctx.step_id.clone(),
1797 executor: executor.clone(),
1798 network,
1799 filesystem,
1800 timeout_seconds,
1801 extra: sandbox_policy.clone(),
1802 };
1803 let mut sandbox_payload = State::new();
1804 sandbox_payload.insert("tool_name".to_string(), Value::String(tool_name.to_string()));
1805 sandbox_payload.insert("executor".to_string(), Value::String(executor));
1806 sandbox_payload.insert("network".to_string(), Value::String(policy.network.clone()));
1807 sandbox_payload.insert("filesystem".to_string(), Value::String(policy.filesystem.clone()));
1808 sandbox_payload.insert("timeout_seconds".to_string(), Value::Number(policy.timeout_seconds as f64));
1809 self.store.append_event(
1810 &ctx.run_id,
1811 Some(&ctx.session_id),
1812 Some(&ctx.step_id),
1813 "sandbox_started",
1814 sandbox_payload,
1815 Some(&ctx.agent_role),
1816 Some(ctx.state_version),
1817 Some(&causal_token),
1818 );
1819 let result = self.sandbox.run_tool(args, &policy);
1820 let mut completed = State::new();
1821 completed.insert("ok".to_string(), Value::Bool(result.ok));
1822 completed.insert("metadata".to_string(), Value::Object(result.metadata.clone()));
1823 if let Some(error) = &result.error {
1824 completed.insert("error".to_string(), Value::String(error.clone()));
1825 }
1826 self.store.append_event(
1827 &ctx.run_id,
1828 Some(&ctx.session_id),
1829 Some(&ctx.step_id),
1830 "sandbox_completed",
1831 completed,
1832 Some(&ctx.agent_role),
1833 Some(ctx.state_version),
1834 Some(&causal_token),
1835 );
1836 if !result.ok {
1837 let message = result.error.unwrap_or_else(|| "sandboxed tool failed".to_string());
1838 let mut failed = State::new();
1839 failed.insert("tool".to_string(), Value::String(tool_name.to_string()));
1840 failed.insert("error".to_string(), Value::String(message.clone()));
1841 self.store.append_event(
1842 &ctx.run_id,
1843 Some(&ctx.session_id),
1844 Some(&ctx.step_id),
1845 "tool_call_failed",
1846 failed,
1847 Some(&ctx.agent_role),
1848 Some(ctx.state_version),
1849 Some(&causal_token),
1850 );
1851 return Err(RuntimeError(message));
1852 }
1853 let value = result.output;
1854 if let Some(schema) = output_schema.as_ref() {
1855 validate_tool_schema(schema, &value, "$result")?;
1856 }
1857 if managed {
1858 self.store.update_ledger(&idempotency_key, "SUCCEEDED", external_id_from_value(Some(&value)), Some(value.clone()), None);
1859 }
1860 let mut payload = State::new();
1861 payload.insert("tool".to_string(), Value::String(tool_name.to_string()));
1862 payload.insert("idempotency_key".to_string(), Value::String(idempotency_key));
1863 self.store.append_event(
1864 &ctx.run_id,
1865 Some(&ctx.session_id),
1866 Some(&ctx.step_id),
1867 "tool_call_completed",
1868 payload,
1869 Some(&ctx.agent_role),
1870 Some(ctx.state_version),
1871 Some(&causal_token),
1872 );
1873 self.store.record_cost(
1874 &ctx.run_id,
1875 &ctx.session_id,
1876 &ctx.step_id,
1877 "tool",
1878 tool_name,
1879 1.0,
1880 "call",
1881 State::new(),
1882 );
1883 return Ok(value);
1884 }
1885 let result = {
1886 let spec = self.registry.get(tool_name)?;
1887 (spec.func)(args)
1888 };
1889 match result {
1890 Ok(value) => {
1891 if let Some(schema) = output_schema.as_ref() {
1892 if let Err(err) = validate_tool_schema(schema, &value, "$result") {
1893 if managed {
1894 self.store.update_ledger(
1895 &idempotency_key,
1896 "PENDING_VERIFICATION",
1897 None,
1898 None,
1899 Some("ToolOutputValidationError".to_string()),
1900 );
1901 }
1902 let mut failed = State::new();
1903 failed.insert("tool".to_string(), Value::String(tool_name.to_string()));
1904 failed.insert("error".to_string(), Value::String(err.0.clone()));
1905 failed.insert(
1906 "phase".to_string(),
1907 Value::String("output_validation".to_string()),
1908 );
1909 self.store.append_event(
1910 &ctx.run_id,
1911 Some(&ctx.session_id),
1912 Some(&ctx.step_id),
1913 "tool_call_failed",
1914 failed,
1915 Some(&ctx.agent_role),
1916 Some(ctx.state_version),
1917 Some(&causal_token),
1918 );
1919 if managed {
1920 return Err(RuntimeError("retryable".to_string()));
1921 }
1922 return Err(err);
1923 }
1924 }
1925 if managed {
1926 self.store.update_ledger(
1927 &idempotency_key,
1928 "SUCCEEDED",
1929 external_id_from_value(Some(&value)),
1930 Some(value.clone()),
1931 None,
1932 );
1933 }
1934 let mut payload = State::new();
1935 payload.insert("tool".to_string(), Value::String(tool_name.to_string()));
1936 payload.insert(
1937 "idempotency_key".to_string(),
1938 Value::String(idempotency_key),
1939 );
1940 self.store.append_event(
1941 &ctx.run_id,
1942 Some(&ctx.session_id),
1943 Some(&ctx.step_id),
1944 "tool_call_completed",
1945 payload,
1946 Some(&ctx.agent_role),
1947 Some(ctx.state_version),
1948 Some(&causal_token),
1949 );
1950 self.store.record_cost(
1951 &ctx.run_id,
1952 &ctx.session_id,
1953 &ctx.step_id,
1954 "tool",
1955 tool_name,
1956 1.0,
1957 "call",
1958 State::new(),
1959 );
1960 Ok(value)
1961 }
1962 Err(err) => {
1963 if managed {
1964 self.store.update_ledger(
1965 &idempotency_key,
1966 "PENDING_VERIFICATION",
1967 None,
1968 None,
1969 Some("RuntimeError".to_string()),
1970 );
1971 }
1972 let mut payload = State::new();
1973 payload.insert("tool".to_string(), Value::String(tool_name.to_string()));
1974 payload.insert("error".to_string(), Value::String(err.0.clone()));
1975 self.store.append_event(
1976 &ctx.run_id,
1977 Some(&ctx.session_id),
1978 Some(&ctx.step_id),
1979 "tool_call_failed",
1980 payload,
1981 Some(&ctx.agent_role),
1982 Some(ctx.state_version),
1983 Some(&causal_token),
1984 );
1985 if managed {
1986 Err(RuntimeError("retryable".to_string()))
1987 } else {
1988 Err(err)
1989 }
1990 }
1991 }
1992 }
1993
1994 fn record_permission(
1995 &mut self,
1996 ctx: &AgentContext,
1997 tool_name: &str,
1998 allowed: bool,
1999 reason: &str,
2000 causal_token: &str,
2001 ) {
2002 let mut permission = State::new();
2003 permission.insert("tool".to_string(), Value::String(tool_name.to_string()));
2004 permission.insert("allowed".to_string(), Value::Bool(allowed));
2005 permission.insert("reason".to_string(), Value::String(reason.to_string()));
2006 self.store.append_event(
2007 &ctx.run_id,
2008 Some(&ctx.session_id),
2009 Some(&ctx.step_id),
2010 "tool_permission_decided",
2011 permission,
2012 Some(&ctx.agent_role),
2013 Some(ctx.state_version),
2014 Some(causal_token),
2015 );
2016 }
2017
2018 pub fn create_artifact(
2019 &mut self,
2020 ctx: &AgentContext,
2021 name: &str,
2022 content: State,
2023 metadata: State,
2024 ) -> Result<String> {
2025 let artifact =
2026 self.store
2027 .create_artifact(&ctx.run_id, Some(&ctx.step_id), name, content, metadata);
2028 let mut payload = State::new();
2029 payload.insert(
2030 "artifact_id".to_string(),
2031 Value::String(artifact.artifact_id.clone()),
2032 );
2033 payload.insert("name".to_string(), Value::String(name.to_string()));
2034 self.store.append_event(
2035 &ctx.run_id,
2036 Some(&ctx.session_id),
2037 Some(&ctx.step_id),
2038 "artifact_created",
2039 payload,
2040 Some(&ctx.agent_role),
2041 Some(ctx.state_version),
2042 None,
2043 );
2044 Ok(artifact.artifact_id)
2045 }
2046
2047 pub fn create_media_artifact(
2048 &mut self,
2049 ctx: &AgentContext,
2050 name: &str,
2051 kind: &str,
2052 options: MediaArtifactOptions,
2053 ) -> Result<String> {
2054 if !is_media_kind(kind) {
2055 return Err(RuntimeError(format!("unsupported media kind: {kind}")));
2056 }
2057 let mut media_metadata = options.media_metadata.clone();
2058 media_metadata.insert(
2059 "schema_version".to_string(),
2060 Value::String(MEDIA_SCHEMA_VERSION.to_string()),
2061 );
2062 media_metadata.insert("kind".to_string(), Value::String(kind.to_string()));
2063 let mut content = State::new();
2064 content.insert(
2065 "schema_version".to_string(),
2066 Value::String(MEDIA_SCHEMA_VERSION.to_string()),
2067 );
2068 content.insert("kind".to_string(), Value::String(kind.to_string()));
2069 if let Some(uri) = &options.uri {
2070 content.insert("uri".to_string(), Value::String(uri.clone()));
2071 }
2072 if let Some(content_ref) = &options.content_ref {
2073 content.insert(
2074 "content_ref".to_string(),
2075 Value::String(content_ref.clone()),
2076 );
2077 }
2078 content.insert(
2079 "metadata".to_string(),
2080 Value::Object(media_metadata.clone()),
2081 );
2082 if !options.lineage.is_empty() {
2083 content.insert(
2084 "lineage".to_string(),
2085 Value::Object(options.lineage.clone()),
2086 );
2087 }
2088 if !options.derived_outputs.is_empty() {
2089 content.insert(
2090 "derived_outputs".to_string(),
2091 Value::Object(options.derived_outputs.clone()),
2092 );
2093 }
2094 let mut media = State::new();
2095 media.insert(
2096 "schema_version".to_string(),
2097 Value::String(MEDIA_SCHEMA_VERSION.to_string()),
2098 );
2099 media.insert("kind".to_string(), Value::String(kind.to_string()));
2100 if let Some(uri) = options.uri {
2101 media.insert("uri".to_string(), Value::String(uri));
2102 }
2103 if let Some(content_ref) = options.content_ref {
2104 media.insert("content_ref".to_string(), Value::String(content_ref));
2105 }
2106 media.insert("metadata".to_string(), Value::Object(media_metadata));
2107 media.insert("lineage".to_string(), Value::Object(options.lineage));
2108 let mut artifact_metadata = options.metadata;
2109 artifact_metadata.insert("agentledger_media".to_string(), Value::Object(media));
2110 self.create_artifact(ctx, name, content, artifact_metadata)
2111 }
2112
2113 pub fn create_stream_checkpoint(
2114 &mut self,
2115 ctx: &AgentContext,
2116 name: &str,
2117 options: StreamCheckpointOptions,
2118 ) -> Result<String> {
2119 if options.stream_id.is_empty() || options.consumer_id.is_empty() {
2120 return Err(RuntimeError(
2121 "stream_id and consumer_id are required".to_string(),
2122 ));
2123 }
2124 let chunk = options.chunk.map(stream_chunk_to_state);
2125 let mut content = State::new();
2126 content.insert(
2127 "schema_version".to_string(),
2128 Value::String(STREAM_SCHEMA_VERSION.to_string()),
2129 );
2130 content.insert(
2131 "stream_id".to_string(),
2132 Value::String(options.stream_id.clone()),
2133 );
2134 content.insert(
2135 "consumer_id".to_string(),
2136 Value::String(options.consumer_id.clone()),
2137 );
2138 content.insert("offset".to_string(), options.offset.clone());
2139 if let Some(watermark) = &options.watermark {
2140 content.insert("watermark".to_string(), watermark.clone());
2141 }
2142 if let Some(chunk_state) = &chunk {
2143 content.insert("chunk".to_string(), Value::Object(chunk_state.clone()));
2144 }
2145 if let Some(partial_ref) = &options.partial_result_ref {
2146 content.insert(
2147 "partial_result_ref".to_string(),
2148 Value::String(partial_ref.clone()),
2149 );
2150 }
2151 if !options.backpressure.is_empty() {
2152 content.insert(
2153 "backpressure".to_string(),
2154 Value::Object(options.backpressure.clone()),
2155 );
2156 }
2157 if !options.metadata.is_empty() {
2158 content.insert("metadata".to_string(), Value::Object(options.metadata));
2159 }
2160 let mut stream = State::new();
2161 stream.insert(
2162 "schema_version".to_string(),
2163 Value::String(STREAM_SCHEMA_VERSION.to_string()),
2164 );
2165 stream.insert("stream_id".to_string(), Value::String(options.stream_id));
2166 stream.insert(
2167 "consumer_id".to_string(),
2168 Value::String(options.consumer_id),
2169 );
2170 stream.insert("offset".to_string(), options.offset);
2171 if let Some(watermark) = options.watermark {
2172 stream.insert("watermark".to_string(), watermark);
2173 }
2174 if let Some(chunk_state) = chunk {
2175 stream.insert("chunk".to_string(), Value::Object(chunk_state));
2176 }
2177 if let Some(partial_ref) = options.partial_result_ref {
2178 stream.insert("partial_result_ref".to_string(), Value::String(partial_ref));
2179 }
2180 if !options.backpressure.is_empty() {
2181 stream.insert(
2182 "backpressure".to_string(),
2183 Value::Object(options.backpressure),
2184 );
2185 }
2186 let mut artifact_metadata = State::new();
2187 artifact_metadata.insert("agentledger_stream".to_string(), Value::Object(stream));
2188 self.create_artifact(ctx, name, content, artifact_metadata)
2189 }
2190
2191 pub fn record_model_call(
2192 &mut self,
2193 ctx: &AgentContext,
2194 model: &str,
2195 input_tokens: f64,
2196 output_tokens: f64,
2197 total_usd: f64,
2198 ) -> Result<()> {
2199 let mut usage = State::new();
2200 usage.insert("input_tokens".to_string(), Value::Number(input_tokens));
2201 usage.insert("output_tokens".to_string(), Value::Number(output_tokens));
2202 self.record_model_call_evidence(ctx, "custom", model, State::new(), State::new(), usage, total_usd, State::new())
2203 }
2204
2205 pub fn record_model_call_evidence(
2206 &mut self,
2207 ctx: &AgentContext,
2208 provider: &str,
2209 model: &str,
2210 request: State,
2211 response: State,
2212 usage: State,
2213 total_usd: f64,
2214 metadata: State,
2215 ) -> Result<()> {
2216 let tokens = usage_total_tokens(&usage);
2217 if let Some(max) = self.budget.max_model_tokens {
2218 let used = self.store.cost_summary(&ctx.run_id).model_tokens;
2219 if used + tokens > max {
2220 let message = format!("model token budget exceeded: {used}+{tokens}/{max}");
2221 let mut payload = State::new();
2222 payload.insert("category".to_string(), Value::String("model".to_string()));
2223 payload.insert("provider".to_string(), Value::String(provider.to_string()));
2224 payload.insert("model".to_string(), Value::String(model.to_string()));
2225 payload.insert("error".to_string(), Value::String(message.clone()));
2226 self.store.append_event(
2227 &ctx.run_id,
2228 Some(&ctx.session_id),
2229 Some(&ctx.step_id),
2230 "budget_check_failed",
2231 payload,
2232 Some(&ctx.agent_role),
2233 Some(ctx.state_version),
2234 None,
2235 );
2236 return Err(RuntimeError(message));
2237 }
2238 }
2239 let mut request_payload = State::new();
2240 request_payload.insert("schema_version".to_string(), Value::String(MODEL_EVIDENCE_SCHEMA_VERSION.to_string()));
2241 request_payload.insert("provider".to_string(), Value::String(provider.to_string()));
2242 request_payload.insert("model".to_string(), Value::String(model.to_string()));
2243 request_payload.insert("request".to_string(), Value::Object(request));
2244 request_payload.insert("metadata".to_string(), Value::Object(metadata.clone()));
2245 self.store.append_event(
2246 &ctx.run_id,
2247 Some(&ctx.session_id),
2248 Some(&ctx.step_id),
2249 "model_call_requested",
2250 request_payload,
2251 Some(&ctx.agent_role),
2252 Some(ctx.state_version),
2253 None,
2254 );
2255 let mut response_payload = State::new();
2256 response_payload.insert("schema_version".to_string(), Value::String(MODEL_EVIDENCE_SCHEMA_VERSION.to_string()));
2257 response_payload.insert("provider".to_string(), Value::String(provider.to_string()));
2258 response_payload.insert("model".to_string(), Value::String(model.to_string()));
2259 response_payload.insert("response".to_string(), Value::Object(response));
2260 response_payload.insert("usage".to_string(), Value::Object(usage.clone()));
2261 response_payload.insert("total_usd".to_string(), Value::Number(total_usd));
2262 response_payload.insert("metadata".to_string(), Value::Object(metadata));
2263 self.store.append_event(
2264 &ctx.run_id,
2265 Some(&ctx.session_id),
2266 Some(&ctx.step_id),
2267 "model_call_completed",
2268 response_payload,
2269 Some(&ctx.agent_role),
2270 Some(ctx.state_version),
2271 None,
2272 );
2273 self.record_model_costs(ctx, provider, model, &usage, total_usd);
2274 Ok(())
2275 }
2276
2277 pub fn record_model_failure(
2278 &mut self,
2279 ctx: &AgentContext,
2280 provider: &str,
2281 model: &str,
2282 error_type: &str,
2283 message: &str,
2284 retryable: Option<bool>,
2285 request: State,
2286 usage: State,
2287 total_usd: f64,
2288 metadata: State,
2289 ) -> Result<()> {
2290 let mut payload = State::new();
2291 payload.insert("schema_version".to_string(), Value::String(MODEL_EVIDENCE_SCHEMA_VERSION.to_string()));
2292 payload.insert("provider".to_string(), Value::String(provider.to_string()));
2293 payload.insert("model".to_string(), Value::String(model.to_string()));
2294 payload.insert("error_type".to_string(), Value::String(error_type.to_string()));
2295 payload.insert("error".to_string(), Value::String(message.to_string()));
2296 if let Some(value) = retryable {
2297 payload.insert("retryable".to_string(), Value::Bool(value));
2298 }
2299 payload.insert("request".to_string(), Value::Object(request));
2300 payload.insert("usage".to_string(), Value::Object(usage.clone()));
2301 payload.insert("total_usd".to_string(), Value::Number(total_usd));
2302 payload.insert("metadata".to_string(), Value::Object(metadata));
2303 self.store.append_event(
2304 &ctx.run_id,
2305 Some(&ctx.session_id),
2306 Some(&ctx.step_id),
2307 "model_call_failed",
2308 payload,
2309 Some(&ctx.agent_role),
2310 Some(ctx.state_version),
2311 None,
2312 );
2313 self.record_model_costs(ctx, provider, model, &usage, total_usd);
2314 Ok(())
2315 }
2316
2317 pub fn record_tool_call_proposal(
2318 &mut self,
2319 ctx: &AgentContext,
2320 tool_name: &str,
2321 arguments: State,
2322 provider: Option<&str>,
2323 model: Option<&str>,
2324 model_call_ref: Option<&str>,
2325 confidence: Option<f64>,
2326 reason: Option<&str>,
2327 metadata: State,
2328 ) {
2329 let mut payload = State::new();
2330 payload.insert("schema_version".to_string(), Value::String(MODEL_EVIDENCE_SCHEMA_VERSION.to_string()));
2331 payload.insert("tool".to_string(), Value::String(tool_name.to_string()));
2332 payload.insert("args".to_string(), Value::Object(arguments));
2333 if let Some(value) = provider {
2334 payload.insert("provider".to_string(), Value::String(value.to_string()));
2335 }
2336 if let Some(value) = model {
2337 payload.insert("model".to_string(), Value::String(value.to_string()));
2338 }
2339 if let Some(value) = model_call_ref {
2340 payload.insert("model_call_ref".to_string(), Value::String(value.to_string()));
2341 }
2342 if let Some(value) = confidence {
2343 payload.insert("confidence".to_string(), Value::Number(value));
2344 }
2345 if let Some(value) = reason {
2346 payload.insert("reason".to_string(), Value::String(value.to_string()));
2347 }
2348 payload.insert("metadata".to_string(), Value::Object(metadata));
2349 self.store.append_event(
2350 &ctx.run_id,
2351 Some(&ctx.session_id),
2352 Some(&ctx.step_id),
2353 "tool_call_proposed",
2354 payload,
2355 Some(&ctx.agent_role),
2356 Some(ctx.state_version),
2357 None,
2358 );
2359 }
2360
2361 fn record_model_costs(&mut self, ctx: &AgentContext, provider: &str, model: &str, usage: &State, total_usd: f64) {
2362 let tokens = usage_total_tokens(usage);
2363 let mut metadata = State::new();
2364 metadata.insert("provider".to_string(), Value::String(provider.to_string()));
2365 metadata.insert("model".to_string(), Value::String(model.to_string()));
2366 metadata.insert("usage".to_string(), Value::Object(usage.clone()));
2367 if tokens > 0.0 {
2368 self.store.record_cost(&ctx.run_id, &ctx.session_id, &ctx.step_id, "model", model, tokens, "token", metadata.clone());
2369 }
2370 if total_usd > 0.0 {
2371 self.store.record_cost(&ctx.run_id, &ctx.session_id, &ctx.step_id, "model", model, total_usd, "usd", metadata);
2372 }
2373 }
2374}
2375
2376impl Default for Runtime {
2377 fn default() -> Self {
2378 Self::new()
2379 }
2380}
2381
2382#[derive(Clone, Debug)]
2383pub struct AgentContext {
2384 pub run_id: String,
2385 pub session_id: String,
2386 pub step_id: String,
2387 pub agent_role: String,
2388 pub lease_token: String,
2389 pub attempt: u64,
2390 pub state_version: u64,
2391 pub pending_patch: State,
2392}
2393
2394impl AgentContext {
2395 pub fn write_state(&mut self, key: &str, value: Value) {
2396 self.pending_patch.insert(key.to_string(), value);
2397 }
2398}
2399
2400pub const OMP_ADAPTER_SCHEMA_VERSION: &str = "agentledger.omp.adapter.v1";
2401
2402#[derive(Clone, Debug, Default)]
2403pub struct OmpSession {
2404 pub session_id: String,
2405 pub initial_state: State,
2406 pub metadata: State,
2407 pub run_id: Option<String>,
2408}
2409
2410#[derive(Clone, Debug, Default)]
2411pub struct OmpTurn {
2412 pub session_id: String,
2413 pub turn_id: String,
2414 pub agent_role: String,
2415 pub state_patch: State,
2416 pub metadata: State,
2417}
2418
2419#[derive(Clone, Debug, Default)]
2420pub struct OmpModelCall {
2421 pub session_id: String,
2422 pub turn_id: String,
2423 pub provider: String,
2424 pub model: String,
2425 pub request: State,
2426 pub response: State,
2427 pub usage: State,
2428 pub total_usd: f64,
2429 pub metadata: State,
2430}
2431
2432#[derive(Clone, Debug, Default)]
2433pub struct OmpToolProposal {
2434 pub session_id: String,
2435 pub turn_id: String,
2436 pub tool_name: String,
2437 pub arguments: State,
2438 pub provider: Option<String>,
2439 pub model: Option<String>,
2440 pub model_call_ref: Option<String>,
2441 pub confidence: Option<f64>,
2442 pub reason: Option<String>,
2443 pub metadata: State,
2444}
2445
2446#[derive(Clone, Debug)]
2447pub struct OmpToolExecution {
2448 pub session_id: String,
2449 pub turn_id: String,
2450 pub tool_name: String,
2451 pub arguments: State,
2452 pub result: Option<Value>,
2453 pub tool_call_id: Option<String>,
2454 pub tool_version: String,
2455 pub idempotency_key: Option<String>,
2456 pub ledger_status: Option<String>,
2457 pub error_type: Option<String>,
2458 pub error_message: Option<String>,
2459 pub external_id: Option<String>,
2460 pub causal_token: Option<String>,
2461 pub metadata: State,
2462}
2463
2464impl Default for OmpToolExecution {
2465 fn default() -> Self {
2466 Self {
2467 session_id: String::new(),
2468 turn_id: String::new(),
2469 tool_name: String::new(),
2470 arguments: State::new(),
2471 result: None,
2472 tool_call_id: None,
2473 tool_version: "external".to_string(),
2474 idempotency_key: None,
2475 ledger_status: None,
2476 error_type: None,
2477 error_message: None,
2478 external_id: None,
2479 causal_token: None,
2480 metadata: State::new(),
2481 }
2482 }
2483}
2484
2485#[derive(Clone, Debug)]
2486pub struct OmpFailure {
2487 pub session_id: String,
2488 pub turn_id: String,
2489 pub error_type: String,
2490 pub message: String,
2491 pub retryable: Option<bool>,
2492 pub status: String,
2493 pub terminal: Option<bool>,
2494 pub category: String,
2495 pub provider: Option<String>,
2496 pub model: Option<String>,
2497 pub request: State,
2498 pub usage: State,
2499 pub total_usd: f64,
2500 pub metadata: State,
2501 pub approval_id: Option<String>,
2502}
2503
2504impl Default for OmpFailure {
2505 fn default() -> Self {
2506 Self {
2507 session_id: String::new(),
2508 turn_id: String::new(),
2509 error_type: String::new(),
2510 message: String::new(),
2511 retryable: None,
2512 status: "failed".to_string(),
2513 terminal: None,
2514 category: "runtime".to_string(),
2515 provider: None,
2516 model: None,
2517 request: State::new(),
2518 usage: State::new(),
2519 total_usd: 0.0,
2520 metadata: State::new(),
2521 approval_id: None,
2522 }
2523 }
2524}
2525
2526#[derive(Clone, Debug)]
2527pub struct OmpStateChange {
2528 pub session_id: String,
2529 pub turn_id: Option<String>,
2530 pub reason: String,
2531 pub patch: State,
2532 pub label: String,
2533 pub commit_status: String,
2534 pub before_snapshot: Option<Value>,
2535 pub after_snapshot: Option<Value>,
2536 pub diff: Option<Value>,
2537 pub metadata: State,
2538}
2539
2540impl Default for OmpStateChange {
2541 fn default() -> Self {
2542 Self {
2543 session_id: String::new(),
2544 turn_id: None,
2545 reason: String::new(),
2546 patch: State::new(),
2547 label: "state".to_string(),
2548 commit_status: "committed".to_string(),
2549 before_snapshot: None,
2550 after_snapshot: None,
2551 diff: None,
2552 metadata: State::new(),
2553 }
2554 }
2555}
2556
2557#[derive(Clone, Debug)]
2558struct OmpBridgeSession {
2559 run_id: String,
2560 session_id: String,
2561 initial_step_id: Option<String>,
2562}
2563
2564#[derive(Clone, Debug)]
2565struct OmpActiveTurn {
2566 run_id: String,
2567 session_id: String,
2568 step_id: String,
2569 lease_token: String,
2570 attempt: u64,
2571 state_version: u64,
2572 agent_role: String,
2573}
2574
2575pub struct OmpLedgerBridge {
2576 pub runtime: Runtime,
2577 pub app_name: String,
2578 pub worker_id: String,
2579 pub lease_seconds: f64,
2580 sessions: HashMap<String, OmpBridgeSession>,
2581 active_turns: HashMap<String, OmpActiveTurn>,
2582}
2583
2584impl OmpLedgerBridge {
2585 pub fn new(runtime: Runtime, app_name: &str) -> Self {
2586 let app = if app_name.is_empty() { "omp" } else { app_name };
2587 Self {
2588 runtime,
2589 app_name: app.to_string(),
2590 worker_id: format!("omp:{app}"),
2591 lease_seconds: 60.0,
2592 sessions: HashMap::new(),
2593 active_turns: HashMap::new(),
2594 }
2595 }
2596
2597 pub fn record_session_started(&mut self, session: OmpSession) -> Result<String> {
2598 if let Some(existing) = self.sessions.get(&session.session_id) {
2599 return Ok(existing.run_id.clone());
2600 }
2601 let bridge_session = self.ensure_session(session.clone())?;
2602 let run = self.runtime.store.run(&bridge_session.run_id)?;
2603 let payload = compact_state([
2604 ("schema_version", OMP_ADAPTER_SCHEMA_VERSION.into()),
2605 ("adapter", "omp-ledger-bridge".into()),
2606 ("app_name", self.app_name.clone().into()),
2607 ("external_session_id", session.session_id.clone().into()),
2608 ("metadata", Value::Object(session.metadata)),
2609 ]);
2610 self.runtime.store.append_event(
2611 &bridge_session.run_id,
2612 Some(&bridge_session.session_id),
2613 None,
2614 "omp_session_started",
2615 payload,
2616 None,
2617 Some(run.state_version),
2618 None,
2619 );
2620 Ok(bridge_session.run_id)
2621 }
2622
2623 pub fn record_turn_started(&mut self, turn: OmpTurn) -> Result<String> {
2624 let key = self.turn_key(&turn.session_id, &turn.turn_id);
2625 if let Some(active) = self.active_turns.get(&key) {
2626 return Ok(active.step_id.clone());
2627 }
2628 let mut session = self.ensure_session(OmpSession {
2629 session_id: turn.session_id.clone(),
2630 ..Default::default()
2631 })?;
2632 if session.initial_step_id.is_some() {
2633 session.initial_step_id = None;
2634 self.sessions.insert(turn.session_id.clone(), session.clone());
2635 } else if self.next_runnable_step_id(&session.run_id).is_none() {
2636 self.runtime.store.create_external_step(&session.run_id)?;
2637 }
2638 let claim = self.runtime.store.claim_step(&self.worker_id, &session.run_id, self.lease_seconds)?;
2639 let role = if turn.agent_role.is_empty() { "OMPAgent".to_string() } else { turn.agent_role.clone() };
2640 let active = OmpActiveTurn {
2641 run_id: claim.run_id.clone(),
2642 session_id: claim.session_id.clone(),
2643 step_id: claim.step_id.clone(),
2644 lease_token: claim.lease_token,
2645 attempt: claim.attempt,
2646 state_version: claim.state_version,
2647 agent_role: role.clone(),
2648 };
2649 self.active_turns.insert(key, active.clone());
2650 let payload = compact_state([
2651 ("schema_version", OMP_ADAPTER_SCHEMA_VERSION.into()),
2652 ("adapter", "omp-ledger-bridge".into()),
2653 ("app_name", self.app_name.clone().into()),
2654 ("external_session_id", turn.session_id.clone().into()),
2655 ("external_turn_id", turn.turn_id.clone().into()),
2656 ("metadata", Value::Object(turn.metadata)),
2657 ]);
2658 self.runtime.store.append_event(
2659 &active.run_id,
2660 Some(&active.session_id),
2661 Some(&active.step_id),
2662 "omp_turn_started",
2663 payload,
2664 Some(&active.agent_role),
2665 Some(active.state_version),
2666 None,
2667 );
2668 Ok(active.step_id)
2669 }
2670
2671 pub fn record_turn_completed(&mut self, turn: OmpTurn) -> Result<u64> {
2672 let active = self.require_turn(&turn.session_id, &turn.turn_id)?;
2673 let next_version = self.runtime.store.commit_state_patch(
2674 &active.run_id,
2675 &active.step_id,
2676 &active.lease_token,
2677 active.state_version,
2678 turn.state_patch,
2679 )?;
2680 let payload = compact_state([
2681 ("schema_version", OMP_ADAPTER_SCHEMA_VERSION.into()),
2682 ("adapter", "omp-ledger-bridge".into()),
2683 ("app_name", self.app_name.clone().into()),
2684 ("external_session_id", turn.session_id.clone().into()),
2685 ("external_turn_id", turn.turn_id.clone().into()),
2686 ("metadata", Value::Object(turn.metadata)),
2687 ]);
2688 self.runtime.store.append_event(
2689 &active.run_id,
2690 Some(&active.session_id),
2691 Some(&active.step_id),
2692 "omp_turn_completed",
2693 payload,
2694 Some(&active.agent_role),
2695 Some(next_version),
2696 None,
2697 );
2698 let key = self.turn_key(&turn.session_id, &turn.turn_id);
2699 self.active_turns.remove(&key);
2700 Ok(next_version)
2701 }
2702
2703 pub fn record_model_call(&mut self, record: OmpModelCall) -> Result<()> {
2704 let ctx = self.context_for(&record.session_id, &record.turn_id)?;
2705 self.runtime.record_model_call_evidence(
2706 &ctx,
2707 &record.provider,
2708 &record.model,
2709 record.request,
2710 record.response,
2711 record.usage,
2712 record.total_usd,
2713 record.metadata,
2714 )
2715 }
2716
2717 pub fn record_tool_proposal(&mut self, proposal: OmpToolProposal) -> Result<()> {
2718 let ctx = self.context_for(&proposal.session_id, &proposal.turn_id)?;
2719 self.runtime.record_tool_call_proposal(
2720 &ctx,
2721 &proposal.tool_name,
2722 proposal.arguments,
2723 proposal.provider.as_deref(),
2724 proposal.model.as_deref(),
2725 proposal.model_call_ref.as_deref(),
2726 proposal.confidence,
2727 proposal.reason.as_deref(),
2728 proposal.metadata,
2729 );
2730 Ok(())
2731 }
2732
2733 pub fn record_tool_execution(&mut self, execution: OmpToolExecution) -> Result<State> {
2734 let active = self.require_turn(&execution.session_id, &execution.turn_id)?;
2735 let tool_call_id = execution.tool_call_id.clone().unwrap_or_else(|| new_id("toolcall"));
2736 let tool_version = if execution.tool_version.is_empty() { "external".to_string() } else { execution.tool_version.clone() };
2737 let request = compact_state([
2738 ("schema_version", OMP_ADAPTER_SCHEMA_VERSION.into()),
2739 ("tool", execution.tool_name.clone().into()),
2740 ("args", Value::Object(execution.arguments.clone())),
2741 ("tool_call_id", tool_call_id.clone().into()),
2742 ("metadata", Value::Object(execution.metadata.clone())),
2743 ]);
2744 let request_ref = format_state(&request);
2745 let request_hash = stable_hash(&request_ref);
2746 let idempotency_key = execution.idempotency_key.clone().unwrap_or_else(|| {
2747 format!(
2748 "omp:{}:{}:{}:{}",
2749 execution.session_id, execution.turn_id, execution.tool_name, tool_call_id
2750 )
2751 });
2752 let causal_token = execution.causal_token.clone().unwrap_or_else(|| format!("omp:{}:{}:{}", execution.session_id, execution.turn_id, tool_call_id));
2753 self.runtime.store.append_event(
2754 &active.run_id,
2755 Some(&active.session_id),
2756 Some(&active.step_id),
2757 "tool_call_requested",
2758 request,
2759 Some(&active.agent_role),
2760 Some(active.state_version),
2761 Some(&causal_token),
2762 );
2763 let now = now_seconds();
2764 let entry = ToolLedgerEntry {
2765 ledger_id: new_id("ledger"),
2766 run_id: active.run_id.clone(),
2767 session_id: active.session_id.clone(),
2768 step_id: active.step_id.clone(),
2769 tool_name: execution.tool_name.clone(),
2770 tool_version,
2771 tool_call_id: tool_call_id.clone(),
2772 idempotency_key: idempotency_key.clone(),
2773 causal_token: causal_token.clone(),
2774 request_hash,
2775 request_ref,
2776 status: "RESERVED".to_string(),
2777 external_id: None,
2778 response_hash: None,
2779 response_ref: None,
2780 error_type: None,
2781 response: None,
2782 created_at: now,
2783 updated_at: now,
2784 };
2785 if let Some(existing) = self.runtime.store.reserve_ledger(&idempotency_key, entry) {
2786 if existing.status == "SUCCEEDED" {
2787 let mut payload = State::new();
2788 payload.insert("tool".into(), execution.tool_name.clone().into());
2789 payload.insert("replayed_from_ledger".into(), Value::Bool(true));
2790 payload.insert("idempotency_key".into(), idempotency_key.clone().into());
2791 payload.insert("tool_call_id".into(), tool_call_id.clone().into());
2792 self.runtime.store.append_event(
2793 &active.run_id,
2794 Some(&active.session_id),
2795 Some(&active.step_id),
2796 "tool_call_completed",
2797 payload,
2798 Some(&active.agent_role),
2799 Some(active.state_version),
2800 Some(&causal_token),
2801 );
2802 return Ok(compact_state([
2803 ("ledger_status", "SUCCEEDED".into()),
2804 ("replayed_from_ledger", Value::Bool(true)),
2805 ("idempotency_key", idempotency_key.into()),
2806 ("tool_call_id", tool_call_id.into()),
2807 ]));
2808 }
2809 if existing.status == "PENDING_VERIFICATION" {
2810 return Err(RuntimeError("tool side effect pending verification".to_string()));
2811 }
2812 if existing.status == "RESERVED" || existing.status == "RUNNING" {
2813 return Err(RuntimeError("tool side effect already in progress".to_string()));
2814 }
2815 }
2816 let ledger_status = normalize_omp_ledger_status(
2817 execution.ledger_status.as_deref(),
2818 execution.error_message.as_deref().or(execution.error_type.as_deref()),
2819 )?;
2820 let (response_hash, response_ref) = execution.result.as_ref().map(|value| {
2821 let response_ref = format_value(value);
2822 (Some(stable_hash(&response_ref)), Some(response_ref))
2823 }).unwrap_or((None, None));
2824 self.runtime.store.update_ledger(
2825 &idempotency_key,
2826 &ledger_status,
2827 execution.external_id.clone().or_else(|| external_id_from_value(execution.result.as_ref())),
2828 execution.result.clone(),
2829 execution.error_type.clone(),
2830 );
2831 if let Some(entry) = self.runtime.store.tool_ledger.get_mut(&idempotency_key) {
2832 entry.response_hash = response_hash.clone();
2833 entry.response_ref = response_ref.clone();
2834 }
2835 let event_type = if ledger_status == "SUCCEEDED" || ledger_status == "COMPENSATED" { "tool_call_completed" } else { "tool_call_failed" };
2836 let payload = compact_state([
2837 ("tool", execution.tool_name.clone().into()),
2838 ("tool_call_id", tool_call_id.clone().into()),
2839 ("idempotency_key", idempotency_key.clone().into()),
2840 ("ledger_status", ledger_status.clone().into()),
2841 ("error", execution.error_message.unwrap_or_default().into()),
2842 ("error_type", execution.error_type.unwrap_or_default().into()),
2843 ]);
2844 self.runtime.store.append_event(
2845 &active.run_id,
2846 Some(&active.session_id),
2847 Some(&active.step_id),
2848 event_type,
2849 payload,
2850 Some(&active.agent_role),
2851 Some(active.state_version),
2852 Some(&causal_token),
2853 );
2854 if ledger_status != "RESERVED" && ledger_status != "RUNNING" {
2855 let mut metadata = State::new();
2856 metadata.insert("external_runtime".into(), "omp".into());
2857 metadata.insert("ledger_status".into(), ledger_status.clone().into());
2858 self.runtime.store.record_cost(&active.run_id, &active.session_id, &active.step_id, "tool", &execution.tool_name, 1.0, "call", metadata);
2859 }
2860 Ok(compact_state([
2861 ("ledger_status", ledger_status.into()),
2862 ("idempotency_key", idempotency_key.into()),
2863 ("tool_call_id", tool_call_id.into()),
2864 ]))
2865 }
2866
2867 pub fn record_failure(&mut self, failure: OmpFailure) -> Result<()> {
2868 let active = self.require_turn(&failure.session_id, &failure.turn_id)?;
2869 if failure.category == "model" {
2870 let ctx = self.context_for(&failure.session_id, &failure.turn_id)?;
2871 self.runtime.record_model_failure(
2872 &ctx,
2873 failure.provider.as_deref().unwrap_or("custom"),
2874 failure.model.as_deref().unwrap_or("unknown"),
2875 &failure.error_type,
2876 &failure.message,
2877 failure.retryable,
2878 failure.request,
2879 failure.usage,
2880 failure.total_usd,
2881 failure.metadata,
2882 )?;
2883 }
2884 if failure.terminal == Some(false) {
2885 return Ok(());
2886 }
2887 let status = if failure.status.is_empty() { "failed".to_string() } else { failure.status.to_lowercase() };
2888 if status == "waiting_human" || status == "approval_required" {
2889 self.runtime.store.mark_waiting_human(&active.run_id, &active.step_id, &failure.message, failure.approval_id.as_deref().unwrap_or(""));
2890 } else if status == "retry_scheduled" || status == "retry" || failure.retryable == Some(true) {
2891 self.runtime.store.mark_retry(&active.run_id, &active.step_id, &failure.error_type, &failure.message);
2892 } else {
2893 self.runtime.store.mark_failed(&active.run_id, &active.step_id, &failure.error_type, &failure.message);
2894 }
2895 let key = self.turn_key(&failure.session_id, &failure.turn_id);
2896 self.active_turns.remove(&key);
2897 Ok(())
2898 }
2899
2900 pub fn record_state_change(&mut self, change: OmpStateChange) -> Result<u64> {
2901 let session = self.ensure_session(OmpSession { session_id: change.session_id.clone(), ..Default::default() })?;
2902 let active_key = change.turn_id.as_ref().map(|turn_id| self.turn_key(&change.session_id, turn_id));
2903 let active = active_key.as_ref().and_then(|key| self.active_turns.get(key)).cloned();
2904 let mut artifacts = State::new();
2905 let label = if change.label.is_empty() { "state".to_string() } else { change.label.clone() };
2906 if let Some(value) = change.before_snapshot.clone() {
2907 let id = self.store_artifact(
2908 &session.run_id,
2909 active.as_ref().map(|item| item.step_id.as_str()),
2910 &format!("omp-{label}-before"),
2911 value,
2912 compact_state([
2913 ("schema_version", OMP_ADAPTER_SCHEMA_VERSION.into()),
2914 ("kind", "before_snapshot".into()),
2915 ("external_session_id", change.session_id.clone().into()),
2916 ]),
2917 active.as_ref(),
2918 )?;
2919 artifacts.insert("before_artifact_id".into(), id.into());
2920 }
2921 if let Some(value) = change.after_snapshot.clone() {
2922 let id = self.store_artifact(
2923 &session.run_id,
2924 active.as_ref().map(|item| item.step_id.as_str()),
2925 &format!("omp-{label}-after"),
2926 value,
2927 compact_state([
2928 ("schema_version", OMP_ADAPTER_SCHEMA_VERSION.into()),
2929 ("kind", "after_snapshot".into()),
2930 ("external_session_id", change.session_id.clone().into()),
2931 ]),
2932 active.as_ref(),
2933 )?;
2934 artifacts.insert("after_artifact_id".into(), id.into());
2935 }
2936 if let Some(value) = change.diff.clone() {
2937 let id = self.store_artifact(
2938 &session.run_id,
2939 active.as_ref().map(|item| item.step_id.as_str()),
2940 &format!("omp-{label}-diff"),
2941 value,
2942 compact_state([
2943 ("schema_version", OMP_ADAPTER_SCHEMA_VERSION.into()),
2944 ("kind", "diff".into()),
2945 ("external_session_id", change.session_id.clone().into()),
2946 ]),
2947 active.as_ref(),
2948 )?;
2949 artifacts.insert("diff_artifact_id".into(), id.into());
2950 }
2951 let mut version = self.runtime.store.run(&session.run_id)?.state_version;
2952 let commit_status = if change.commit_status.is_empty() { "committed".to_string() } else { change.commit_status.clone() };
2953 if !change.patch.is_empty() && (commit_status == "committed" || commit_status == "applied") {
2954 version = self.runtime.store.apply_system_state_patch(&session.run_id, change.patch.clone(), &change.reason)?;
2955 if let Some(key) = active_key.as_ref() {
2956 if let Some(item) = self.active_turns.get_mut(key) {
2957 item.state_version = version;
2958 }
2959 }
2960 }
2961 let payload = compact_state([
2962 ("schema_version", OMP_ADAPTER_SCHEMA_VERSION.into()),
2963 ("adapter", "omp-ledger-bridge".into()),
2964 ("app_name", self.app_name.clone().into()),
2965 ("external_session_id", change.session_id.clone().into()),
2966 ("external_turn_id", change.turn_id.clone().unwrap_or_default().into()),
2967 ("reason", change.reason.into()),
2968 ("commit_status", commit_status.into()),
2969 ("patch", Value::Object(change.patch)),
2970 ("artifacts", Value::Object(artifacts)),
2971 ("metadata", Value::Object(change.metadata)),
2972 ]);
2973 let active_after = active_key.as_ref().and_then(|key| self.active_turns.get(key)).cloned();
2974 self.runtime.store.append_event(
2975 &session.run_id,
2976 Some(&session.session_id),
2977 active_after.as_ref().map(|item| item.step_id.as_str()),
2978 "omp_state_change_recorded",
2979 payload,
2980 active_after.as_ref().map(|item| item.agent_role.as_str()),
2981 Some(version),
2982 None,
2983 );
2984 Ok(version)
2985 }
2986
2987 fn ensure_session(&mut self, session: OmpSession) -> Result<OmpBridgeSession> {
2988 if let Some(existing) = self.sessions.get(&session.session_id) {
2989 return Ok(existing.clone());
2990 }
2991 let bridge_session = if let Some(run_id) = session.run_id.clone() {
2992 let run = self.runtime.store.run(&run_id)?;
2993 let initial_step_id = self
2994 .runtime
2995 .store
2996 .steps(&run_id)
2997 .into_iter()
2998 .find(|step| step.status == "pending" || step.status == "retry_scheduled")
2999 .map(|step| step.step_id);
3000 OmpBridgeSession { run_id, session_id: run.session_id, initial_step_id }
3001 } else {
3002 let (run_id, step_id) = self.runtime.create_run(session.initial_state);
3003 let run = self.runtime.store.run(&run_id)?;
3004 OmpBridgeSession { run_id, session_id: run.session_id, initial_step_id: Some(step_id) }
3005 };
3006 self.sessions.insert(session.session_id, bridge_session.clone());
3007 Ok(bridge_session)
3008 }
3009
3010 fn require_turn(&self, session_id: &str, turn_id: &str) -> Result<OmpActiveTurn> {
3011 self.active_turns
3012 .get(&self.turn_key(session_id, turn_id))
3013 .cloned()
3014 .ok_or_else(|| RuntimeError(format!("OMP turn not active: {session_id}/{turn_id}")))
3015 }
3016
3017 fn context_for(&self, session_id: &str, turn_id: &str) -> Result<AgentContext> {
3018 let active = self.require_turn(session_id, turn_id)?;
3019 Ok(AgentContext {
3020 run_id: active.run_id,
3021 session_id: active.session_id,
3022 step_id: active.step_id,
3023 agent_role: active.agent_role,
3024 lease_token: active.lease_token,
3025 attempt: active.attempt,
3026 state_version: active.state_version,
3027 pending_patch: State::new(),
3028 })
3029 }
3030
3031 fn store_artifact(
3032 &mut self,
3033 run_id: &str,
3034 step_id: Option<&str>,
3035 name: &str,
3036 content: Value,
3037 metadata: State,
3038 active: Option<&OmpActiveTurn>,
3039 ) -> Result<String> {
3040 let mut artifact_content = State::new();
3041 artifact_content.insert("value".to_string(), content);
3042 let artifact = self.runtime.store.create_artifact(run_id, step_id, name, artifact_content, metadata);
3043 let run = self.runtime.store.run(run_id)?;
3044 let mut payload = State::new();
3045 payload.insert("artifact_id".to_string(), artifact.artifact_id.clone().into());
3046 payload.insert("name".to_string(), name.into());
3047 self.runtime.store.append_event(
3048 run_id,
3049 Some(&run.session_id),
3050 step_id,
3051 "artifact_created",
3052 payload,
3053 active.map(|item| item.agent_role.as_str()),
3054 Some(active.map(|item| item.state_version).unwrap_or(run.state_version)),
3055 None,
3056 );
3057 Ok(artifact.artifact_id)
3058 }
3059
3060 fn turn_key(&self, session_id: &str, turn_id: &str) -> String {
3061 format!("{session_id}\x1f{turn_id}")
3062 }
3063
3064 fn next_runnable_step_id(&self, run_id: &str) -> Option<String> {
3065 self.runtime
3066 .store
3067 .steps(run_id)
3068 .into_iter()
3069 .find(|step| step.status == "pending" || step.status == "retry_scheduled")
3070 .map(|step| step.step_id)
3071 }
3072}
3073
3074fn compact_state<const N: usize>(items: [(&str, Value); N]) -> State {
3075 let mut out = State::new();
3076 for (key, value) in items {
3077 match &value {
3078 Value::Null => continue,
3079 Value::String(item) if item.is_empty() => continue,
3080 Value::Object(item) if item.is_empty() => continue,
3081 _ => {}
3082 }
3083 out.insert(key.to_string(), value);
3084 }
3085 out
3086}
3087
3088fn normalize_omp_ledger_status(value: Option<&str>, error_message: Option<&str>) -> Result<String> {
3089 let raw = value.unwrap_or("");
3090 if raw.is_empty() {
3091 return Ok(if error_message.unwrap_or("").is_empty() { "SUCCEEDED" } else { "PENDING_VERIFICATION" }.to_string());
3092 }
3093 let normalized = raw.trim().to_ascii_uppercase();
3094 let status = match normalized.as_str() {
3095 "SUCCESS" | "SUCCEEDED" | "COMPLETED" | "OK" => "SUCCEEDED",
3096 "FAILED" | "PENDING_VERIFICATION" | "UNKNOWN" => "PENDING_VERIFICATION",
3097 "FAILED_NO_EFFECT" | "NO_EFFECT" => "FAILED_NO_EFFECT",
3098 "COMPENSATED" => "COMPENSATED",
3099 "RUNNING" => "RUNNING",
3100 "RESERVED" => "RESERVED",
3101 other => other,
3102 };
3103 match status {
3104 "SUCCEEDED" | "COMPENSATED" | "FAILED_NO_EFFECT" | "PENDING_VERIFICATION" | "RESERVED" | "RUNNING" => Ok(status.to_string()),
3105 _ => Err(RuntimeError(format!("unsupported Tool Ledger status: {raw}"))),
3106 }
3107}
3108
3109fn external_id_from_value(value: Option<&Value>) -> Option<String> {
3110 match value {
3111 Some(Value::Object(item)) => match item.get("external_id") {
3112 Some(Value::String(value)) => Some(value.clone()),
3113 Some(value) => Some(format_value(value)),
3114 None => None,
3115 },
3116 _ => None,
3117 }
3118}
3119
3120#[derive(Clone, Debug)]
3121pub struct EvidenceBundle {
3122 pub schema_version: String,
3123 pub bundle_hash: String,
3124 pub run: Run,
3125 pub steps: Vec<Step>,
3126 pub events: Vec<Event>,
3127 pub tool_ledger: Vec<ToolLedgerEntry>,
3128 pub approvals: Vec<ApprovalRequest>,
3129 pub artifacts: Vec<Artifact>,
3130 pub media_artifacts: Vec<State>,
3131 pub stream_checkpoints: Vec<State>,
3132 pub cost_records: Vec<CostRecord>,
3133 pub cost_summary: CostSummary,
3134 pub summary: State,
3135 pub final_state: State,
3136}
3137
3138#[derive(Clone, Debug, Default)]
3139pub struct WorkerRunSummary {
3140 pub worker_id: String,
3141 pub run_id: Option<String>,
3142 pub iterations: u64,
3143 pub attempts: u64,
3144 pub succeeded_attempts: u64,
3145 pub recovered_leases: u64,
3146 pub final_status: Option<String>,
3147 pub stopped_reason: String,
3148}
3149
3150pub struct LocalWorker {
3151 pub worker_id: String,
3152 pub agent_role: String,
3153 pub lease_seconds: f64,
3154 pub recover_expired: bool,
3155}
3156
3157impl LocalWorker {
3158 pub fn new(worker_id: &str, agent_role: &str) -> Self {
3159 Self {
3160 worker_id: worker_id.to_string(),
3161 agent_role: agent_role.to_string(),
3162 lease_seconds: 60.0,
3163 recover_expired: true,
3164 }
3165 }
3166
3167 pub fn run_until_idle(
3168 &self,
3169 runtime: &mut Runtime,
3170 run_id: &str,
3171 max_iterations: u64,
3172 agent: AgentFunc,
3173 ) -> Result<WorkerRunSummary> {
3174 let mut summary = WorkerRunSummary {
3175 worker_id: self.worker_id.clone(),
3176 run_id: Some(run_id.to_string()),
3177 stopped_reason: "max_iterations".to_string(),
3178 ..WorkerRunSummary::default()
3179 };
3180 for index in 1..=max_iterations {
3181 summary.iterations = index;
3182 if self.recover_expired {
3183 summary.recovered_leases += runtime.store.recover_expired_leases() as u64;
3184 }
3185 let status = runtime.store.run(run_id)?.status;
3186 if is_terminal_status(&status) {
3187 summary.final_status = Some(status);
3188 summary.stopped_reason = "terminal_status".to_string();
3189 break;
3190 }
3191 let ok = runtime.run_once(
3192 run_id,
3193 &self.worker_id,
3194 &self.agent_role,
3195 self.lease_seconds,
3196 agent,
3197 )?;
3198 if !ok {
3199 summary.stopped_reason = "idle".to_string();
3200 break;
3201 }
3202 summary.attempts += 1;
3203 summary.succeeded_attempts += 1;
3204 }
3205 let status = runtime.store.run(run_id)?.status;
3206 summary.final_status = Some(status.clone());
3207 if is_terminal_status(&status) {
3208 summary.stopped_reason = "terminal_status".to_string();
3209 }
3210 Ok(summary)
3211 }
3212}
3213
3214#[derive(Clone, Debug, Default)]
3215pub struct WorkerServiceSummary {
3216 pub worker_id: String,
3217 pub run_id: Option<String>,
3218 pub loops: u64,
3219 pub attempts: u64,
3220 pub succeeded_attempts: u64,
3221 pub recovered_leases: u64,
3222 pub idle_polls: u64,
3223 pub stopped_reason: String,
3224 pub final_status: Option<String>,
3225 pub stop_requested: bool,
3226}
3227
3228pub struct WorkerService {
3229 pub worker: LocalWorker,
3230 pub stop_requested: bool,
3231 pub stop_reason: String,
3232}
3233
3234impl WorkerService {
3235 pub fn new(worker: LocalWorker) -> Self {
3236 Self {
3237 worker,
3238 stop_requested: false,
3239 stop_reason: "stop_requested".to_string(),
3240 }
3241 }
3242
3243 pub fn request_stop(&mut self, reason: &str) {
3244 self.stop_requested = true;
3245 self.stop_reason = reason.to_string();
3246 }
3247
3248 pub fn serve(
3249 &mut self,
3250 runtime: &mut Runtime,
3251 run_id: Option<&str>,
3252 max_loops: u64,
3253 max_idle_polls: Option<u64>,
3254 agent: AgentFunc,
3255 ) -> Result<WorkerServiceSummary> {
3256 let mut summary = WorkerServiceSummary {
3257 worker_id: self.worker.worker_id.clone(),
3258 run_id: run_id.map(str::to_string),
3259 stopped_reason: "max_loops".to_string(),
3260 ..WorkerServiceSummary::default()
3261 };
3262 while summary.loops < max_loops {
3263 if self.stop_requested {
3264 summary.stopped_reason = self.stop_reason.clone();
3265 summary.stop_requested = true;
3266 break;
3267 }
3268 summary.loops += 1;
3269 let Some(run_id) = run_id else {
3270 summary.idle_polls += 1;
3271 if max_idle_polls.is_some_and(|limit| summary.idle_polls >= limit) {
3272 summary.stopped_reason = "idle".to_string();
3273 break;
3274 }
3275 continue;
3276 };
3277 let run_summary = self.worker.run_until_idle(runtime, run_id, 1, agent)?;
3278 summary.attempts += run_summary.attempts;
3279 summary.succeeded_attempts += run_summary.succeeded_attempts;
3280 summary.recovered_leases += run_summary.recovered_leases;
3281 summary.final_status = run_summary.final_status.clone();
3282 if summary
3283 .final_status
3284 .as_deref()
3285 .is_some_and(is_terminal_status)
3286 {
3287 summary.stopped_reason = "terminal_status".to_string();
3288 break;
3289 }
3290 if run_summary.attempts == 0 {
3291 summary.idle_polls += 1;
3292 if max_idle_polls.is_some_and(|limit| summary.idle_polls >= limit) {
3293 summary.stopped_reason = "idle".to_string();
3294 break;
3295 }
3296 } else {
3297 summary.idle_polls = 0;
3298 }
3299 }
3300 Ok(summary)
3301 }
3302}
3303
3304fn is_terminal_status(status: &str) -> bool {
3305 matches!(status, "completed" | "failed" | "cancelled")
3306}
3307
3308pub fn export_evidence(store: &MemoryStore, run_id: &str) -> Result<EvidenceBundle> {
3309 let run = store.run(run_id)?;
3310 let steps = store.steps(run_id);
3311 let events = store.events(run_id);
3312 let tool_ledger = store.ledger(run_id);
3313 let approvals = store.approval_requests(run_id);
3314 let artifacts = store.artifacts(run_id);
3315 let media_artifacts = media_artifacts_from(&artifacts);
3316 let stream_checkpoints = stream_checkpoints_from(&artifacts);
3317 let cost_records = store.cost_records(run_id);
3318 let cost_summary = store.cost_summary(run_id);
3319 let summary = evidence_summary(
3320 &steps,
3321 &events,
3322 &tool_ledger,
3323 &approvals,
3324 &artifacts,
3325 &media_artifacts,
3326 &stream_checkpoints,
3327 &cost_records,
3328 &cost_summary,
3329 );
3330 let final_state = store.final_state(run_id)?;
3331 let basis = format!(
3332 "{}:{}:{}:{}:{}",
3333 run_id,
3334 steps.len(),
3335 events.len(),
3336 tool_ledger.len(),
3337 cost_records.len()
3338 );
3339 Ok(EvidenceBundle {
3340 schema_version: "agentledger.evidence.v1".to_string(),
3341 bundle_hash: stable_hash(&basis),
3342 run,
3343 steps,
3344 events,
3345 tool_ledger,
3346 approvals,
3347 artifacts,
3348 media_artifacts,
3349 stream_checkpoints,
3350 cost_records,
3351 cost_summary,
3352 summary,
3353 final_state,
3354 })
3355}
3356
3357fn evidence_summary(
3358 steps: &[Step],
3359 events: &[Event],
3360 ledger: &[ToolLedgerEntry],
3361 approvals: &[ApprovalRequest],
3362 artifacts: &[Artifact],
3363 media_artifacts: &[State],
3364 stream_checkpoints: &[State],
3365 cost_records: &[CostRecord],
3366 cost_summary: &CostSummary,
3367) -> State {
3368 let mut summary = State::new();
3369 summary.insert("event_count".into(), Value::Number(events.len() as f64));
3370 summary.insert("step_count".into(), Value::Number(steps.len() as f64));
3371 summary.insert(
3372 "tool_ledger_count".into(),
3373 Value::Number(ledger.len() as f64),
3374 );
3375 summary.insert(
3376 "approval_count".into(),
3377 Value::Number(approvals.len() as f64),
3378 );
3379 summary.insert(
3380 "artifact_count".into(),
3381 Value::Number(artifacts.len() as f64),
3382 );
3383 summary.insert(
3384 "media_artifact_count".into(),
3385 Value::Number(media_artifacts.len() as f64),
3386 );
3387 summary.insert(
3388 "stream_checkpoint_count".into(),
3389 Value::Number(stream_checkpoints.len() as f64),
3390 );
3391 summary.insert(
3392 "cost_record_count".into(),
3393 Value::Number(cost_records.len() as f64),
3394 );
3395 summary.insert(
3396 "has_failed_steps".into(),
3397 Value::Bool(steps.iter().any(|step| step.status == "failed")),
3398 );
3399 summary.insert(
3400 "has_pending_verification".into(),
3401 Value::Bool(
3402 ledger
3403 .iter()
3404 .any(|row| row.status == "PENDING_VERIFICATION"),
3405 ),
3406 );
3407 summary.insert(
3408 "has_pending_approvals".into(),
3409 Value::Bool(approvals.iter().any(|row| row.status == "PENDING")),
3410 );
3411 let mut cost = State::new();
3412 cost.insert("tool_calls".into(), Value::Number(cost_summary.tool_calls));
3413 cost.insert(
3414 "model_tokens".into(),
3415 Value::Number(cost_summary.model_tokens),
3416 );
3417 cost.insert("total_usd".into(), Value::Number(cost_summary.total_usd));
3418 summary.insert("cost_summary".into(), Value::Object(cost));
3419 summary
3420}
3421
3422#[derive(Clone, Debug)]
3423pub struct ReplaySummary {
3424 pub run_id: String,
3425 pub event_count: usize,
3426 pub tool_call_count: usize,
3427 pub final_state: State,
3428 pub event_hash: String,
3429 pub replay_safe: bool,
3430 pub artifact_count: usize,
3431 pub media_artifact_count: usize,
3432 pub stream_checkpoint_count: usize,
3433}
3434
3435pub fn replay(store: &MemoryStore, run_id: &str) -> Result<ReplaySummary> {
3436 let events = store.events(run_id);
3437 let tool_call_count = events
3438 .iter()
3439 .filter(|event| event.event_type.starts_with("tool_call_"))
3440 .count();
3441 let digest = events
3442 .iter()
3443 .map(|event| format!("{}:{}:{}", event.seq, event.event_type, event.payload_hash))
3444 .collect::<Vec<_>>()
3445 .join("|");
3446 let artifacts = store.artifacts(run_id);
3447 Ok(ReplaySummary {
3448 run_id: run_id.to_string(),
3449 event_count: events.len(),
3450 tool_call_count,
3451 final_state: store.final_state(run_id)?,
3452 event_hash: stable_hash(&digest),
3453 replay_safe: true,
3454 artifact_count: artifacts.len(),
3455 media_artifact_count: media_artifacts_from(&artifacts).len(),
3456 stream_checkpoint_count: stream_checkpoints_from(&artifacts).len(),
3457 })
3458}
3459
3460#[derive(Clone, Debug)]
3461pub struct CostAttributionReport {
3462 pub run_id: String,
3463 pub total: CostSummary,
3464 pub by_agent: HashMap<String, CostSummary>,
3465 pub by_step: HashMap<String, CostSummary>,
3466 pub by_name: HashMap<String, CostSummary>,
3467}
3468
3469pub fn cost_attribution(store: &MemoryStore, run_id: &str) -> CostAttributionReport {
3470 let mut step_roles = HashMap::new();
3471 for event in store.events(run_id) {
3472 if let (Some(step_id), Some(agent_role)) = (event.step_id, event.agent_role) {
3473 step_roles.insert(step_id, agent_role);
3474 }
3475 }
3476 let mut report = CostAttributionReport {
3477 run_id: run_id.to_string(),
3478 total: CostSummary::default(),
3479 by_agent: HashMap::new(),
3480 by_step: HashMap::new(),
3481 by_name: HashMap::new(),
3482 };
3483 for record in store.cost_records(run_id) {
3484 add_cost(&mut report.total, &record);
3485 let agent = step_roles
3486 .get(&record.step_id)
3487 .cloned()
3488 .unwrap_or_else(|| "<unknown>".to_string());
3489 add_cost(report.by_agent.entry(agent).or_default(), &record);
3490 add_cost(
3491 report.by_step.entry(record.step_id.clone()).or_default(),
3492 &record,
3493 );
3494 add_cost(
3495 report.by_name.entry(record.name.clone()).or_default(),
3496 &record,
3497 );
3498 }
3499 report
3500}
3501
3502#[derive(Clone, Debug)]
3503pub struct FailureAttributionReport {
3504 pub run_id: String,
3505 pub run_status: String,
3506 pub failed_steps: Vec<Step>,
3507 pub pending_verification: Vec<ToolLedgerEntry>,
3508 pub pending_approvals: Vec<ApprovalRequest>,
3509 pub failure_events: Vec<Event>,
3510 pub failure_envelopes: Vec<State>,
3511 pub failure_lifecycle: State,
3512 pub failure_causal_graph: State,
3513 pub failure_replay_plan: State,
3514 pub failure_alerts: State,
3515 pub failure_export: State,
3516 pub summary: State,
3517}
3518
3519pub fn failure_attribution(store: &MemoryStore, run_id: &str) -> Result<FailureAttributionReport> {
3520 let run = store.run(run_id)?;
3521 let failed_steps: Vec<Step> = store
3522 .steps(run_id)
3523 .into_iter()
3524 .filter(|step| step.status == "failed")
3525 .collect();
3526 let pending_verification: Vec<ToolLedgerEntry> = store
3527 .ledger(run_id)
3528 .into_iter()
3529 .filter(|entry| entry.status == "PENDING_VERIFICATION")
3530 .collect();
3531 let pending_approvals: Vec<ApprovalRequest> = store
3532 .approval_requests(run_id)
3533 .into_iter()
3534 .filter(|entry| entry.status == "PENDING")
3535 .collect();
3536 let failure_events: Vec<Event> = store
3537 .events(run_id)
3538 .into_iter()
3539 .filter(|event| is_failure_event(&event.event_type))
3540 .collect();
3541 let steps = store.steps(run_id);
3542 let ledger = store.ledger(run_id);
3543 let approvals = store.approval_requests(run_id);
3544 let events = store.events(run_id);
3545 let costs = store.cost_records(run_id);
3546 let failure_envelopes = failure_envelopes(run_id, &run.status, &steps, &ledger, &approvals, &failure_events);
3547 let failure_lifecycle = failure_lifecycle(run_id, &run.status, &failure_envelopes);
3548 let failure_causal_graph = failure_causal_graph(run_id, &run.status, &failure_envelopes, &steps, &ledger, &approvals, &events, &costs);
3549 let failure_replay_plan = failure_replay_plan(run_id, &failure_envelopes, &ledger, &events);
3550 let failure_alerts = failure_alerts(run_id, &failure_envelopes, &failure_replay_plan);
3551 let mut summary = State::new();
3552 summary.insert(
3553 "failed_step_count".to_string(),
3554 Value::Number(failed_steps.len() as f64),
3555 );
3556 summary.insert(
3557 "pending_verification_count".to_string(),
3558 Value::Number(pending_verification.len() as f64),
3559 );
3560 summary.insert(
3561 "pending_approval_count".to_string(),
3562 Value::Number(pending_approvals.len() as f64),
3563 );
3564 summary.insert(
3565 "failure_event_count".to_string(),
3566 Value::Number(failure_events.len() as f64),
3567 );
3568 summary.insert(
3569 "failure_envelope_count".to_string(),
3570 Value::Number(failure_envelopes.len() as f64),
3571 );
3572 summary.insert(
3573 "failure_lifecycle_event_count".to_string(),
3574 Value::Number(state_array_len(&failure_lifecycle, "events") as f64),
3575 );
3576 summary.insert(
3577 "failure_alert_count".to_string(),
3578 failure_alerts
3579 .get("alert_count")
3580 .cloned()
3581 .unwrap_or(Value::Number(0.0)),
3582 );
3583 summary.insert(
3584 "unsafe_replay_side_effect_count".to_string(),
3585 failure_replay_plan
3586 .get("unsafe_side_effect_count")
3587 .cloned()
3588 .unwrap_or(Value::Number(0.0)),
3589 );
3590 summary.insert(
3591 "terminal_failure_count".to_string(),
3592 Value::Number(count_state_field(&failure_envelopes, "status", "terminal") as f64),
3593 );
3594 summary.insert(
3595 "recoverable_failure_count".to_string(),
3596 Value::Number(count_recoverable_failures(&failure_envelopes) as f64),
3597 );
3598 let failure_export = failure_export(
3599 run_id,
3600 &run.status,
3601 &summary,
3602 &failure_envelopes,
3603 &failure_lifecycle,
3604 &failure_causal_graph,
3605 &failure_replay_plan,
3606 &failure_alerts,
3607 );
3608 Ok(FailureAttributionReport {
3609 run_id: run_id.to_string(),
3610 run_status: run.status,
3611 failed_steps,
3612 pending_verification,
3613 pending_approvals,
3614 failure_events,
3615 failure_envelopes,
3616 failure_lifecycle,
3617 failure_causal_graph,
3618 failure_replay_plan,
3619 failure_alerts,
3620 failure_export,
3621 summary,
3622 })
3623}
3624
3625fn failure_envelopes(
3626 run_id: &str,
3627 run_status: &str,
3628 steps: &[Step],
3629 ledger: &[ToolLedgerEntry],
3630 approvals: &[ApprovalRequest],
3631 events: &[Event],
3632) -> Vec<State> {
3633 let mut rows = Vec::new();
3634 for step in steps {
3635 if step.status == "failed" || step.status == "retry_scheduled" || step.status == "waiting_human" {
3636 let (status, severity, recoverability, retryability) = if step.status == "retry_scheduled" {
3637 ("recovery_scheduled", "warn", "auto_retry", "retryable")
3638 } else if step.status == "waiting_human" {
3639 ("waiting_human", "warn", "human_required", "unknown")
3640 } else {
3641 ("terminal", "risk", "terminal", "not_retryable")
3642 };
3643 rows.push(failure_envelope(
3644 run_id,
3645 "step",
3646 &step.step_id,
3647 &failure_category(&format!(
3648 "{} {}",
3649 step.last_error_type.clone().unwrap_or_default(),
3650 step.last_error.clone().unwrap_or_default()
3651 ), "agent"),
3652 status,
3653 severity,
3654 recoverability,
3655 retryability,
3656 "agent",
3657 &first_text(&[
3658 step.last_error.as_deref(),
3659 step.last_error_type.as_deref(),
3660 Some("step failure"),
3661 ]),
3662 state(&[
3663 ("step_id", step.step_id.clone().into()),
3664 ("occurred_at", Value::Number(step.updated_at)),
3665 ]),
3666 vec![ref_state("step", &step.step_id)],
3667 ));
3668 }
3669 }
3670 for entry in ledger {
3671 if entry.status == "PENDING_VERIFICATION" || entry.status == "FAILED" || entry.status == "ERROR" {
3672 let terminal = entry.status == "FAILED" || entry.status == "ERROR";
3673 rows.push(failure_envelope(
3674 run_id,
3675 "tool_ledger",
3676 &first_text(&[Some(&entry.ledger_id), Some(&entry.tool_name), Some(&entry.step_id)]),
3677 "tool",
3678 if terminal { "terminal" } else { "unknown_side_effect" },
3679 if terminal { "risk" } else { "warn" },
3680 if terminal { "terminal" } else { "manual_verification" },
3681 if terminal { "not_retryable" } else { "unknown" },
3682 "tool",
3683 &first_text(&[entry.error_type.as_deref(), Some("tool side effect requires verification")]),
3684 state(&[
3685 ("step_id", entry.step_id.clone().into()),
3686 ("tool_name", entry.tool_name.clone().into()),
3687 ("occurred_at", Value::Number(entry.updated_at)),
3688 ]),
3689 vec![ref_state("step", &entry.step_id), ref_state("tool", &entry.tool_name)],
3690 ));
3691 }
3692 }
3693 for approval in approvals {
3694 if approval.status == "PENDING" || approval.status == "DENIED" {
3695 let denied = approval.status == "DENIED";
3696 rows.push(failure_envelope(
3697 run_id,
3698 "approval",
3699 &first_text(&[Some(&approval.approval_id), Some(&approval.tool_name), Some(&approval.step_id)]),
3700 if denied { "policy" } else { "approval" },
3701 if denied { "blocked" } else { "waiting_human" },
3702 if denied { "risk" } else { "warn" },
3703 if denied { "terminal" } else { "human_required" },
3704 if denied { "not_retryable" } else { "unknown" },
3705 "policy",
3706 &first_text(&[
3707 approval.decision_reason.as_deref(),
3708 Some(&approval.reason),
3709 Some(if denied { "approval denied" } else { "approval pending" }),
3710 ]),
3711 state(&[
3712 ("step_id", approval.step_id.clone().into()),
3713 ("tool_name", approval.tool_name.clone().into()),
3714 ("approval_id", approval.approval_id.clone().into()),
3715 ("occurred_at", Value::Number(approval.updated_at)),
3716 ]),
3717 vec![
3718 ref_state("step", &approval.step_id),
3719 ref_state("tool", &approval.tool_name),
3720 ref_state("approval", &approval.approval_id),
3721 ],
3722 ));
3723 }
3724 }
3725 for event in events {
3726 let category = event_failure_category(event);
3727 let status = event_failure_status(&event.event_type, run_status);
3728 let message = first_text(&[
3729 state_string(&event.payload, "error").as_deref(),
3730 state_string(&event.payload, "reason").as_deref(),
3731 state_string(&event.payload, "error_type").as_deref(),
3732 Some(&event.event_type),
3733 ]);
3734 rows.push(failure_envelope(
3735 run_id,
3736 "event",
3737 &event.seq.to_string(),
3738 &category,
3739 &status,
3740 if status == "terminal" || status == "blocked" || status == "failed" { "risk" } else { "warn" },
3741 &event_recoverability(&event.event_type, run_status),
3742 &event_retryability(&event.event_type),
3743 &owner_for_failure_category(&category),
3744 &message,
3745 state(&[
3746 ("step_id", event.step_id.clone().unwrap_or_default().into()),
3747 ("event_seq", Value::Number(event.seq as f64)),
3748 ("event_type", event.event_type.clone().into()),
3749 ("occurred_at", Value::Number(event.timestamp)),
3750 ]),
3751 vec![
3752 ref_state("event", &event.seq.to_string()),
3753 ref_state("step", event.step_id.as_deref().unwrap_or("")),
3754 ],
3755 ));
3756 }
3757 dedupe_states(rows, "failure_id")
3758}
3759
3760fn failure_envelope(
3761 run_id: &str,
3762 source_kind: &str,
3763 source_id: &str,
3764 category: &str,
3765 status: &str,
3766 severity: &str,
3767 recoverability: &str,
3768 retryability: &str,
3769 owner: &str,
3770 message: &str,
3771 extra: State,
3772 refs: Vec<State>,
3773) -> State {
3774 let mut row = state(&[
3775 ("schema_version", "agentledger.failure.envelope.v1".into()),
3776 ("failure_id", format!("failure-{}", slug(&format!("{run_id}-{source_kind}-{source_id}"))).into()),
3777 ("run_id", run_id.into()),
3778 ("source_kind", source_kind.into()),
3779 ("source_id", source_id.into()),
3780 ("category", category.into()),
3781 ("status", status.into()),
3782 ("severity", severity.into()),
3783 ("recoverability", recoverability.into()),
3784 ("retryability", retryability.into()),
3785 ("owner", owner.into()),
3786 ("message", message.into()),
3787 ("causal_refs", Value::Array(refs.iter().cloned().map(Value::Object).collect())),
3788 ("evidence_refs", Value::Array(refs.into_iter().map(Value::Object).collect())),
3789 ]);
3790 for (key, value) in extra {
3791 if value != Value::String(String::new()) && value != Value::Null {
3792 row.insert(key, value);
3793 }
3794 }
3795 row
3796}
3797
3798fn failure_lifecycle(run_id: &str, run_status: &str, envelopes: &[State]) -> State {
3799 let mut events = Vec::new();
3800 for env in envelopes {
3801 events.push(lifecycle_row(run_id, env, "failure_detected", state_value_string(env, "message"), state_value_string(env, "severity")));
3802 events.push(lifecycle_row(run_id, env, "failure_classified", state_value_string(env, "category"), state_value_string(env, "severity")));
3803 let status = state_value_string(env, "status");
3804 let recoverability = state_value_string(env, "recoverability");
3805 if ["recovery_scheduled", "waiting_human", "unknown_side_effect"].contains(&status.as_str()) || ["auto_retry", "human_required", "manual_verification"].contains(&recoverability.as_str()) {
3806 events.push(lifecycle_row(run_id, env, "failure_recovery_scheduled", "recovery scheduled".to_string(), "warn".to_string()));
3807 }
3808 if ["terminal", "blocked"].contains(&status.as_str()) || recoverability == "terminal" {
3809 events.push(lifecycle_row(run_id, env, "failure_terminal", state_value_string(env, "message"), "risk".to_string()));
3810 }
3811 }
3812 state(&[
3813 ("schema_version", "agentledger.failure.lifecycle.v1".into()),
3814 ("run_id", run_id.into()),
3815 ("run_status", run_status.into()),
3816 ("events", Value::Array(events.iter().cloned().map(Value::Object).collect())),
3817 ("terminal", Value::Bool(events.iter().any(|row| state_value_string(row, "stage") == "failure_terminal"))),
3818 ("recoverable", Value::Bool(events.iter().any(|row| state_value_string(row, "stage") == "failure_recovery_scheduled"))),
3819 ])
3820}
3821
3822fn lifecycle_row(run_id: &str, env: &State, stage: &str, message: String, severity: String) -> State {
3823 state(&[
3824 ("schema_version", "agentledger.failure.lifecycle.v1".into()),
3825 ("stage", stage.into()),
3826 ("run_id", run_id.into()),
3827 ("failure_id", state_value_string(env, "failure_id").into()),
3828 ("category", state_value_string(env, "category").into()),
3829 ("recoverability", state_value_string(env, "recoverability").into()),
3830 ("retryability", state_value_string(env, "retryability").into()),
3831 ("owner", state_value_string(env, "owner").into()),
3832 ("message", message.into()),
3833 ("severity", severity.into()),
3834 ("causal_refs", env.get("causal_refs").cloned().unwrap_or(Value::Array(vec![]))),
3835 ])
3836}
3837
3838fn failure_causal_graph(
3839 run_id: &str,
3840 run_status: &str,
3841 envelopes: &[State],
3842 steps: &[Step],
3843 ledger: &[ToolLedgerEntry],
3844 approvals: &[ApprovalRequest],
3845 events: &[Event],
3846 costs: &[CostRecord],
3847) -> State {
3848 let mut nodes = vec![state(&[("id", format!("run:{}", slug(run_id)).into()), ("kind", "run".into()), ("status", run_status.into())])];
3849 let mut edges = Vec::new();
3850 for step in steps {
3851 nodes.push(state(&[("id", format!("step:{}", slug(&step.step_id)).into()), ("kind", "step".into()), ("status", step.status.clone().into())]));
3852 edges.push(state(&[("source", format!("run:{}", slug(run_id)).into()), ("target", format!("step:{}", slug(&step.step_id)).into()), ("kind", "contains_step".into())]));
3853 }
3854 for event in events {
3855 nodes.push(state(&[("id", format!("event:{}", event.seq).into()), ("kind", "event".into()), ("event_type", event.event_type.clone().into())]));
3856 edges.push(state(&[("source", format!("run:{}", slug(run_id)).into()), ("target", format!("event:{}", event.seq).into()), ("kind", "emitted_event".into())]));
3857 }
3858 for entry in ledger {
3859 nodes.push(state(&[("id", format!("tool:{}", slug(&entry.tool_name)).into()), ("kind", "tool".into()), ("status", entry.status.clone().into())]));
3860 }
3861 for approval in approvals {
3862 nodes.push(state(&[("id", format!("approval:{}", slug(&approval.approval_id)).into()), ("kind", "approval".into()), ("status", approval.status.clone().into())]));
3863 }
3864 for cost in costs {
3865 nodes.push(state(&[("id", format!("cost:{}", slug(&cost.cost_id)).into()), ("kind", "cost".into()), ("category", cost.category.clone().into()), ("amount", Value::Number(cost.amount)), ("unit", cost.unit.clone().into())]));
3866 }
3867 for env in envelopes {
3868 let id = format!("failure:{}", slug(&state_value_string(env, "failure_id")));
3869 nodes.push(state(&[("id", id.clone().into()), ("kind", "failure".into()), ("category", state_value_string(env, "category").into()), ("status", state_value_string(env, "status").into()), ("owner", state_value_string(env, "owner").into())]));
3870 edges.push(state(&[("source", format!("run:{}", slug(run_id)).into()), ("target", id.into()), ("kind", "has_failure".into())]));
3871 }
3872 let nodes = dedupe_states(nodes, "id");
3873 state(&[
3874 ("schema_version", "agentledger.failure.causal_graph.v1".into()),
3875 ("run_id", run_id.into()),
3876 ("nodes", Value::Array(nodes.iter().cloned().map(Value::Object).collect())),
3877 ("edges", Value::Array(edges.iter().cloned().map(Value::Object).collect())),
3878 ("summary", Value::Object(state(&[
3879 ("node_count", Value::Number(nodes.len() as f64)),
3880 ("edge_count", Value::Number(edges.len() as f64)),
3881 ("failure_node_count", Value::Number(count_state_field(&nodes, "kind", "failure") as f64)),
3882 ]))),
3883 ])
3884}
3885
3886fn failure_replay_plan(run_id: &str, envelopes: &[State], ledger: &[ToolLedgerEntry], events: &[Event]) -> State {
3887 let mut actions = Vec::new();
3888 let mut unsafe_count = 0;
3889 let mut manual_count = 0;
3890 for env in envelopes {
3891 let status = state_value_string(env, "status");
3892 let recoverability = state_value_string(env, "recoverability");
3893 let mut action = state(&[
3894 ("failure_id", state_value_string(env, "failure_id").into()),
3895 ("category", state_value_string(env, "category").into()),
3896 ("status", status.clone().into()),
3897 ("replay_action", "reuse_recorded_evidence".into()),
3898 ("replay_safe", Value::Bool(true)),
3899 ("requires_manual_verification", Value::Bool(false)),
3900 ("reason", "recorded runtime evidence can be inspected without calling external systems".into()),
3901 ]);
3902 if status == "unknown_side_effect" || recoverability == "manual_verification" {
3903 action.insert("replay_action".into(), "manual_verify_side_effect".into());
3904 action.insert("replay_safe".into(), Value::Bool(false));
3905 action.insert("requires_manual_verification".into(), Value::Bool(true));
3906 action.insert("reason".into(), "Tool Ledger recorded an unknown side-effect state".into());
3907 unsafe_count += 1;
3908 manual_count += 1;
3909 } else if status == "waiting_human" {
3910 action.insert("replay_action".into(), "resume_after_approval".into());
3911 } else if status == "recovery_scheduled" {
3912 action.insert("replay_action".into(), "retry_from_checkpoint".into());
3913 } else if status == "terminal" || status == "blocked" {
3914 action.insert("replay_action".into(), "terminal_stop".into());
3915 }
3916 actions.push(action);
3917 }
3918 state(&[
3919 ("schema_version", "agentledger.failure.replay_plan.v1".into()),
3920 ("run_id", run_id.into()),
3921 ("mode", "evidence_only".into()),
3922 ("safe_to_replay", Value::Bool(unsafe_count == 0)),
3923 ("unsafe_side_effect_count", Value::Number(unsafe_count as f64)),
3924 ("manual_verification_count", Value::Number(manual_count as f64)),
3925 ("recorded_tool_call_count", Value::Number(ledger.len() as f64)),
3926 ("recorded_event_count", Value::Number(events.len() as f64)),
3927 ("actions", Value::Array(actions.into_iter().map(Value::Object).collect())),
3928 ])
3929}
3930
3931fn failure_alerts(run_id: &str, envelopes: &[State], replay_plan: &State) -> State {
3932 let mut alerts = Vec::new();
3933 if count_state_field(envelopes, "status", "terminal") > 0 {
3934 alerts.push(alert_state(run_id, "terminal_failure", "risk", "terminal failure recorded"));
3935 }
3936 if count_state_field(envelopes, "status", "unknown_side_effect") > 0 {
3937 alerts.push(alert_state(run_id, "unknown_side_effect", "risk", "tool side-effect state requires manual verification"));
3938 }
3939 if state_number(replay_plan, "unsafe_side_effect_count") > 0.0 {
3940 alerts.push(alert_state(run_id, "unsafe_replay_blocked", "risk", "failure replay plan blocks unsafe automatic replay"));
3941 }
3942 state(&[
3943 ("schema_version", "agentledger.failure.alerts.v1".into()),
3944 ("run_id", run_id.into()),
3945 ("alerts", Value::Array(alerts.iter().cloned().map(Value::Object).collect())),
3946 ("alert_count", Value::Number(alerts.len() as f64)),
3947 ])
3948}
3949
3950fn alert_state(run_id: &str, kind: &str, severity: &str, message: &str) -> State {
3951 state(&[
3952 ("schema_version", "agentledger.failure.alerts.v1".into()),
3953 ("run_id", run_id.into()),
3954 ("kind", kind.into()),
3955 ("severity", severity.into()),
3956 ("message", message.into()),
3957 ])
3958}
3959
3960fn failure_export(
3961 run_id: &str,
3962 run_status: &str,
3963 summary: &State,
3964 envelopes: &[State],
3965 lifecycle: &State,
3966 graph: &State,
3967 replay_plan: &State,
3968 alerts: &State,
3969) -> State {
3970 state(&[
3971 ("schema_version", "agentledger.failure.export.v1".into()),
3972 ("run_id", run_id.into()),
3973 ("run_status", run_status.into()),
3974 ("summary", Value::Object(summary.clone())),
3975 ("failure_envelopes", Value::Array(envelopes.iter().cloned().map(Value::Object).collect())),
3976 ("failure_lifecycle", Value::Object(lifecycle.clone())),
3977 ("failure_causal_graph", Value::Object(graph.clone())),
3978 ("failure_replay_plan", Value::Object(replay_plan.clone())),
3979 ("failure_alerts", Value::Object(alerts.clone())),
3980 ("external_mappings", Value::Object(state(&[
3981 ("opentelemetry", Value::Object(state(&[("span_event_count", Value::Number(state_array_len(lifecycle, "events") as f64))]))),
3982 ("langfuse", Value::Object(state(&[("trace_id", run_id.into()), ("observation_count", Value::Number(envelopes.len() as f64))]))),
3983 ("langsmith", Value::Object(state(&[("run_id", run_id.into()), ("feedback_count", Value::Number(envelopes.len() as f64))]))),
3984 ("temporal", Value::Object(state(&[("workflow_id", run_id.into()), ("failure_count", Value::Number(envelopes.len() as f64)), ("safe_to_replay", replay_plan.get("safe_to_replay").cloned().unwrap_or(Value::Bool(false)))]))),
3985 ]))),
3986 ])
3987}
3988
3989fn encode_store(store: &MemoryStore) -> String {
3990 let mut lines = vec!["AGENTLEDGER_RUST_STORE_V1".to_string()];
3991 let mut runs: Vec<_> = store.runs.values().collect();
3992 runs.sort_by(|a, b| a.run_id.cmp(&b.run_id));
3993 for run in runs {
3994 lines.push(join_fields(&[
3995 "R".to_string(),
3996 hex_encode(&run.run_id),
3997 hex_encode(&run.session_id),
3998 hex_encode(&run.status),
3999 run.state_version.to_string(),
4000 run.created_at.to_string(),
4001 run.updated_at.to_string(),
4002 encode_state(&run.state),
4003 ]));
4004 }
4005
4006 let mut steps: Vec<_> = store.steps.values().collect();
4007 steps.sort_by(|a, b| a.step_id.cmp(&b.step_id));
4008 for step in steps {
4009 lines.push(join_fields(&[
4010 "S".to_string(),
4011 hex_encode(&step.step_id),
4012 hex_encode(&step.run_id),
4013 hex_encode(&step.session_id),
4014 hex_encode(&step.status),
4015 encode_option_string(&step.owner),
4016 encode_option_string(&step.lease_token),
4017 encode_option_f64(step.lease_until),
4018 step.attempt.to_string(),
4019 step.state_version.to_string(),
4020 encode_option_string(&step.checkpoint_id),
4021 encode_option_string(&step.last_error_type),
4022 encode_option_string(&step.last_error),
4023 encode_option_f64(step.cancelled_at),
4024 step.created_at.to_string(),
4025 step.updated_at.to_string(),
4026 ]));
4027 }
4028
4029 let mut run_ids: Vec<_> = store.events.keys().collect();
4030 run_ids.sort();
4031 for run_id in run_ids {
4032 if let Some(events) = store.events.get(run_id) {
4033 for event in events {
4034 lines.push(join_fields(&[
4035 "E".to_string(),
4036 hex_encode(&event.event_id),
4037 hex_encode(&event.run_id),
4038 encode_option_string(&event.session_id),
4039 encode_option_string(&event.step_id),
4040 event.seq.to_string(),
4041 hex_encode(&event.event_type),
4042 event.timestamp.to_string(),
4043 encode_option_string(&event.agent_role),
4044 encode_option_u64(event.state_version),
4045 encode_option_string(&event.causal_token),
4046 hex_encode(&event.payload_hash),
4047 hex_encode(&event.payload_ref),
4048 encode_state(&event.payload),
4049 ]));
4050 }
4051 }
4052 }
4053
4054 let mut ledgers: Vec<_> = store.tool_ledger.values().collect();
4055 ledgers.sort_by(|a, b| a.idempotency_key.cmp(&b.idempotency_key));
4056 for entry in ledgers {
4057 lines.push(join_fields(&[
4058 "L".to_string(),
4059 hex_encode(&entry.ledger_id),
4060 hex_encode(&entry.run_id),
4061 hex_encode(&entry.session_id),
4062 hex_encode(&entry.step_id),
4063 hex_encode(&entry.tool_name),
4064 hex_encode(&entry.tool_version),
4065 hex_encode(&entry.tool_call_id),
4066 hex_encode(&entry.idempotency_key),
4067 hex_encode(&entry.causal_token),
4068 hex_encode(&entry.request_hash),
4069 hex_encode(&entry.request_ref),
4070 hex_encode(&entry.status),
4071 encode_option_string(&entry.external_id),
4072 encode_option_string(&entry.response_hash),
4073 encode_option_string(&entry.response_ref),
4074 encode_option_string(&entry.error_type),
4075 encode_option_value(&entry.response),
4076 entry.created_at.to_string(),
4077 entry.updated_at.to_string(),
4078 ]));
4079 }
4080
4081 let mut approvals: Vec<_> = store.approval_requests.values().collect();
4082 approvals.sort_by(|a, b| a.approval_key.cmp(&b.approval_key));
4083 for approval in approvals {
4084 lines.push(join_fields(&[
4085 "A".to_string(),
4086 hex_encode(&approval.approval_id),
4087 hex_encode(&approval.approval_key),
4088 hex_encode(&approval.run_id),
4089 hex_encode(&approval.session_id),
4090 hex_encode(&approval.step_id),
4091 hex_encode(&approval.tool_name),
4092 hex_encode(&approval.risk_level),
4093 hex_encode(&approval.status),
4094 hex_encode(&approval.reason),
4095 hex_encode(&approval.request_hash),
4096 hex_encode(&approval.request_ref),
4097 hex_encode(&approval.requested_by),
4098 encode_option_string(&approval.approved_by),
4099 encode_option_string(&approval.decision_reason),
4100 approval.created_at.to_string(),
4101 approval.updated_at.to_string(),
4102 ]));
4103 }
4104
4105 let mut cost_run_ids: Vec<_> = store.cost_records.keys().collect();
4106 cost_run_ids.sort();
4107 for run_id in cost_run_ids {
4108 if let Some(records) = store.cost_records.get(run_id) {
4109 for record in records {
4110 lines.push(join_fields(&[
4111 "C".to_string(),
4112 hex_encode(&record.cost_id),
4113 hex_encode(&record.run_id),
4114 hex_encode(&record.session_id),
4115 hex_encode(&record.step_id),
4116 hex_encode(&record.category),
4117 hex_encode(&record.name),
4118 record.amount.to_string(),
4119 hex_encode(&record.unit),
4120 encode_state(&record.metadata),
4121 record.created_at.to_string(),
4122 ]));
4123 }
4124 }
4125 }
4126
4127 let mut artifact_run_ids: Vec<_> = store.artifacts.keys().collect();
4128 artifact_run_ids.sort();
4129 for run_id in artifact_run_ids {
4130 if let Some(artifacts) = store.artifacts.get(run_id) {
4131 for artifact in artifacts {
4132 lines.push(join_fields(&[
4133 "F".to_string(),
4134 hex_encode(&artifact.artifact_id),
4135 hex_encode(&artifact.run_id),
4136 encode_option_string(&artifact.step_id),
4137 hex_encode(&artifact.name),
4138 hex_encode(&artifact.blob_hash),
4139 hex_encode(&artifact.blob_ref),
4140 encode_state(&artifact.metadata),
4141 artifact.created_at.to_string(),
4142 ]));
4143 }
4144 }
4145 }
4146
4147 lines.push(String::new());
4148 lines.join("\n")
4149}
4150
4151fn decode_store(body: &str) -> Result<MemoryStore> {
4152 let mut lines = body.lines();
4153 match lines.next() {
4154 Some("AGENTLEDGER_RUST_STORE_V1") => {}
4155 _ => {
4156 return Err(RuntimeError(
4157 "invalid Rust store snapshot header".to_string(),
4158 ))
4159 }
4160 }
4161 let mut store = MemoryStore::new();
4162 for line in lines.filter(|line| !line.trim().is_empty()) {
4163 let fields: Vec<&str> = line.split('\t').collect();
4164 let tag = fields.first().copied().unwrap_or_default();
4165 match tag {
4166 "R" => {
4167 require_len(tag, &fields, 8)?;
4168 let run = Run {
4169 run_id: hex_decode(fields[1])?,
4170 session_id: hex_decode(fields[2])?,
4171 status: hex_decode(fields[3])?,
4172 state_version: parse_u64(fields[4])?,
4173 created_at: parse_f64(fields[5])?,
4174 updated_at: parse_f64(fields[6])?,
4175 state: decode_state(fields[7])?,
4176 };
4177 store.runs.insert(run.run_id.clone(), run);
4178 }
4179 "S" => {
4180 require_len(tag, &fields, 16)?;
4181 let step = Step {
4182 step_id: hex_decode(fields[1])?,
4183 run_id: hex_decode(fields[2])?,
4184 session_id: hex_decode(fields[3])?,
4185 status: hex_decode(fields[4])?,
4186 owner: decode_option_string(fields[5])?,
4187 lease_token: decode_option_string(fields[6])?,
4188 lease_until: decode_option_f64(fields[7])?,
4189 attempt: parse_u64(fields[8])?,
4190 state_version: parse_u64(fields[9])?,
4191 checkpoint_id: decode_option_string(fields[10])?,
4192 last_error_type: decode_option_string(fields[11])?,
4193 last_error: decode_option_string(fields[12])?,
4194 cancelled_at: decode_option_f64(fields[13])?,
4195 created_at: parse_f64(fields[14])?,
4196 updated_at: parse_f64(fields[15])?,
4197 };
4198 store.steps.insert(step.step_id.clone(), step);
4199 }
4200 "E" => {
4201 require_len(tag, &fields, 14)?;
4202 let event = Event {
4203 event_id: hex_decode(fields[1])?,
4204 run_id: hex_decode(fields[2])?,
4205 session_id: decode_option_string(fields[3])?,
4206 step_id: decode_option_string(fields[4])?,
4207 seq: parse_u64(fields[5])?,
4208 event_type: hex_decode(fields[6])?,
4209 timestamp: parse_f64(fields[7])?,
4210 agent_role: decode_option_string(fields[8])?,
4211 state_version: decode_option_u64(fields[9])?,
4212 causal_token: decode_option_string(fields[10])?,
4213 payload_hash: hex_decode(fields[11])?,
4214 payload_ref: hex_decode(fields[12])?,
4215 payload: decode_state(fields[13])?,
4216 };
4217 store
4218 .events
4219 .entry(event.run_id.clone())
4220 .or_default()
4221 .push(event);
4222 }
4223 "L" => {
4224 require_len(tag, &fields, 20)?;
4225 let entry = ToolLedgerEntry {
4226 ledger_id: hex_decode(fields[1])?,
4227 run_id: hex_decode(fields[2])?,
4228 session_id: hex_decode(fields[3])?,
4229 step_id: hex_decode(fields[4])?,
4230 tool_name: hex_decode(fields[5])?,
4231 tool_version: hex_decode(fields[6])?,
4232 tool_call_id: hex_decode(fields[7])?,
4233 idempotency_key: hex_decode(fields[8])?,
4234 causal_token: hex_decode(fields[9])?,
4235 request_hash: hex_decode(fields[10])?,
4236 request_ref: hex_decode(fields[11])?,
4237 status: hex_decode(fields[12])?,
4238 external_id: decode_option_string(fields[13])?,
4239 response_hash: decode_option_string(fields[14])?,
4240 response_ref: decode_option_string(fields[15])?,
4241 error_type: decode_option_string(fields[16])?,
4242 response: decode_option_value(fields[17])?,
4243 created_at: parse_f64(fields[18])?,
4244 updated_at: parse_f64(fields[19])?,
4245 };
4246 store
4247 .tool_ledger
4248 .insert(entry.idempotency_key.clone(), entry);
4249 }
4250 "A" => {
4251 require_len(tag, &fields, 17)?;
4252 let approval = ApprovalRequest {
4253 approval_id: hex_decode(fields[1])?,
4254 approval_key: hex_decode(fields[2])?,
4255 run_id: hex_decode(fields[3])?,
4256 session_id: hex_decode(fields[4])?,
4257 step_id: hex_decode(fields[5])?,
4258 tool_name: hex_decode(fields[6])?,
4259 risk_level: hex_decode(fields[7])?,
4260 status: hex_decode(fields[8])?,
4261 reason: hex_decode(fields[9])?,
4262 request_hash: hex_decode(fields[10])?,
4263 request_ref: hex_decode(fields[11])?,
4264 requested_by: hex_decode(fields[12])?,
4265 approved_by: decode_option_string(fields[13])?,
4266 decision_reason: decode_option_string(fields[14])?,
4267 created_at: parse_f64(fields[15])?,
4268 updated_at: parse_f64(fields[16])?,
4269 };
4270 store
4271 .approval_requests
4272 .insert(approval.approval_key.clone(), approval);
4273 }
4274 "C" => {
4275 require_len(tag, &fields, 11)?;
4276 let record = CostRecord {
4277 cost_id: hex_decode(fields[1])?,
4278 run_id: hex_decode(fields[2])?,
4279 session_id: hex_decode(fields[3])?,
4280 step_id: hex_decode(fields[4])?,
4281 category: hex_decode(fields[5])?,
4282 name: hex_decode(fields[6])?,
4283 amount: parse_f64(fields[7])?,
4284 unit: hex_decode(fields[8])?,
4285 metadata: decode_state(fields[9])?,
4286 created_at: parse_f64(fields[10])?,
4287 };
4288 store
4289 .cost_records
4290 .entry(record.run_id.clone())
4291 .or_default()
4292 .push(record);
4293 }
4294 "F" => {
4295 require_len(tag, &fields, 9)?;
4296 let artifact = Artifact {
4297 artifact_id: hex_decode(fields[1])?,
4298 run_id: hex_decode(fields[2])?,
4299 step_id: decode_option_string(fields[3])?,
4300 name: hex_decode(fields[4])?,
4301 blob_hash: hex_decode(fields[5])?,
4302 blob_ref: hex_decode(fields[6])?,
4303 metadata: decode_state(fields[7])?,
4304 created_at: parse_f64(fields[8])?,
4305 };
4306 store
4307 .artifacts
4308 .entry(artifact.run_id.clone())
4309 .or_default()
4310 .push(artifact);
4311 }
4312 _ => {
4313 return Err(RuntimeError(format!(
4314 "unknown Rust store snapshot row: {tag}"
4315 )))
4316 }
4317 }
4318 }
4319 Ok(store)
4320}
4321
4322fn join_fields(fields: &[String]) -> String {
4323 fields.join("\t")
4324}
4325
4326fn require_len(tag: &str, fields: &[&str], expected: usize) -> Result<()> {
4327 if fields.len() != expected {
4328 return Err(RuntimeError(format!(
4329 "invalid {tag} row: expected {expected} fields, got {}",
4330 fields.len()
4331 )));
4332 }
4333 Ok(())
4334}
4335
4336fn encode_option_string(value: &Option<String>) -> String {
4337 value
4338 .as_ref()
4339 .map(|item| hex_encode(item))
4340 .unwrap_or_else(|| "-".to_string())
4341}
4342
4343fn decode_option_string(value: &str) -> Result<Option<String>> {
4344 if value == "-" {
4345 Ok(None)
4346 } else {
4347 Ok(Some(hex_decode(value)?))
4348 }
4349}
4350
4351fn encode_option_f64(value: Option<f64>) -> String {
4352 value
4353 .map(|item| item.to_string())
4354 .unwrap_or_else(|| "-".to_string())
4355}
4356
4357fn decode_option_f64(value: &str) -> Result<Option<f64>> {
4358 if value == "-" {
4359 Ok(None)
4360 } else {
4361 Ok(Some(parse_f64(value)?))
4362 }
4363}
4364
4365fn encode_option_u64(value: Option<u64>) -> String {
4366 value
4367 .map(|item| item.to_string())
4368 .unwrap_or_else(|| "-".to_string())
4369}
4370
4371fn decode_option_u64(value: &str) -> Result<Option<u64>> {
4372 if value == "-" {
4373 Ok(None)
4374 } else {
4375 Ok(Some(parse_u64(value)?))
4376 }
4377}
4378
4379fn encode_option_value(value: &Option<Value>) -> String {
4380 value
4381 .as_ref()
4382 .map(encode_value)
4383 .unwrap_or_else(|| "-".to_string())
4384}
4385
4386fn decode_option_value(value: &str) -> Result<Option<Value>> {
4387 if value == "-" {
4388 Ok(None)
4389 } else {
4390 Ok(Some(decode_value(value)?))
4391 }
4392}
4393
4394fn encode_state(state: &State) -> String {
4395 encode_value(&Value::Object(state.clone()))
4396}
4397
4398fn decode_state(encoded: &str) -> Result<State> {
4399 match decode_value(encoded)? {
4400 Value::Object(state) => Ok(state),
4401 _ => Err(RuntimeError("encoded state was not an object".to_string())),
4402 }
4403}
4404
4405fn encode_value(value: &Value) -> String {
4406 match value {
4407 Value::Null => "Z".to_string(),
4408 Value::Bool(true) => "T".to_string(),
4409 Value::Bool(false) => "F".to_string(),
4410 Value::Number(item) => format!("N{}:", hex_encode(&item.to_string())),
4411 Value::String(item) => format!("S{}:", hex_encode(item)),
4412 Value::Object(state) => {
4413 let mut keys: Vec<_> = state.keys().collect();
4414 keys.sort();
4415 let mut out = format!("O{}:", keys.len());
4416 for key in keys {
4417 out.push_str(&hex_encode(key));
4418 out.push(':');
4419 out.push_str(&encode_value(&state[key]));
4420 }
4421 out
4422 }
4423 Value::Array(values) => {
4424 let mut out = format!("A{}:", values.len());
4425 for value in values {
4426 out.push_str(&encode_value(value));
4427 }
4428 out
4429 }
4430 }
4431}
4432
4433fn decode_value(encoded: &str) -> Result<Value> {
4434 let (value, index) = parse_value(encoded, 0)?;
4435 if index != encoded.len() {
4436 return Err(RuntimeError("trailing bytes in encoded value".to_string()));
4437 }
4438 Ok(value)
4439}
4440
4441fn parse_value(input: &str, index: usize) -> Result<(Value, usize)> {
4442 let bytes = input.as_bytes();
4443 let tag = *bytes
4444 .get(index)
4445 .ok_or_else(|| RuntimeError("unexpected end of encoded value".to_string()))?
4446 as char;
4447 match tag {
4448 'Z' => Ok((Value::Null, index + 1)),
4449 'T' => Ok((Value::Bool(true), index + 1)),
4450 'F' => Ok((Value::Bool(false), index + 1)),
4451 'N' => {
4452 let (hex, next) = read_until_colon(input, index + 1)?;
4453 Ok((Value::Number(parse_f64(&hex_decode(hex)?)?), next))
4454 }
4455 'S' => {
4456 let (hex, next) = read_until_colon(input, index + 1)?;
4457 Ok((Value::String(hex_decode(hex)?), next))
4458 }
4459 'O' => {
4460 let (count_text, mut next) = read_until_colon(input, index + 1)?;
4461 let count = count_text
4462 .parse::<usize>()
4463 .map_err(|err| RuntimeError(err.to_string()))?;
4464 let mut state = State::new();
4465 for _ in 0..count {
4466 let (key_hex, after_key) = read_until_colon(input, next)?;
4467 let key = hex_decode(key_hex)?;
4468 let (value, after_value) = parse_value(input, after_key)?;
4469 state.insert(key, value);
4470 next = after_value;
4471 }
4472 Ok((Value::Object(state), next))
4473 }
4474 'A' => {
4475 let (count_text, mut next) = read_until_colon(input, index + 1)?;
4476 let count = count_text
4477 .parse::<usize>()
4478 .map_err(|err| RuntimeError(err.to_string()))?;
4479 let mut values = Vec::with_capacity(count);
4480 for _ in 0..count {
4481 let (value, after_value) = parse_value(input, next)?;
4482 values.push(value);
4483 next = after_value;
4484 }
4485 Ok((Value::Array(values), next))
4486 }
4487 _ => Err(RuntimeError(format!("unknown encoded value tag: {tag}"))),
4488 }
4489}
4490
4491fn read_until_colon(input: &str, index: usize) -> Result<(&str, usize)> {
4492 let rest = input
4493 .get(index..)
4494 .ok_or_else(|| RuntimeError("invalid encoded value index".to_string()))?;
4495 let offset = rest
4496 .find(':')
4497 .ok_or_else(|| RuntimeError("missing encoded value delimiter".to_string()))?;
4498 Ok((&rest[..offset], index + offset + 1))
4499}
4500
4501fn hex_encode(value: &str) -> String {
4502 value
4503 .as_bytes()
4504 .iter()
4505 .map(|byte| format!("{byte:02x}"))
4506 .collect()
4507}
4508
4509fn hex_decode(value: &str) -> Result<String> {
4510 if value.len() % 2 != 0 {
4511 return Err(RuntimeError("invalid hex string length".to_string()));
4512 }
4513 let mut bytes = Vec::with_capacity(value.len() / 2);
4514 for index in (0..value.len()).step_by(2) {
4515 let byte = u8::from_str_radix(&value[index..index + 2], 16)
4516 .map_err(|err| RuntimeError(err.to_string()))?;
4517 bytes.push(byte);
4518 }
4519 String::from_utf8(bytes).map_err(|err| RuntimeError(err.to_string()))
4520}
4521
4522fn parse_f64(value: &str) -> Result<f64> {
4523 value
4524 .parse::<f64>()
4525 .map_err(|err| RuntimeError(err.to_string()))
4526}
4527
4528fn parse_u64(value: &str) -> Result<u64> {
4529 value
4530 .parse::<u64>()
4531 .map_err(|err| RuntimeError(err.to_string()))
4532}
4533
4534fn merge_patch(base: &State, patch: &State) -> State {
4535 let mut out = base.clone();
4536 for (key, value) in patch {
4537 match value {
4538 Value::Null => {
4539 out.remove(key);
4540 }
4541 Value::Object(patch_map) => {
4542 if let Some(Value::Object(base_map)) = out.get(key) {
4543 out.insert(key.clone(), Value::Object(merge_patch(base_map, patch_map)));
4544 } else {
4545 out.insert(key.clone(), value.clone());
4546 }
4547 }
4548 _ => {
4549 out.insert(key.clone(), value.clone());
4550 }
4551 }
4552 }
4553 out
4554}
4555
4556fn add_cost(summary: &mut CostSummary, record: &CostRecord) {
4557 if (record.category == "tool" || record.category == "tool_shadow") && record.unit == "call" {
4558 summary.tool_calls += record.amount;
4559 }
4560 if record.category == "model" && record.unit == "token" {
4561 summary.model_tokens += record.amount;
4562 }
4563 if record.unit == "usd" {
4564 summary.total_usd += record.amount;
4565 }
4566 let key = format!("{}:{}", record.category, record.unit);
4567 *summary.by_category.entry(key).or_insert(0.0) += record.amount;
4568}
4569
4570pub fn validate_tool_schema(schema: &Value, value: &Value, path: &str) -> Result<()> {
4571 let schema = match schema {
4572 Value::Object(schema) => schema,
4573 _ => return Err(RuntimeError(format!("{path} schema must be object"))),
4574 };
4575 if let Some(expected) = schema.get("const") {
4576 if expected != value {
4577 return Err(RuntimeError(format!("{path} expected const")));
4578 }
4579 }
4580 if let Some(Value::Array(items)) = schema.get("enum") {
4581 if !items.iter().any(|item| item == value) {
4582 return Err(RuntimeError(format!("{path} value not in enum")));
4583 }
4584 }
4585 let Some(Value::String(kind)) = schema.get("type") else {
4586 return Ok(());
4587 };
4588 match kind.as_str() {
4589 "object" => {
4590 let Value::Object(object) = value else {
4591 return Err(RuntimeError(format!("{path} expected object")));
4592 };
4593 if let Some(Value::Array(required)) = schema.get("required") {
4594 for item in required {
4595 if let Value::String(key) = item {
4596 if !object.contains_key(key) {
4597 return Err(RuntimeError(format!("{path}.{key} is required")));
4598 }
4599 }
4600 }
4601 }
4602 let properties = match schema.get("properties") {
4603 Some(Value::Object(properties)) => properties,
4604 _ => return Ok(()),
4605 };
4606 for (key, child_schema) in properties {
4607 if let Some(child) = object.get(key) {
4608 validate_tool_schema(child_schema, child, &format!("{path}.{key}"))?;
4609 }
4610 }
4611 if schema.get("additionalProperties") == Some(&Value::Bool(false)) {
4612 for key in object.keys() {
4613 if !properties.contains_key(key) {
4614 return Err(RuntimeError(format!("{path}.{key} is not allowed")));
4615 }
4616 }
4617 }
4618 }
4619 "string" => {
4620 let Value::String(text) = value else {
4621 return Err(RuntimeError(format!("{path} expected string")));
4622 };
4623 if let Some(Value::Number(min)) = schema.get("minLength") {
4624 if (text.len() as f64) < *min {
4625 return Err(RuntimeError(format!("{path} shorter than minLength")));
4626 }
4627 }
4628 if let Some(Value::Number(max)) = schema.get("maxLength") {
4629 if (text.len() as f64) > *max {
4630 return Err(RuntimeError(format!("{path} longer than maxLength")));
4631 }
4632 }
4633 }
4634 "number" | "integer" => {
4635 let Value::Number(number) = value else {
4636 return Err(RuntimeError(format!("{path} expected number")));
4637 };
4638 if kind == "integer" && number.fract() != 0.0 {
4639 return Err(RuntimeError(format!("{path} expected integer")));
4640 }
4641 if let Some(Value::Number(min)) = schema.get("minimum") {
4642 if number < min {
4643 return Err(RuntimeError(format!("{path} below minimum")));
4644 }
4645 }
4646 if let Some(Value::Number(max)) = schema.get("maximum") {
4647 if number > max {
4648 return Err(RuntimeError(format!("{path} above maximum")));
4649 }
4650 }
4651 }
4652 "boolean" => {
4653 if !matches!(value, Value::Bool(_)) {
4654 return Err(RuntimeError(format!("{path} expected boolean")));
4655 }
4656 }
4657 _ => {}
4658 }
4659 Ok(())
4660}
4661
4662fn is_high_risk(risk: &str) -> bool {
4663 matches!(
4664 risk,
4665 "high" | "destructive" | "sensitive" | "financial_or_legal"
4666 )
4667}
4668
4669fn is_media_kind(kind: &str) -> bool {
4670 matches!(
4671 kind,
4672 "image"
4673 | "audio"
4674 | "video"
4675 | "frame"
4676 | "audio_segment"
4677 | "video_segment"
4678 | "transcript"
4679 | "embedding"
4680 | "derived"
4681 )
4682}
4683
4684fn stream_chunk_to_state(chunk: StreamChunkRef) -> State {
4685 let mut state = State::new();
4686 state.insert(
4687 "schema_version".to_string(),
4688 Value::String(STREAM_SCHEMA_VERSION.to_string()),
4689 );
4690 state.insert("stream_id".to_string(), Value::String(chunk.stream_id));
4691 state.insert("chunk_id".to_string(), Value::String(chunk.chunk_id));
4692 state.insert("offset".to_string(), chunk.offset);
4693 if let Some(content_ref) = chunk.content_ref {
4694 state.insert("content_ref".to_string(), Value::String(content_ref));
4695 }
4696 if let Some(content_hash) = chunk.content_hash {
4697 state.insert("content_hash".to_string(), Value::String(content_hash));
4698 }
4699 if let Some(sequence) = chunk.sequence {
4700 state.insert("sequence".to_string(), Value::Number(sequence));
4701 }
4702 if let Some(event_time) = chunk.event_time {
4703 state.insert("event_time".to_string(), Value::Number(event_time));
4704 }
4705 if !chunk.metadata.is_empty() {
4706 state.insert("metadata".to_string(), Value::Object(chunk.metadata));
4707 }
4708 state
4709}
4710
4711fn media_artifacts_from(artifacts: &[Artifact]) -> Vec<State> {
4712 artifacts
4713 .iter()
4714 .filter_map(
4715 |artifact| match artifact.metadata.get("agentledger_media") {
4716 Some(Value::Object(metadata)) => {
4717 let mut row = State::new();
4718 row.insert(
4719 "artifact_id".to_string(),
4720 Value::String(artifact.artifact_id.clone()),
4721 );
4722 row.insert("name".to_string(), Value::String(artifact.name.clone()));
4723 row.insert(
4724 "blob_hash".to_string(),
4725 Value::String(artifact.blob_hash.clone()),
4726 );
4727 row.insert(
4728 "blob_ref".to_string(),
4729 Value::String(artifact.blob_ref.clone()),
4730 );
4731 for key in ["kind", "uri", "content_ref", "metadata", "lineage"] {
4732 if let Some(value) = metadata.get(key) {
4733 row.insert(key.to_string(), value.clone());
4734 }
4735 }
4736 Some(row)
4737 }
4738 _ => None,
4739 },
4740 )
4741 .collect()
4742}
4743
4744fn stream_checkpoints_from(artifacts: &[Artifact]) -> Vec<State> {
4745 artifacts
4746 .iter()
4747 .filter_map(
4748 |artifact| match artifact.metadata.get("agentledger_stream") {
4749 Some(Value::Object(metadata)) => {
4750 let mut row = State::new();
4751 row.insert(
4752 "artifact_id".to_string(),
4753 Value::String(artifact.artifact_id.clone()),
4754 );
4755 row.insert("name".to_string(), Value::String(artifact.name.clone()));
4756 row.insert(
4757 "blob_hash".to_string(),
4758 Value::String(artifact.blob_hash.clone()),
4759 );
4760 row.insert(
4761 "blob_ref".to_string(),
4762 Value::String(artifact.blob_ref.clone()),
4763 );
4764 for key in [
4765 "stream_id",
4766 "consumer_id",
4767 "offset",
4768 "watermark",
4769 "chunk",
4770 "partial_result_ref",
4771 "backpressure",
4772 ] {
4773 if let Some(value) = metadata.get(key) {
4774 row.insert(key.to_string(), value.clone());
4775 }
4776 }
4777 Some(row)
4778 }
4779 _ => None,
4780 },
4781 )
4782 .collect()
4783}
4784
4785fn classify_runtime_error(message: &str) -> &'static str {
4786 if message.contains("budget exceeded") || message.contains("budget") {
4787 "BudgetExceededError"
4788 } else if message.contains("sandbox executor") {
4789 "SandboxUnavailableError"
4790 } else if message.contains("high-risk") || message.contains("denied") {
4791 "PermissionDeniedError"
4792 } else {
4793 "RuntimeError"
4794 }
4795}
4796
4797fn failure_source(error_type: &str) -> &'static str {
4798 match error_type {
4799 "BudgetExceededError" => "budget",
4800 "SandboxUnavailableError" => "sandbox",
4801 "PermissionDeniedError" | "ApprovalDenied" => "policy",
4802 _ => "agent",
4803 }
4804}
4805
4806fn is_failure_event(kind: &str) -> bool {
4807 matches!(
4808 kind,
4809 "failure_classified"
4810 | "error_raised"
4811 | "step_failed"
4812 | "step_retry_scheduled"
4813 | "step_waiting_human"
4814 | "lease_expired"
4815 | "run_cancel_requested"
4816 | "run_cancelled"
4817 | "model_call_failed"
4818 | "tool_call_failed"
4819 | "tool_approval_required"
4820 | "budget_check_failed"
4821 )
4822}
4823
4824fn failure_category(text: &str, fallback: &str) -> String {
4825 let lower = text.to_lowercase();
4826 for category in ["sandbox", "budget", "policy", "model", "tool", "runtime"] {
4827 if lower.contains(category) {
4828 return category.to_string();
4829 }
4830 }
4831 if lower.contains("approval") || lower.contains("permission") || lower.contains("denied") {
4832 return "policy".to_string();
4833 }
4834 if lower.contains("lease") || lower.contains("worker") {
4835 return "runtime".to_string();
4836 }
4837 if lower.contains("cancel") {
4838 return "cancellation".to_string();
4839 }
4840 fallback.to_string()
4841}
4842
4843fn event_failure_category(event: &Event) -> String {
4844 match event.event_type.as_str() {
4845 "model_call_failed" => "model".to_string(),
4846 "tool_call_failed" | "tool_call_blocked" | "tool_approval_required" => "tool".to_string(),
4847 "run_cancel_requested" | "run_cancelled" | "step_cancelled" => "cancellation".to_string(),
4848 "lease_expired" => "runtime".to_string(),
4849 "step_retry_scheduled" => "retry".to_string(),
4850 "step_waiting_human" => "approval".to_string(),
4851 _ => failure_category(
4852 &format!(
4853 "{} {} {} {}",
4854 event.event_type,
4855 state_value_string(&event.payload, "error_type"),
4856 state_value_string(&event.payload, "error"),
4857 state_value_string(&event.payload, "reason")
4858 ),
4859 "agent",
4860 ),
4861 }
4862}
4863
4864fn event_failure_status(kind: &str, run_status: &str) -> String {
4865 match kind {
4866 "step_failed" | "run_cancelled" | "step_cancelled" => "terminal".to_string(),
4867 "tool_call_blocked" => "blocked".to_string(),
4868 "step_retry_scheduled" | "lease_expired" => "recovery_scheduled".to_string(),
4869 "step_waiting_human" | "tool_approval_required" => "waiting_human".to_string(),
4870 "failure_classified" => "classified".to_string(),
4871 "error_raised" if run_status == "failed" => "terminal".to_string(),
4872 _ => "failed".to_string(),
4873 }
4874}
4875
4876fn event_recoverability(kind: &str, run_status: &str) -> String {
4877 if run_status == "failed" && matches!(kind, "step_failed" | "run_cancelled" | "step_cancelled") {
4878 return "terminal".to_string();
4879 }
4880 match kind {
4881 "step_retry_scheduled" | "lease_expired" => "auto_retry".to_string(),
4882 "step_waiting_human" | "tool_approval_required" => "human_required".to_string(),
4883 "tool_call_blocked" => "manual_intervention".to_string(),
4884 _ => "unknown".to_string(),
4885 }
4886}
4887
4888fn event_retryability(kind: &str) -> String {
4889 match kind {
4890 "step_retry_scheduled" | "lease_expired" => "retryable".to_string(),
4891 "tool_call_blocked" | "run_cancelled" | "step_cancelled" => "not_retryable".to_string(),
4892 _ => "unknown".to_string(),
4893 }
4894}
4895
4896fn owner_for_failure_category(category: &str) -> String {
4897 match category {
4898 "tool" | "model" | "policy" | "sandbox" | "budget" | "runtime" => category.to_string(),
4899 "approval" | "cancellation" | "retry" => "runtime".to_string(),
4900 _ => "agent".to_string(),
4901 }
4902}
4903
4904fn first_text(values: &[Option<&str>]) -> String {
4905 for value in values {
4906 if let Some(text) = value {
4907 if !text.is_empty() {
4908 return (*text).to_string();
4909 }
4910 }
4911 }
4912 "failure signal".to_string()
4913}
4914
4915fn ref_state(kind: &str, value: &str) -> State {
4916 if value.is_empty() {
4917 return State::new();
4918 }
4919 state(&[("kind", kind.into()), ("value", value.into())])
4920}
4921
4922fn state_value_string(row: &State, key: &str) -> String {
4923 match row.get(key) {
4924 Some(Value::String(value)) => value.clone(),
4925 Some(Value::Number(value)) => value.to_string(),
4926 Some(Value::Bool(value)) => value.to_string(),
4927 Some(value) => format_value(value),
4928 None => String::new(),
4929 }
4930}
4931
4932fn state_string(row: &State, key: &str) -> Option<String> {
4933 let value = state_value_string(row, key);
4934 if value.is_empty() {
4935 None
4936 } else {
4937 Some(value)
4938 }
4939}
4940
4941fn state_number(row: &State, key: &str) -> f64 {
4942 match row.get(key) {
4943 Some(Value::Number(value)) => *value,
4944 _ => 0.0,
4945 }
4946}
4947
4948fn usage_total_tokens(usage: &State) -> f64 {
4949 for key in ["total_tokens", "totalTokens", "tokens"] {
4950 let value = state_number(usage, key);
4951 if value > 0.0 {
4952 return value;
4953 }
4954 }
4955 state_number(usage, "input_tokens")
4956 + state_number(usage, "prompt_tokens")
4957 + state_number(usage, "inputTokens")
4958 + state_number(usage, "output_tokens")
4959 + state_number(usage, "completion_tokens")
4960 + state_number(usage, "outputTokens")
4961}
4962
4963fn state_array_len(row: &State, key: &str) -> usize {
4964 match row.get(key) {
4965 Some(Value::Array(values)) => values.len(),
4966 _ => 0,
4967 }
4968}
4969
4970fn dedupe_states(rows: Vec<State>, key: &str) -> Vec<State> {
4971 let mut out = Vec::new();
4972 let mut seen = std::collections::HashSet::new();
4973 for row in rows {
4974 let value = state_value_string(&row, key);
4975 if value.is_empty() || seen.contains(&value) {
4976 continue;
4977 }
4978 seen.insert(value);
4979 out.push(row);
4980 }
4981 out
4982}
4983
4984fn count_state_field(rows: &[State], key: &str, expected: &str) -> usize {
4985 rows.iter()
4986 .filter(|row| state_value_string(row, key) == expected)
4987 .count()
4988}
4989
4990fn count_recoverable_failures(rows: &[State]) -> usize {
4991 rows.iter()
4992 .filter(|row| matches!(state_value_string(row, "recoverability").as_str(), "auto_retry" | "recoverable" | "manual_verification" | "human_required"))
4993 .count()
4994}
4995
4996fn slug(value: &str) -> String {
4997 let mut out = String::new();
4998 let mut previous_dash = false;
4999 for ch in value.chars() {
5000 if ch.is_ascii_alphanumeric() {
5001 out.push(ch.to_ascii_lowercase());
5002 previous_dash = false;
5003 } else if !previous_dash {
5004 out.push('-');
5005 previous_dash = true;
5006 }
5007 }
5008 let trimmed = out.trim_matches('-').to_string();
5009 if trimmed.is_empty() {
5010 "unknown".to_string()
5011 } else {
5012 trimmed
5013 }
5014}
5015
5016fn format_state(state: &State) -> String {
5017 let mut keys: Vec<&String> = state.keys().collect();
5018 keys.sort();
5019 keys.into_iter()
5020 .map(|key| format!("{}={}", key, format_value(&state[key])))
5021 .collect::<Vec<_>>()
5022 .join(",")
5023}
5024
5025fn format_value(value: &Value) -> String {
5026 match value {
5027 Value::Null => "null".to_string(),
5028 Value::Bool(value) => value.to_string(),
5029 Value::Number(value) => value.to_string(),
5030 Value::String(value) => format!("\"{}\"", value),
5031 Value::Object(value) => format!("{{{}}}", format_state(value)),
5032 Value::Array(values) => format!(
5033 "[{}]",
5034 values
5035 .iter()
5036 .map(format_value)
5037 .collect::<Vec<_>>()
5038 .join(",")
5039 ),
5040 }
5041}
5042
5043fn stable_hash(input: &str) -> String {
5044 let mut hash: u64 = 0xcbf29ce484222325;
5045 for byte in input.as_bytes() {
5046 hash ^= *byte as u64;
5047 hash = hash.wrapping_mul(0x100000001b3);
5048 }
5049 format!("{hash:016x}")
5050}
5051
5052fn new_id(prefix: &str) -> String {
5053 let value = ID_COUNTER.fetch_add(1, Ordering::Relaxed);
5054 format!("{prefix}_{value:016x}")
5055}
5056
5057fn now_seconds() -> f64 {
5058 SystemTime::now()
5059 .duration_since(UNIX_EPOCH)
5060 .unwrap()
5061 .as_secs_f64()
5062}
5063
5064#[cfg(test)]
5065mod tests {
5066 use super::*;
5067
5068 fn state(items: &[(&str, Value)]) -> State {
5069 items
5070 .iter()
5071 .map(|(key, value)| ((*key).to_string(), value.clone()))
5072 .collect()
5073 }
5074
5075 fn event_exists(events: &[Event], event_type: &str) -> bool {
5076 events.iter().any(|event| event.event_type == event_type)
5077 }
5078
5079 fn claim_context(
5080 runtime: &mut Runtime,
5081 run_id: &str,
5082 worker: &str,
5083 role: &str,
5084 ) -> AgentContext {
5085 let claim = runtime.store.claim_step(worker, run_id, 60.0).unwrap();
5086 let (_state, version, session_id) = runtime.store.load_state(run_id).unwrap();
5087 let mut payload = State::new();
5088 payload.insert("agent_role".to_string(), Value::String(role.to_string()));
5089 runtime.store.append_event(
5090 run_id,
5091 Some(&session_id),
5092 Some(&claim.step_id),
5093 "agent_started",
5094 payload,
5095 Some(role),
5096 Some(version),
5097 None,
5098 );
5099 AgentContext {
5100 run_id: run_id.to_string(),
5101 session_id,
5102 step_id: claim.step_id,
5103 agent_role: role.to_string(),
5104 lease_token: claim.lease_token,
5105 attempt: claim.attempt,
5106 state_version: version,
5107 pending_patch: State::new(),
5108 }
5109 }
5110
5111 #[test]
5112 fn runtime_creates_evidence_and_replay() {
5113 let mut runtime = Runtime::new();
5114 runtime.register_tool(ToolSpec::new(
5115 "docs.echo",
5116 Box::new(|args| Ok(Value::Object(state(&[("echo", args["text"].clone())])))),
5117 ));
5118 let (run_id, _) = runtime.create_run(state(&[("input", "hello".into())]));
5119 let ok = runtime
5120 .run_once(&run_id, "worker-a", "Researcher", 60.0, |ctx, state| {
5121 let mut result = State::new();
5122 result.insert("from_state".to_string(), state["input"].clone());
5123 ctx.write_state("tool_result", Value::Object(result));
5124 Ok(())
5125 })
5126 .unwrap();
5127 assert!(ok);
5128 let bundle = export_evidence(&runtime.store, &run_id).unwrap();
5129 assert_eq!(bundle.schema_version, "agentledger.evidence.v1");
5130 let summary = replay(&runtime.store, &run_id).unwrap();
5131 assert!(summary.replay_safe);
5132 assert_eq!(summary.event_count, bundle.events.len());
5133 }
5134
5135 #[test]
5136 fn local_snapshot_store_round_trips_completed_run() {
5137 let mut runtime = Runtime::new();
5138 let (run_id, _) = runtime.create_run(state(&[("input", "hello".into())]));
5139 runtime
5140 .run_once(&run_id, "worker-a", "Researcher", 60.0, |ctx, state| {
5141 let mut result = State::new();
5142 result.insert("echo".to_string(), state["input"].clone());
5143 ctx.write_state("tool_result", Value::Object(result));
5144 Ok(())
5145 })
5146 .unwrap();
5147 let path =
5148 std::env::temp_dir().join(format!("agentledger-rust-{}.store", new_id("snapshot")));
5149 runtime.store.save_to_path(&path).unwrap();
5150
5151 let reopened = MemoryStore::load_from_path(&path).unwrap();
5152 let final_state = reopened.final_state(&run_id).unwrap();
5153 assert_eq!(
5154 final_state.get("tool_result"),
5155 Some(&Value::Object(state(&[("echo", "hello".into())])))
5156 );
5157 let bundle = export_evidence(&reopened, &run_id).unwrap();
5158 assert_eq!(reopened.steps(&run_id).len(), 1);
5159 assert_eq!(
5160 replay(&reopened, &run_id).unwrap().event_count,
5161 bundle.events.len()
5162 );
5163 let _ = std::fs::remove_file(path);
5164 }
5165
5166 #[test]
5167 fn local_blob_store_round_trips_json_values() {
5168 let root = std::env::temp_dir().join(format!("agentledger-rust-blobs-{}", new_id("blob")));
5169 let blobs = LocalBlobStore::open(&root).unwrap();
5170 let value = Value::Object(state(&[(
5171 "hello",
5172 Value::Object(state(&[("nested", "world".into())])),
5173 )]));
5174 let first = blobs.put_json(&value).unwrap();
5175 let second = blobs.put_json(&value).unwrap();
5176 assert!(first.0.starts_with("sha256:"));
5177 assert!(first.1.starts_with("blob://sha256/"));
5178 assert_eq!(first, second);
5179 assert_eq!(blobs.get_json(&first.1).unwrap(), value);
5180 assert!(blobs.get_json("unsupported://blob").is_err());
5181 let _ = std::fs::remove_dir_all(root);
5182 }
5183
5184 #[test]
5185 fn tool_schema_validation_rejects_invalid_input() {
5186 let input_schema = Value::Object(state(&[
5187 ("type", "object".into()),
5188 (
5189 "required",
5190 Value::Array(vec![Value::String("text".to_string())]),
5191 ),
5192 ("additionalProperties", false.into()),
5193 (
5194 "properties",
5195 Value::Object(state(&[(
5196 "text",
5197 Value::Object(state(&[("type", "string".into())])),
5198 )])),
5199 ),
5200 ]));
5201 let mut runtime = Runtime::new();
5202 runtime.register_tool(
5203 ToolSpec::new(
5204 "docs.echo",
5205 Box::new(|args| Ok(Value::Object(state(&[("echo", args["text"].clone())])))),
5206 )
5207 .input_schema(input_schema),
5208 );
5209 let (run_id, _) = runtime.create_run(State::new());
5210 let ctx = claim_context(&mut runtime, &run_id, "worker", "SchemaAgent");
5211 assert!(runtime.call_tool(&ctx, "docs.echo", State::new()).is_err());
5212 assert!(runtime.store.events(&run_id).iter().any(|event| {
5213 event.event_type == "tool_call_failed"
5214 && event.payload.get("phase")
5215 == Some(&Value::String("input_validation".to_string()))
5216 }));
5217 }
5218
5219 #[test]
5220 fn tool_ledger_reuses_side_effect_after_retry() {
5221 let mut runtime = Runtime::new();
5222 runtime.register_tool(
5223 ToolSpec::new(
5224 "github.create_pr",
5225 Box::new(|args| {
5226 Ok(Value::Object(state(&[
5227 ("external_id", "pr-123".into()),
5228 ("title", args["title"].clone()),
5229 ])))
5230 }),
5231 )
5232 .side_effect("external")
5233 .idempotency_required(true),
5234 );
5235 let (run_id, _) = runtime.create_run(state(&[("title", "runtime parity".into())]));
5236 let ctx = claim_context(&mut runtime, &run_id, "worker-a", "Coder");
5237 let first = runtime
5238 .call_tool(
5239 &ctx,
5240 "github.create_pr",
5241 state(&[("title", "runtime parity".into())]),
5242 )
5243 .unwrap();
5244 runtime
5245 .store
5246 .mark_retry(&run_id, &ctx.step_id, "RetryableAgentError", "retryable");
5247 let ctx2 = claim_context(&mut runtime, &run_id, "worker-b", "Coder");
5248 let second = runtime
5249 .call_tool(
5250 &ctx2,
5251 "github.create_pr",
5252 state(&[("title", "runtime parity".into())]),
5253 )
5254 .unwrap();
5255 assert_eq!(first, second);
5256 assert_eq!(runtime.store.ledger(&run_id).len(), 1);
5257 }
5258
5259 #[test]
5260 fn failed_no_effect_ledger_entry_allows_retry() {
5261 let calls = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
5262 let calls_for_tool = calls.clone();
5263 let mut runtime = Runtime::new();
5264 runtime.register_tool(
5265 ToolSpec::new(
5266 "payments.charge",
5267 Box::new(move |_| {
5268 let attempt = calls_for_tool.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
5269 if attempt == 0 {
5270 return Err(RuntimeError("provider timeout".to_string()));
5271 }
5272 Ok(Value::Object(state(&[("external_id", "charge-2".into())])))
5273 }),
5274 )
5275 .side_effect("external")
5276 .idempotency_required(true),
5277 );
5278 let (run_id, _) = runtime.create_run(State::new());
5279 let ctx = claim_context(&mut runtime, &run_id, "worker-a", "BillingAgent");
5280 let err = runtime
5281 .call_tool(&ctx, "payments.charge", state(&[("amount", 10.0.into())]))
5282 .unwrap_err();
5283 assert_eq!(err.0, "retryable");
5284 let key = runtime.store.ledger(&run_id)[0].idempotency_key.clone();
5285 runtime
5286 .store
5287 .mark_retry(&run_id, &ctx.step_id, "RuntimeError", "provider timeout");
5288 runtime.store.update_ledger(
5289 &key,
5290 "FAILED_NO_EFFECT",
5291 None,
5292 None,
5293 Some("ManualNoEffect".to_string()),
5294 );
5295 let ctx2 = claim_context(&mut runtime, &run_id, "worker-b", "BillingAgent");
5296 let result = runtime
5297 .call_tool(&ctx2, "payments.charge", state(&[("amount", 10.0.into())]))
5298 .unwrap();
5299 assert_eq!(calls.load(std::sync::atomic::Ordering::SeqCst), 2);
5300 assert_eq!(runtime.store.ledger(&run_id)[0].status, "SUCCEEDED");
5301 assert_eq!(runtime.store.ledger(&run_id)[0].external_id.as_deref(), Some("charge-2"));
5302 assert_eq!(
5303 match result {
5304 Value::Object(row) => row.get("external_id").cloned(),
5305 _ => None,
5306 },
5307 Some(Value::String("charge-2".to_string()))
5308 );
5309 }
5310
5311 #[test]
5312 fn policy_denies_unapproved_high_risk_tool() {
5313 let mut runtime = Runtime::new();
5314 runtime.register_tool(
5315 ToolSpec::new("repo.write", Box::new(|_| Ok(Value::Bool(true)))).risk_level("high"),
5316 );
5317 let (run_id, _) = runtime.create_run(State::new());
5318 let ctx = claim_context(&mut runtime, &run_id, "worker", "Reviewer");
5319 let err = runtime
5320 .call_tool(&ctx, "repo.write", state(&[("path", "README.md".into())]))
5321 .unwrap_err();
5322 assert!(err.0.contains("high-risk"));
5323 assert!(runtime
5324 .store
5325 .events(&run_id)
5326 .iter()
5327 .any(|event| event.event_type == "tool_permission_decided"
5328 && event.payload.get("allowed") == Some(&Value::Bool(false))));
5329 }
5330
5331 #[test]
5332 fn approval_pauses_and_resumes_step() {
5333 let mut runtime = Runtime::new();
5334 runtime.register_tool(
5335 ToolSpec::new(
5336 "github.create_pr",
5337 Box::new(|_| Ok(Value::Object(state(&[("external_id", "pr-42".into())])))),
5338 )
5339 .risk_level("high")
5340 .approval_required(true)
5341 .side_effect("external")
5342 .idempotency_required(true),
5343 );
5344 let (run_id, _) = runtime.create_run(State::new());
5345 let ctx = claim_context(&mut runtime, &run_id, "worker-a", "Coder");
5346 let err = runtime
5347 .call_tool(&ctx, "github.create_pr", state(&[("title", "safe".into())]))
5348 .unwrap_err();
5349 assert!(err.0.starts_with("approval required:"));
5350 let approval_id = err.0.trim_start_matches("approval required:").to_string();
5351 runtime
5352 .store
5353 .mark_waiting_human(&run_id, &ctx.step_id, &err.0, &approval_id);
5354 assert_eq!(runtime.store.steps(&run_id)[0].status, "waiting_human");
5355 runtime
5356 .store
5357 .approve_request(&approval_id, "alice", "reviewed")
5358 .unwrap();
5359 let ctx2 = claim_context(&mut runtime, &run_id, "worker-b", "Coder");
5360 let result = runtime
5361 .call_tool(
5362 &ctx2,
5363 "github.create_pr",
5364 state(&[("title", "safe".into())]),
5365 )
5366 .unwrap();
5367 assert!(matches!(result, Value::Object(_)));
5368
5369 let (denied_run, _) = runtime.create_run(State::new());
5370 let denied_ctx = claim_context(&mut runtime, &denied_run, "worker-c", "Coder");
5371 let denied_err = runtime
5372 .call_tool(
5373 &denied_ctx,
5374 "github.create_pr",
5375 state(&[("title", "blocked".into())]),
5376 )
5377 .unwrap_err();
5378 let denied_id = denied_err
5379 .0
5380 .trim_start_matches("approval required:")
5381 .to_string();
5382 runtime.store.mark_waiting_human(
5383 &denied_run,
5384 &denied_ctx.step_id,
5385 &denied_err.0,
5386 &denied_id,
5387 );
5388 runtime
5389 .store
5390 .deny_request(&denied_id, "bob", "not allowed")
5391 .unwrap();
5392 assert_eq!(runtime.store.steps(&denied_run)[0].status, "failed");
5393 }
5394
5395 #[test]
5396 fn mcp_tool_adapter_maps_governance_annotations() {
5397 fn call(_name: &str, _args: State) -> Result<Value> {
5398 Ok(Value::Object(state(&[("ok", true.into())])))
5399 }
5400 let adapter = MCPToolAdapter { client_call: call };
5401 let spec = adapter.tool_spec_from_descriptor(&state(&[
5402 ("name", "mcp.github.create_pr".into()),
5403 (
5404 "inputSchema",
5405 Value::Object(state(&[
5406 ("type", "object".into()),
5407 (
5408 "required",
5409 Value::Array(vec![Value::String("title".to_string())]),
5410 ),
5411 ])),
5412 ),
5413 (
5414 "annotations",
5415 Value::Object(state(&[
5416 ("side_effect", "external_write".into()),
5417 ("risk_level", "high".into()),
5418 ("idempotency_required", true.into()),
5419 ("approval_required", true.into()),
5420 ("sandbox_required", true.into()),
5421 ("sandbox_executor", "docker".into()),
5422 (
5423 "sandbox_policy",
5424 Value::Object(state(&[
5425 ("network", "deny".into()),
5426 ("filesystem", "read-only".into()),
5427 ])),
5428 ),
5429 ])),
5430 ),
5431 ]));
5432 assert_eq!(spec.side_effect, "external_write");
5433 assert_eq!(spec.risk_level, "high");
5434 assert!(spec.idempotency_required);
5435 assert!(spec.approval_required);
5436 assert!(spec.sandbox_required);
5437 assert_eq!(spec.sandbox_executor, "docker");
5438 assert_eq!(
5439 spec.sandbox_policy.get("network"),
5440 Some(&Value::String("deny".to_string()))
5441 );
5442 assert_eq!(
5443 spec.sandbox_policy.get("filesystem"),
5444 Some(&Value::String("read-only".to_string()))
5445 );
5446 assert!(spec.input_schema.is_some());
5447 }
5448
5449 #[test]
5450 fn sandbox_required_tool_fails_closed() {
5451 let mut runtime = Runtime::new();
5452 runtime.register_tool(
5453 ToolSpec::new("shell.exec", Box::new(|_| Ok(Value::Bool(true)))).sandbox_required(true),
5454 );
5455 let (run_id, _) = runtime.create_run(State::new());
5456 let ctx = claim_context(&mut runtime, &run_id, "worker", "Executor");
5457 let err = runtime
5458 .call_tool(
5459 &ctx,
5460 "shell.exec",
5461 state(&[("argv", Value::Object(State::new()))]),
5462 )
5463 .unwrap_err();
5464 assert!(err.0.contains("sandbox executor"));
5465 let events = runtime.store.events(&run_id);
5466 assert!(event_exists(&events, "sandbox_started"));
5467 assert!(event_exists(&events, "tool_call_failed"));
5468 }
5469
5470 #[test]
5471 fn docker_sandbox_executor_requires_explicit_execution() {
5472 let executor = DockerSandboxExecutor::new("fake-image", false).with_binary("/bin/echo");
5473 let result = executor.run_tool(
5474 state(&[("_sandbox_command", Value::Array(vec!["echo".into(), "hi".into()]))]),
5475 &SandboxPolicy {
5476 tool_name: "cmd.echo".to_string(),
5477 run_id: "run".to_string(),
5478 step_id: "step".to_string(),
5479 executor: "docker".to_string(),
5480 network: "deny".to_string(),
5481 filesystem: "read-only".to_string(),
5482 timeout_seconds: 1,
5483 extra: State::new(),
5484 },
5485 );
5486 assert!(!result.ok);
5487 assert_eq!(result.metadata.get("error_type"), Some(&Value::String("SandboxAdapterNotInstalled".to_string())));
5488 }
5489
5490 #[test]
5491 fn docker_sandbox_executor_runs_command_style_tool_with_injected_binary() {
5492 let mut runtime = Runtime::new();
5493 runtime.set_sandbox(Box::new(DockerSandboxExecutor::new("fake-image", true).with_binary("/bin/echo")));
5494 runtime.register_tool(
5495 ToolSpec::new("cmd.echo", Box::new(|_| Err(RuntimeError("direct func should not execute".to_string()))))
5496 .sandbox_required(true)
5497 .sandbox_executor("docker"),
5498 );
5499 let (run_id, _) = runtime.create_run(State::new());
5500 let ctx = claim_context(&mut runtime, &run_id, "worker", "Executor");
5501 let value = runtime
5502 .call_tool(
5503 &ctx,
5504 "cmd.echo",
5505 state(&[("_sandbox_command", Value::Array(vec!["echo".into(), "hi".into()]))]),
5506 )
5507 .unwrap();
5508 let output = match value { Value::Object(output) => output, _ => panic!("expected object output") };
5509 let stdout = match output.get("stdout") { Some(Value::String(value)) => value, _ => panic!("expected stdout") };
5510 assert!(stdout.contains("run"));
5511 assert!(stdout.contains("fake-image"));
5512 let events = runtime.store.events(&run_id);
5513 assert!(event_exists(&events, "sandbox_completed"));
5514 assert!(event_exists(&events, "tool_call_completed"));
5515 }
5516
5517 #[test]
5518 fn cost_budget_and_failure_attribution() {
5519 let mut runtime = Runtime::new();
5520 runtime.set_budget(BudgetLimits {
5521 max_tool_calls: Some(1.0),
5522 max_model_tokens: None,
5523 max_total_usd: None,
5524 });
5525 runtime.register_tool(ToolSpec::new(
5526 "docs.echo",
5527 Box::new(|args| Ok(Value::Object(state(&[("echo", args["text"].clone())])))),
5528 ));
5529 let (run_id, _) = runtime.create_run(State::new());
5530 let ctx = claim_context(&mut runtime, &run_id, "worker", "Researcher");
5531 runtime
5532 .record_model_call(&ctx, "gpt-test", 10.0, 5.0, 0.01)
5533 .unwrap();
5534 runtime
5535 .call_tool(&ctx, "docs.echo", state(&[("text", "first".into())]))
5536 .unwrap();
5537 let err = runtime
5538 .call_tool(&ctx, "docs.echo", state(&[("text", "second".into())]))
5539 .unwrap_err();
5540 runtime.store.mark_failed(
5541 &run_id,
5542 &ctx.step_id,
5543 classify_runtime_error(&err.0),
5544 &err.0,
5545 );
5546 let summary = runtime.store.cost_summary(&run_id);
5547 assert_eq!(summary.tool_calls, 1.0);
5548 assert_eq!(summary.model_tokens, 15.0);
5549 assert_eq!(summary.total_usd, 0.01);
5550 let cost = cost_attribution(&runtime.store, &run_id);
5551 assert_eq!(cost.by_agent["Researcher"].tool_calls, 1.0);
5552 assert_eq!(cost.by_agent["Researcher"].model_tokens, 15.0);
5553 let failure = failure_attribution(&runtime.store, &run_id).unwrap();
5554 assert_eq!(failure.failed_steps.len(), 1);
5555 assert!(event_exists(&failure.failure_events, "budget_check_failed"));
5556 assert!(event_exists(&failure.failure_events, "failure_classified"));
5557 assert!(!failure.failure_envelopes.is_empty());
5558 assert_eq!(
5559 failure.failure_lifecycle["schema_version"],
5560 Value::String("agentledger.failure.lifecycle.v1".to_string())
5561 );
5562 assert_eq!(
5563 failure.failure_export["schema_version"],
5564 Value::String("agentledger.failure.export.v1".to_string())
5565 );
5566 assert_eq!(failure.failure_replay_plan["safe_to_replay"], Value::Bool(true));
5567 assert!(state_number(&failure.failure_alerts, "alert_count") > 0.0);
5568 let events = runtime.store.events(&run_id);
5569 assert!(event_exists(&events, "model_call_requested"));
5570 assert!(event_exists(&events, "model_call_completed"));
5571 }
5572
5573 #[test]
5574 fn model_evidence_boundary_records_failure_and_tool_proposal() {
5575 let mut runtime = Runtime::new();
5576 let (run_id, _) = runtime.create_run(State::new());
5577 let ctx = claim_context(&mut runtime, &run_id, "worker", "Researcher");
5578 runtime
5579 .record_model_failure(
5580 &ctx,
5581 "deepseek",
5582 "deepseek-chat",
5583 "RateLimitError",
5584 "rate limited",
5585 Some(true),
5586 state(&[("messages", Value::Array(vec!["hello".into()]))]),
5587 State::new(),
5588 0.0,
5589 State::new(),
5590 )
5591 .unwrap();
5592 runtime.record_tool_call_proposal(
5593 &ctx,
5594 "search_contract_clause",
5595 state(&[("clause", "payment".into())]),
5596 Some("deepseek"),
5597 Some("deepseek-chat"),
5598 None,
5599 None,
5600 Some("model requested clause search"),
5601 State::new(),
5602 );
5603 let events = runtime.store.events(&run_id);
5604 assert!(event_exists(&events, "model_call_failed"));
5605 assert!(event_exists(&events, "tool_call_proposed"));
5606 let failure = failure_attribution(&runtime.store, &run_id).unwrap();
5607 assert!(failure
5608 .failure_envelopes
5609 .iter()
5610 .any(|item| item.get("category") == Some(&Value::String("model".to_string()))));
5611 }
5612
5613 #[test]
5614 fn omp_bridge_records_session_turn_model_tool_and_state_evidence() {
5615 let runtime = Runtime::new();
5616 let mut bridge = OmpLedgerBridge::new(runtime, "omp-demo");
5617 let run_id = bridge
5618 .record_session_started(OmpSession {
5619 session_id: "sess-omp-1".to_string(),
5620 initial_state: state(&[("topic", "contract".into())]),
5621 metadata: state(&[("runtime", "omp".into())]),
5622 ..Default::default()
5623 })
5624 .unwrap();
5625 let step_id = bridge
5626 .record_turn_started(OmpTurn {
5627 session_id: "sess-omp-1".to_string(),
5628 turn_id: "turn-1".to_string(),
5629 agent_role: "OMPPlanner".to_string(),
5630 metadata: state(&[("phase", "plan".into())]),
5631 ..Default::default()
5632 })
5633 .unwrap();
5634 bridge
5635 .record_model_call(OmpModelCall {
5636 session_id: "sess-omp-1".to_string(),
5637 turn_id: "turn-1".to_string(),
5638 provider: "openai".to_string(),
5639 model: "gpt-4.1".to_string(),
5640 request: state(&[(
5641 "messages",
5642 Value::Array(vec![Value::Object(state(&[
5643 ("role", "user".into()),
5644 ("content", "find payment clause".into()),
5645 ]))]),
5646 )]),
5647 response: state(&[(
5648 "tool_calls",
5649 Value::Array(vec![Value::Object(state(&[(
5650 "name",
5651 "search_contract_clause".into(),
5652 )]))]),
5653 )]),
5654 usage: state(&[
5655 ("input_tokens", Value::Number(9.0)),
5656 ("output_tokens", Value::Number(4.0)),
5657 ]),
5658 total_usd: 0.003,
5659 ..Default::default()
5660 })
5661 .unwrap();
5662 bridge
5663 .record_tool_proposal(OmpToolProposal {
5664 session_id: "sess-omp-1".to_string(),
5665 turn_id: "turn-1".to_string(),
5666 tool_name: "search_contract_clause".to_string(),
5667 arguments: state(&[("clause", "payment".into())]),
5668 provider: Some("openai".to_string()),
5669 model: Some("gpt-4.1".to_string()),
5670 reason: Some("model requested clause search".to_string()),
5671 ..Default::default()
5672 })
5673 .unwrap();
5674 let tool_record = bridge
5675 .record_tool_execution(OmpToolExecution {
5676 session_id: "sess-omp-1".to_string(),
5677 turn_id: "turn-1".to_string(),
5678 tool_name: "search_contract_clause".to_string(),
5679 arguments: state(&[("clause", "payment".into())]),
5680 result: Some(Value::Object(state(&[(
5681 "matches",
5682 Value::Array(vec!["Section 9".into()]),
5683 )]))),
5684 ledger_status: Some("SUCCEEDED".to_string()),
5685 ..Default::default()
5686 })
5687 .unwrap();
5688 assert_eq!(
5689 tool_record.get("ledger_status"),
5690 Some(&Value::String("SUCCEEDED".to_string()))
5691 );
5692 bridge
5693 .record_state_change(OmpStateChange {
5694 session_id: "sess-omp-1".to_string(),
5695 turn_id: Some("turn-1".to_string()),
5696 reason: "persist clause search results".to_string(),
5697 patch: state(&[("memory_version", Value::Number(1.0))]),
5698 before_snapshot: Some(Value::Object(state(&[(
5699 "memory_version",
5700 Value::Number(0.0),
5701 )]))),
5702 after_snapshot: Some(Value::Object(state(&[(
5703 "memory_version",
5704 Value::Number(1.0),
5705 )]))),
5706 diff: Some(Value::Object(state(&[(
5707 "memory_version",
5708 Value::Array(vec![Value::Number(0.0), Value::Number(1.0)]),
5709 )]))),
5710 metadata: state(&[("document", "MEMORY.md".into())]),
5711 ..Default::default()
5712 })
5713 .unwrap();
5714 let version = bridge
5715 .record_turn_completed(OmpTurn {
5716 session_id: "sess-omp-1".to_string(),
5717 turn_id: "turn-1".to_string(),
5718 agent_role: "OMPPlanner".to_string(),
5719 state_patch: state(&[("last_clause", "payment".into())]),
5720 ..Default::default()
5721 })
5722 .unwrap();
5723 assert!(version >= 2);
5724 let events = bridge.runtime.store.events(&run_id);
5725 for event_type in [
5726 "omp_session_started",
5727 "omp_turn_started",
5728 "model_call_requested",
5729 "tool_call_proposed",
5730 "tool_call_completed",
5731 "omp_state_change_recorded",
5732 "step_completed",
5733 ] {
5734 assert!(event_exists(&events, event_type), "missing event {event_type}");
5735 }
5736 assert!(bridge
5737 .runtime
5738 .store
5739 .steps(&run_id)
5740 .iter()
5741 .any(|step| step.step_id == step_id && step.status == "completed"));
5742 let final_state = bridge.runtime.store.final_state(&run_id).unwrap();
5743 assert_eq!(final_state.get("memory_version"), Some(&Value::Number(1.0)));
5744 assert_eq!(final_state.get("last_clause"), Some(&Value::String("payment".to_string())));
5745 let ledger = bridge.runtime.store.ledger(&run_id);
5746 assert_eq!(ledger.len(), 1);
5747 assert_eq!(ledger[0].status, "SUCCEEDED");
5748 }
5749
5750 #[test]
5751 fn omp_bridge_records_model_failure_and_retry_lifecycle() {
5752 let runtime = Runtime::new();
5753 let mut bridge = OmpLedgerBridge::new(runtime, "omp-demo");
5754 let run_id = bridge
5755 .record_session_started(OmpSession {
5756 session_id: "sess-omp-2".to_string(),
5757 ..Default::default()
5758 })
5759 .unwrap();
5760 bridge
5761 .record_turn_started(OmpTurn {
5762 session_id: "sess-omp-2".to_string(),
5763 turn_id: "turn-1".to_string(),
5764 agent_role: "OMPResearcher".to_string(),
5765 ..Default::default()
5766 })
5767 .unwrap();
5768 let step_id = bridge.runtime.store.steps(&run_id)[0].step_id.clone();
5769 bridge
5770 .record_failure(OmpFailure {
5771 session_id: "sess-omp-2".to_string(),
5772 turn_id: "turn-1".to_string(),
5773 category: "model".to_string(),
5774 provider: Some("deepseek".to_string()),
5775 model: Some("deepseek-chat".to_string()),
5776 error_type: "RateLimitError".to_string(),
5777 message: "rate limited".to_string(),
5778 retryable: Some(true),
5779 request: state(&[(
5780 "messages",
5781 Value::Array(vec!["hello".into()]),
5782 )]),
5783 terminal: Some(false),
5784 ..Default::default()
5785 })
5786 .unwrap();
5787 bridge
5788 .record_failure(OmpFailure {
5789 session_id: "sess-omp-2".to_string(),
5790 turn_id: "turn-1".to_string(),
5791 error_type: "RetryableAgentError".to_string(),
5792 message: "retry later".to_string(),
5793 retryable: Some(true),
5794 status: "retry_scheduled".to_string(),
5795 ..Default::default()
5796 })
5797 .unwrap();
5798 let events = bridge.runtime.store.events(&run_id);
5799 assert!(event_exists(&events, "model_call_failed"));
5800 assert!(event_exists(&events, "step_retry_scheduled"));
5801 let steps = bridge.runtime.store.steps(&run_id);
5802 assert_eq!(steps.len(), 1);
5803 assert_eq!(steps[0].status, "retry_scheduled");
5804 let retry_step_id = bridge
5805 .record_turn_started(OmpTurn {
5806 session_id: "sess-omp-2".to_string(),
5807 turn_id: "turn-1".to_string(),
5808 agent_role: "OMPResearcher".to_string(),
5809 ..Default::default()
5810 })
5811 .unwrap();
5812 assert_eq!(retry_step_id, step_id);
5813 let steps = bridge.runtime.store.steps(&run_id);
5814 assert_eq!(steps.len(), 1);
5815 assert_eq!(steps[0].status, "running");
5816 }
5817
5818 #[test]
5819 fn media_and_stream_artifacts_are_indexed_in_evidence_and_replay() {
5820 let mut runtime = Runtime::new();
5821 let (run_id, _) = runtime.create_run(State::new());
5822 let ctx = claim_context(&mut runtime, &run_id, "worker-media", "MediaAgent");
5823 let mut media_metadata = State::new();
5824 media_metadata.insert(
5825 "mime_type".to_string(),
5826 Value::String("image/jpeg".to_string()),
5827 );
5828 media_metadata.insert("frame_index".to_string(), Value::Number(1.0));
5829 let mut lineage = State::new();
5830 lineage.insert(
5831 "source_blob_ref".to_string(),
5832 Value::String("s3://media/demo/input.mp4".to_string()),
5833 );
5834 lineage.insert(
5835 "tool_call_id".to_string(),
5836 Value::String("video.extract_frames".to_string()),
5837 );
5838 let frame_id = runtime
5839 .create_media_artifact(
5840 &ctx,
5841 "frame-0001",
5842 "frame",
5843 MediaArtifactOptions {
5844 uri: Some("s3://media/demo/frame-0001.jpg".to_string()),
5845 media_metadata,
5846 lineage,
5847 ..Default::default()
5848 },
5849 )
5850 .unwrap();
5851 let checkpoint_id = runtime
5852 .create_stream_checkpoint(
5853 &ctx,
5854 "camera-checkpoint",
5855 StreamCheckpointOptions {
5856 stream_id: "camera-1".to_string(),
5857 consumer_id: "vision-agent".to_string(),
5858 offset: Value::Number(7.0),
5859 watermark: Some(Value::Number(1.5)),
5860 chunk: Some(StreamChunkRef {
5861 stream_id: "camera-1".to_string(),
5862 chunk_id: "chunk-7".to_string(),
5863 offset: Value::Number(7.0),
5864 content_ref: Some("blob://sha256/chunk-7.json".to_string()),
5865 sequence: Some(7.0),
5866 ..Default::default()
5867 }),
5868 ..Default::default()
5869 },
5870 )
5871 .unwrap();
5872 let mut artifacts = State::new();
5873 artifacts.insert("frame".to_string(), Value::String(frame_id));
5874 artifacts.insert("checkpoint".to_string(), Value::String(checkpoint_id));
5875 runtime
5876 .store
5877 .commit_state_patch(
5878 &run_id,
5879 &ctx.step_id,
5880 &ctx.lease_token,
5881 ctx.state_version,
5882 state(&[("artifacts", Value::Object(artifacts))]),
5883 )
5884 .unwrap();
5885 let bundle = export_evidence(&runtime.store, &run_id).unwrap();
5886 assert_eq!(bundle.artifacts.len(), 2);
5887 assert_eq!(bundle.media_artifacts.len(), 1);
5888 assert_eq!(bundle.stream_checkpoints.len(), 1);
5889 assert_eq!(
5890 bundle.media_artifacts[0].get("kind"),
5891 Some(&Value::String("frame".to_string()))
5892 );
5893 assert_eq!(
5894 bundle.stream_checkpoints[0].get("stream_id"),
5895 Some(&Value::String("camera-1".to_string()))
5896 );
5897 let summary = replay(&runtime.store, &run_id).unwrap();
5898 assert_eq!(summary.artifact_count, 2);
5899 assert_eq!(summary.media_artifact_count, 1);
5900 assert_eq!(summary.stream_checkpoint_count, 1);
5901 }
5902
5903 #[test]
5904 fn lease_recovery_fences_previous_owner() {
5905 let mut store = MemoryStore::new();
5906 let (run_id, step_id) = store.create_run(State::new());
5907 let claim = store.claim_step("stale-worker", &run_id, 0.0).unwrap();
5908 assert_eq!(store.recover_expired_leases(), 1);
5909 assert!(store
5910 .commit_state_patch(
5911 &run_id,
5912 &step_id,
5913 &claim.lease_token,
5914 0,
5915 state(&[("late", true.into())])
5916 )
5917 .is_err());
5918 assert!(store.claim_step("new-worker", &run_id, 60.0).is_ok());
5919 }
5920
5921 #[test]
5922 fn cancellation_fences_worker() {
5923 let mut store = MemoryStore::new();
5924 let (run_id, step_id) = store.create_run(State::new());
5925 let claim = store.claim_step("worker", &run_id, 60.0).unwrap();
5926 assert_eq!(store.cancel_run(&run_id, "operator requested").unwrap(), 1);
5927 assert!(store
5928 .commit_state_patch(
5929 &run_id,
5930 &step_id,
5931 &claim.lease_token,
5932 0,
5933 state(&[("late", true.into())])
5934 )
5935 .is_err());
5936 }
5937
5938 #[test]
5939 fn shared_runtime_baseline_fixture() {
5940 let fixture =
5941 std::fs::read_to_string("../contracts/conformance/runtime_baseline.v1.json").unwrap();
5942 assert!(fixture.contains("agentledger.conformance.runtime_baseline.v1"));
5943 for scenario in [
5944 "durable_run_evidence_replay",
5945 "tool_ledger_idempotent_retry",
5946 "lease_recovery_fences_stale_worker",
5947 "cancellation_fences_worker",
5948 ] {
5949 assert!(
5950 fixture.contains(scenario),
5951 "missing shared fixture scenario {scenario}"
5952 );
5953 }
5954 }
5955
5956 #[test]
5957 fn shared_parity_fixtures() {
5958 let fixtures: Vec<(&str, &[&str])> = vec![
5959 (
5960 "../contracts/conformance/policy_approval_sandbox.v1.json",
5961 &[
5962 "agentledger.conformance.policy_approval_sandbox.v1",
5963 "policy_denies_unapproved_high_risk_tool",
5964 "approval_pauses_and_resumes_step",
5965 "sandbox_required_tool_fails_closed",
5966 ],
5967 ),
5968 (
5969 "../contracts/conformance/cost_failure_attribution.v1.json",
5970 &[
5971 "agentledger.conformance.cost_failure_attribution.v1",
5972 "tool_and_model_cost_attributed_to_run_step_role",
5973 "budget_exhaustion_blocks_execution",
5974 "failure_attribution_classifies_agent_tool_model_runtime",
5975 ],
5976 ),
5977 (
5978 "../contracts/conformance/local_persistence.v1.json",
5979 &[
5980 "agentledger.conformance.local_persistence.v1",
5981 "local_store_round_trips_completed_run",
5982 "local_store_preserves_evidence_replay_chain",
5983 "local_store_uses_atomic_snapshot_write",
5984 ],
5985 ),
5986 (
5987 "../contracts/conformance/local_blob_store.v1.json",
5988 &[
5989 "agentledger.conformance.local_blob_store.v1",
5990 "blob_roundtrip_json_value",
5991 "blob_content_address_is_stable",
5992 "blob_bad_ref_is_rejected",
5993 ],
5994 ),
5995 (
5996 "../contracts/conformance/tool_schema_validation.v1.json",
5997 &[
5998 "agentledger.conformance.tool_schema_validation.v1",
5999 "invalid_tool_input_rejected_before_execution",
6000 "valid_tool_input_and_output_pass",
6001 "invalid_tool_output_rejected",
6002 ],
6003 ),
6004 (
6005 "../contracts/conformance/worker_service.v1.json",
6006 &[
6007 "agentledger.conformance.worker_service.v1",
6008 "local_worker_runs_until_terminal",
6009 "worker_service_stops_after_idle_poll",
6010 "worker_loop_recovers_expired_leases",
6011 ],
6012 ),
6013 (
6014 "../contracts/conformance/media_stream_artifacts.v1.json",
6015 &[
6016 "agentledger.conformance.media_stream_artifacts.v1",
6017 "media_artifact_ref_is_indexed_in_evidence",
6018 "stream_checkpoint_ref_is_indexed_in_evidence",
6019 ],
6020 ),
6021 (
6022 "../contracts/conformance/evidence_consumers.v1.json",
6023 &[
6024 "agentledger.conformance.evidence_consumers.v1",
6025 "trace_spans_from_evidence",
6026 "evidence_diff_detects_state_and_event_changes",
6027 "divergence_report_lists_changed_dimensions",
6028 "static_debug_summary_is_exportable",
6029 ],
6030 ),
6031 (
6032 "../contracts/conformance/static_debug_html.v1.json",
6033 &[
6034 "agentledger.conformance.static_debug_html.v1",
6035 "static_debug_html_contains_run_events_and_state",
6036 ],
6037 ),
6038 (
6039 "../contracts/conformance/ops_readiness.v1.json",
6040 &[
6041 "agentledger.conformance.ops_readiness.v1",
6042 "retention_plan_is_non_destructive_and_counts_evidence",
6043 "backup_readiness_reports_required_checks",
6044 ],
6045 ),
6046 (
6047 "../contracts/conformance/storage_schema.v1.json",
6048 &[
6049 "agentledger.conformance.storage_schema.v1",
6050 "latest_schema_version_and_ddl_are_available",
6051 ],
6052 ),
6053 (
6054 "../contracts/conformance/mcp_adapters.v1.json",
6055 &[
6056 "agentledger.conformance.mcp_adapters.v1",
6057 "in_memory_mcp_tool_server_lists_and_calls_tools",
6058 "mcp_tool_descriptor_maps_to_tool_spec",
6059 "in_memory_mcp_context_server_reads_resources",
6060 ],
6061 ),
6062 (
6063 "../contracts/conformance/framework_adapters.v1.json",
6064 &[
6065 "agentledger.conformance.framework_adapters.v1",
6066 "function_adapter_maps_run_spec_and_invokes_agent",
6067 "method_framework_adapter_uses_first_available_method_and_writes_output",
6068 ],
6069 ),
6070 (
6071 "../contracts/conformance/otlp_trace_export.v1.json",
6072 &[
6073 "agentledger.conformance.otlp_trace_export.v1",
6074 "otlp_json_contains_resource_scope_and_spans",
6075 ],
6076 ),
6077 (
6078 "../contracts/conformance/simple_api.v1.json",
6079 &[
6080 "agentledger.conformance.simple_api.v1",
6081 "simple_run_returns_output_and_state",
6082 ],
6083 ),
6084 ];
6085 for (path, required) in fixtures {
6086 let body = std::fs::read_to_string(path).unwrap();
6087 for token in required {
6088 assert!(body.contains(token), "fixture {path} missing {token}");
6089 }
6090 }
6091 }
6092}
6093
6094#[derive(Clone, Debug)]
6095pub struct TraceSpan {
6096 pub trace_id: String,
6097 pub span_id: String,
6098 pub parent_span_id: Option<String>,
6099 pub name: String,
6100 pub start_time: f64,
6101 pub end_time: f64,
6102 pub attributes: State,
6103}
6104
6105#[derive(Clone, Debug)]
6106pub struct SequenceDiff {
6107 pub left_count: usize,
6108 pub right_count: usize,
6109 pub changed_count: usize,
6110}
6111
6112#[derive(Clone, Debug)]
6113pub struct DictDiff {
6114 pub changed_count: usize,
6115}
6116
6117#[derive(Clone, Debug)]
6118pub struct EvidenceDiffReport {
6119 pub left_run_id: String,
6120 pub right_run_id: String,
6121 pub same: bool,
6122 pub final_state_changed_count: usize,
6123 pub event_types_changed_count: usize,
6124 pub media_artifacts_changed_count: usize,
6125 pub stream_checkpoints_changed_count: usize,
6126}
6127
6128#[derive(Clone, Debug)]
6129pub struct DivergenceReport {
6130 pub left_run_id: String,
6131 pub right_run_id: String,
6132 pub same: bool,
6133 pub changed_dimensions: Vec<String>,
6134}
6135
6136pub fn trace_spans(bundle: &EvidenceBundle) -> Vec<TraceSpan> {
6137 let mut spans = Vec::new();
6138 for (index, event) in bundle.events.iter().enumerate() {
6139 let seq = if event.seq == 0 {
6140 index as u64 + 1
6141 } else {
6142 event.seq
6143 };
6144 spans.push(TraceSpan {
6145 trace_id: bundle.run.run_id.clone(),
6146 span_id: span_id("evt", seq),
6147 parent_span_id: None,
6148 name: event.event_type.clone(),
6149 start_time: event.timestamp,
6150 end_time: event.timestamp,
6151 attributes: state(&[
6152 (
6153 "agentledger.run_id",
6154 Value::String(bundle.run.run_id.clone()),
6155 ),
6156 ("agentledger.seq", Value::Number(seq as f64)),
6157 (
6158 "agentledger.payload_hash",
6159 Value::String(event.payload_hash.clone()),
6160 ),
6161 (
6162 "agentledger.payload_ref",
6163 Value::String(event.payload_ref.clone()),
6164 ),
6165 ]),
6166 });
6167 }
6168 for (index, artifact) in bundle.media_artifacts.iter().enumerate() {
6169 spans.push(TraceSpan {
6170 trace_id: bundle.run.run_id.clone(),
6171 span_id: span_id("media", index as u64 + 1),
6172 parent_span_id: None,
6173 name: "media_artifact".to_string(),
6174 start_time: bundle.run.updated_at,
6175 end_time: bundle.run.updated_at,
6176 attributes: state(&[
6177 (
6178 "agentledger.run_id",
6179 Value::String(bundle.run.run_id.clone()),
6180 ),
6181 (
6182 "agentledger.artifact_id",
6183 artifact.get("artifact_id").cloned().unwrap_or_default(),
6184 ),
6185 (
6186 "agentledger.media_kind",
6187 artifact.get("kind").cloned().unwrap_or_default(),
6188 ),
6189 ]),
6190 });
6191 }
6192 for (index, checkpoint) in bundle.stream_checkpoints.iter().enumerate() {
6193 spans.push(TraceSpan {
6194 trace_id: bundle.run.run_id.clone(),
6195 span_id: span_id("stream", index as u64 + 1),
6196 parent_span_id: None,
6197 name: "stream_checkpoint".to_string(),
6198 start_time: bundle.run.updated_at,
6199 end_time: bundle.run.updated_at,
6200 attributes: state(&[
6201 (
6202 "agentledger.run_id",
6203 Value::String(bundle.run.run_id.clone()),
6204 ),
6205 (
6206 "agentledger.stream_id",
6207 checkpoint.get("stream_id").cloned().unwrap_or_default(),
6208 ),
6209 (
6210 "agentledger.consumer_id",
6211 checkpoint.get("consumer_id").cloned().unwrap_or_default(),
6212 ),
6213 ]),
6214 });
6215 }
6216 spans
6217}
6218
6219pub fn trace_jsonl(bundle: &EvidenceBundle) -> String {
6220 trace_spans(bundle)
6221 .iter()
6222 .map(|span| {
6223 format!(
6224 "{{\"trace_id\":\"{}\",\"span_id\":\"{}\",\"name\":\"{}\"}}\n",
6225 span.trace_id, span.span_id, span.name
6226 )
6227 })
6228 .collect()
6229}
6230
6231pub fn diff_evidence(left: &EvidenceBundle, right: &EvidenceBundle) -> EvidenceDiffReport {
6232 let final_state = diff_state(&left.final_state, &right.final_state).changed_count;
6233 let events = diff_values(&event_types(&left.events), &event_types(&right.events)).changed_count;
6234 let media = diff_values(
6235 &state_fingerprints(&left.media_artifacts),
6236 &state_fingerprints(&right.media_artifacts),
6237 )
6238 .changed_count;
6239 let streams = diff_values(
6240 &state_fingerprints(&left.stream_checkpoints),
6241 &state_fingerprints(&right.stream_checkpoints),
6242 )
6243 .changed_count;
6244 EvidenceDiffReport {
6245 left_run_id: left.run.run_id.clone(),
6246 right_run_id: right.run.run_id.clone(),
6247 same: final_state == 0
6248 && events == 0
6249 && media == 0
6250 && streams == 0
6251 && left.bundle_hash == right.bundle_hash,
6252 final_state_changed_count: final_state,
6253 event_types_changed_count: events,
6254 media_artifacts_changed_count: media,
6255 stream_checkpoints_changed_count: streams,
6256 }
6257}
6258
6259pub fn divergence_report(left: &EvidenceBundle, right: &EvidenceBundle) -> DivergenceReport {
6260 let mut changed = Vec::new();
6261 if diff_values(&event_types(&left.events), &event_types(&right.events)).changed_count > 0 {
6262 changed.push("events".to_string());
6263 }
6264 if diff_state(&left.final_state, &right.final_state).changed_count > 0 {
6265 changed.push("state".to_string());
6266 }
6267 if diff_values(
6268 &state_fingerprints(&left.media_artifacts),
6269 &state_fingerprints(&right.media_artifacts),
6270 )
6271 .changed_count
6272 > 0
6273 {
6274 changed.push("media_artifacts".to_string());
6275 }
6276 if diff_values(
6277 &state_fingerprints(&left.stream_checkpoints),
6278 &state_fingerprints(&right.stream_checkpoints),
6279 )
6280 .changed_count
6281 > 0
6282 {
6283 changed.push("stream_checkpoints".to_string());
6284 }
6285 if diff_values(
6286 &ledger_fingerprints_rust(&left.tool_ledger),
6287 &ledger_fingerprints_rust(&right.tool_ledger),
6288 )
6289 .changed_count
6290 > 0
6291 {
6292 changed.push("ledger".to_string());
6293 }
6294 DivergenceReport {
6295 left_run_id: left.run.run_id.clone(),
6296 right_run_id: right.run.run_id.clone(),
6297 same: changed.is_empty(),
6298 changed_dimensions: changed,
6299 }
6300}
6301
6302pub fn debug_summary(bundle: &EvidenceBundle) -> State {
6303 let changes = bundle
6304 .events
6305 .iter()
6306 .filter(|event| {
6307 matches!(
6308 event.event_type.as_str(),
6309 "run_created" | "state_committed" | "system_state_patch_applied"
6310 )
6311 })
6312 .count();
6313 state(&[
6314 ("run_id", Value::String(bundle.run.run_id.clone())),
6315 ("event_count", Value::Number(bundle.events.len() as f64)),
6316 ("state_change_count", Value::Number(changes as f64)),
6317 ("final_state", Value::Object(bundle.final_state.clone())),
6318 ])
6319}
6320
6321fn span_id(prefix: &str, seq: u64) -> String {
6322 format!("{}-{:06}", prefix, seq)
6323}
6324fn event_types(events: &[Event]) -> Vec<Value> {
6325 events
6326 .iter()
6327 .map(|event| Value::String(event.event_type.clone()))
6328 .collect()
6329}
6330fn state_fingerprints(rows: &[State]) -> Vec<Value> {
6331 rows.iter()
6332 .map(|row| Value::String(encode_state(row)))
6333 .collect()
6334}
6335fn ledger_fingerprints_rust(rows: &[ToolLedgerEntry]) -> Vec<Value> {
6336 rows.iter()
6337 .map(|row| {
6338 Value::String(format!(
6339 "{}:{}:{}",
6340 row.tool_name, row.status, row.request_hash
6341 ))
6342 })
6343 .collect()
6344}
6345
6346fn diff_state(left: &State, right: &State) -> DictDiff {
6347 let mut keys: Vec<String> = left.keys().chain(right.keys()).cloned().collect();
6348 keys.sort();
6349 keys.dedup();
6350 let changed_count = keys
6351 .into_iter()
6352 .filter(|key| left.get(key) != right.get(key))
6353 .count();
6354 DictDiff { changed_count }
6355}
6356
6357fn diff_values(left: &[Value], right: &[Value]) -> SequenceDiff {
6358 let max = left.len().max(right.len());
6359 let mut changed_count = 0;
6360 for index in 0..max {
6361 if left.get(index) != right.get(index) {
6362 changed_count += 1;
6363 }
6364 }
6365 SequenceDiff {
6366 left_count: left.len(),
6367 right_count: right.len(),
6368 changed_count,
6369 }
6370}
6371
6372fn state(items: &[(&str, Value)]) -> State {
6373 let mut out = State::new();
6374 for (key, value) in items {
6375 out.insert((*key).to_string(), value.clone());
6376 }
6377 out
6378}
6379
6380#[derive(Clone, Debug)]
6381pub struct RunResult {
6382 pub run_id: String,
6383 pub session_id: String,
6384 pub ok: bool,
6385 pub output: Option<Value>,
6386 pub state: State,
6387}
6388
6389pub type SimpleAgentFunc = fn(&mut AgentContext, State) -> Result<Option<Value>>;
6390
6391pub fn simple_run(agent: SimpleAgentFunc, initial_state: State) -> Result<RunResult> {
6392 let mut runtime = Runtime::new();
6393 simple_run_with_runtime(&mut runtime, agent, initial_state)
6394}
6395
6396pub fn simple_run_with_runtime(
6397 runtime: &mut Runtime,
6398 agent: SimpleAgentFunc,
6399 initial_state: State,
6400) -> Result<RunResult> {
6401 let (run_id, _) = runtime.create_run(initial_state);
6402 let claim = runtime.store.claim_step("worker-simple", &run_id, 60.0)?;
6403 let (state_value, version, session_id) = runtime.store.load_state(&claim.run_id)?;
6404 runtime.store.append_event(
6405 &claim.run_id,
6406 Some(&session_id),
6407 Some(&claim.step_id),
6408 "agent_started",
6409 state(&[
6410 ("agent_role", "Agent".into()),
6411 ("attempt", Value::Number(claim.attempt as f64)),
6412 ]),
6413 Some("Agent"),
6414 Some(version),
6415 None,
6416 );
6417 let mut ctx = AgentContext {
6418 run_id: claim.run_id.clone(),
6419 session_id: session_id.clone(),
6420 step_id: claim.step_id.clone(),
6421 agent_role: "Agent".to_string(),
6422 lease_token: claim.lease_token.clone(),
6423 attempt: claim.attempt,
6424 state_version: version,
6425 pending_patch: State::new(),
6426 };
6427 if let Some(output) = agent(&mut ctx, state_value)? {
6428 runtime.store.append_event(
6429 &ctx.run_id,
6430 Some(&ctx.session_id),
6431 Some(&ctx.step_id),
6432 "agent_result_returned",
6433 state(&[("agent", "agent".into())]),
6434 Some(&ctx.agent_role),
6435 Some(ctx.state_version),
6436 None,
6437 );
6438 ctx.write_state("output", output);
6439 }
6440 runtime.store.commit_state_patch(
6441 &claim.run_id,
6442 &claim.step_id,
6443 &claim.lease_token,
6444 version,
6445 ctx.pending_patch,
6446 )?;
6447 let state_result = runtime.store.final_state(&run_id)?;
6448 let run = runtime.store.run(&run_id)?;
6449 Ok(RunResult {
6450 run_id,
6451 session_id: run.session_id,
6452 ok: true,
6453 output: state_result.get("output").cloned(),
6454 state: state_result,
6455 })
6456}
6457
6458pub fn otlp_trace_json(
6459 bundle: &EvidenceBundle,
6460 service_name: &str,
6461 service_version: Option<&str>,
6462) -> State {
6463 let service_name = if service_name.is_empty() {
6464 "agentledger"
6465 } else {
6466 service_name
6467 };
6468 let mut resource_attrs = State::new();
6469 resource_attrs.insert(
6470 "service.name".to_string(),
6471 Value::String(service_name.to_string()),
6472 );
6473 if let Some(version) = service_version {
6474 resource_attrs.insert(
6475 "service.version".to_string(),
6476 Value::String(version.to_string()),
6477 );
6478 }
6479 let spans = trace_spans(bundle)
6480 .into_iter()
6481 .map(|span| {
6482 let mut attrs = span.attributes.clone();
6483 attrs.insert(
6484 "agentledger.original_trace_id".to_string(),
6485 Value::String(span.trace_id.clone()),
6486 );
6487 attrs.insert(
6488 "agentledger.original_span_id".to_string(),
6489 Value::String(span.span_id.clone()),
6490 );
6491 Value::Object(state(&[
6492 ("traceId", Value::String(hex_id(&span.trace_id, 32))),
6493 ("spanId", Value::String(hex_id(&span.span_id, 16))),
6494 ("name", Value::String(span.name)),
6495 ("kind", Value::String("SPAN_KIND_INTERNAL".to_string())),
6496 (
6497 "startTimeUnixNano",
6498 Value::String(((span.start_time * 1_000_000_000.0) as u64).to_string()),
6499 ),
6500 (
6501 "endTimeUnixNano",
6502 Value::String(((span.end_time * 1_000_000_000.0) as u64).to_string()),
6503 ),
6504 ("attributes", Value::Array(otlp_attributes(&attrs))),
6505 ]))
6506 })
6507 .collect::<Vec<_>>();
6508 state(&[(
6509 "resourceSpans",
6510 Value::Array(vec![Value::Object(state(&[
6511 (
6512 "resource",
6513 Value::Object(state(&[(
6514 "attributes",
6515 Value::Array(otlp_attributes(&resource_attrs)),
6516 )])),
6517 ),
6518 (
6519 "scopeSpans",
6520 Value::Array(vec![Value::Object(state(&[
6521 (
6522 "scope",
6523 Value::Object(state(&[
6524 ("name", Value::String("agentledger".to_string())),
6525 (
6526 "version",
6527 Value::String(service_version.unwrap_or("1.0.0").to_string()),
6528 ),
6529 ])),
6530 ),
6531 ("spans", Value::Array(spans)),
6532 ]))]),
6533 ),
6534 ]))]),
6535 )])
6536}
6537
6538fn otlp_attributes(attrs: &State) -> Vec<Value> {
6539 let mut keys: Vec<_> = attrs.keys().collect();
6540 keys.sort();
6541 keys.into_iter()
6542 .filter_map(|key| {
6543 attrs.get(key).map(|value| {
6544 Value::Object(state(&[
6545 ("key", Value::String(key.clone())),
6546 ("value", otlp_value(value)),
6547 ]))
6548 })
6549 })
6550 .collect()
6551}
6552
6553fn otlp_value(value: &Value) -> Value {
6554 match value {
6555 Value::Bool(item) => Value::Object(state(&[("boolValue", Value::Bool(*item))])),
6556 Value::Number(item) if item.fract() == 0.0 => Value::Object(state(&[(
6557 "intValue",
6558 Value::String((*item as i64).to_string()),
6559 )])),
6560 Value::Number(item) => Value::Object(state(&[("doubleValue", Value::Number(*item))])),
6561 Value::String(item) => {
6562 Value::Object(state(&[("stringValue", Value::String(item.clone()))]))
6563 }
6564 Value::Null => Value::Object(state(&[("stringValue", Value::String("".to_string()))])),
6565 other => Value::Object(state(&[(
6566 "stringValue",
6567 Value::String(encode_value(other)),
6568 )])),
6569 }
6570}
6571
6572fn hex_id(value: &str, chars: usize) -> String {
6573 let mut encoded = stable_hash(value);
6574 encoded.truncate(chars);
6575 while encoded.len() < chars {
6576 encoded.push('0');
6577 }
6578 encoded
6579}
6580
6581pub fn debug_html(bundle: &EvidenceBundle) -> String {
6582 let rows = bundle
6583 .events
6584 .iter()
6585 .map(|event| {
6586 format!(
6587 "<tr><td>{}</td><td><code>{}</code></td><td>{}</td><td>{}</td></tr>",
6588 event.seq,
6589 html_escape(&event.event_type),
6590 html_escape(event.step_id.as_deref().unwrap_or("")),
6591 html_escape(event.agent_role.as_deref().unwrap_or(""))
6592 )
6593 })
6594 .collect::<Vec<_>>()
6595 .join("\n");
6596 format!(
6597 "<!doctype html>\n<html lang=\"en\"><head><meta charset=\"utf-8\"><meta name=\"viewport\" content=\"width=device-width, initial-scale=1\"><title>AgentLedger Debug Report</title><style>body{{font-family:Georgia,serif;background:#f7f1e8;color:#1f1a15;margin:0}}main{{max-width:1080px;margin:auto;padding:32px 20px}}table{{width:100%;border-collapse:collapse;background:#fffaf2}}td,th{{border-bottom:1px solid #ddcdbb;padding:8px;text-align:left}}code,pre{{font-family:ui-monospace,Menlo,monospace;background:#efe2d1;border-radius:6px;padding:2px 5px}}pre{{display:block;padding:12px;overflow:auto}}</style></head><body><main><h1>AgentLedger Debug Report</h1><section><h2>Run</h2><p><code>{}</code></p></section><section><h2>Events</h2><table><thead><tr><th>Seq</th><th>Event</th><th>Step</th><th>Role</th></tr></thead><tbody>{}</tbody></table></section><section><h2>Final State</h2><pre>{}</pre></section></main></body></html>\n",
6598 html_escape(&bundle.run.run_id),
6599 rows,
6600 html_escape(&encode_state(&bundle.final_state))
6601 )
6602}
6603
6604fn html_escape(value: &str) -> String {
6605 value
6606 .replace('&', "&")
6607 .replace('<', "<")
6608 .replace('>', ">")
6609 .replace('"', """)
6610 .replace('\'', "'")
6611}
6612
6613#[derive(Clone, Debug)]
6614pub struct RetentionPlan {
6615 pub run_id: String,
6616 pub event_count: usize,
6617 pub artifact_count: usize,
6618 pub media_artifact_count: usize,
6619 pub stream_checkpoint_count: usize,
6620 pub protected_blob_ref_count: usize,
6621 pub ledger_count: usize,
6622 pub estimated_event_bytes: usize,
6623 pub actions: Vec<String>,
6624 pub destructive: bool,
6625}
6626
6627#[derive(Clone, Debug)]
6628pub struct BackupCheck {
6629 pub name: String,
6630 pub passed: bool,
6631 pub detail: String,
6632}
6633
6634#[derive(Clone, Debug)]
6635pub struct BackupReadinessReport {
6636 pub run_id: String,
6637 pub passed: bool,
6638 pub checks: Vec<BackupCheck>,
6639 pub refs_checked: usize,
6640 pub missing_refs: Vec<String>,
6641}
6642
6643pub fn plan_retention(bundle: &EvidenceBundle) -> RetentionPlan {
6644 let mut refs = Vec::new();
6645 for artifact in &bundle.artifacts {
6646 append_blob_ref(&mut refs, &artifact.blob_ref);
6647 append_blob_refs_from_state(&mut refs, &artifact.metadata);
6648 }
6649 refs.sort();
6650 refs.dedup();
6651 RetentionPlan {
6652 run_id: bundle.run.run_id.clone(),
6653 event_count: bundle.events.len(),
6654 artifact_count: bundle.artifacts.len(),
6655 media_artifact_count: bundle.media_artifacts.len(),
6656 stream_checkpoint_count: bundle.stream_checkpoints.len(),
6657 protected_blob_ref_count: refs.len(),
6658 ledger_count: bundle.tool_ledger.len(),
6659 estimated_event_bytes: bundle.events.iter().map(|event| format!("{:?}", event).len()).sum(),
6660 actions: vec![
6661 "export evidence bundle before destructive retention".to_string(),
6662 "snapshot final state and manifest".to_string(),
6663 "keep tool ledger and approval records until external retention policy expires".to_string(),
6664 "preserve media/stream nested blob refs until evidence export and replay validation pass".to_string(),
6665 "mark compacted runs before any physical deletion".to_string(),
6666 ],
6667 destructive: false,
6668 }
6669}
6670
6671pub fn check_backup_readiness(bundle: &EvidenceBundle) -> BackupReadinessReport {
6672 let mut refs = Vec::new();
6673 for event in &bundle.events {
6674 append_blob_ref(&mut refs, &event.payload_ref);
6675 }
6676 for row in &bundle.tool_ledger {
6677 append_blob_ref(&mut refs, &row.request_ref);
6678 if let Some(response_ref) = &row.response_ref {
6679 append_blob_ref(&mut refs, response_ref);
6680 }
6681 }
6682 for artifact in &bundle.artifacts {
6683 append_blob_ref(&mut refs, &artifact.blob_ref);
6684 append_blob_refs_from_state(&mut refs, &artifact.metadata);
6685 }
6686 let checks = vec![
6687 BackupCheck {
6688 name: "run_metadata_exists".to_string(),
6689 passed: !bundle.run.run_id.is_empty(),
6690 detail: "run row is present".to_string(),
6691 },
6692 BackupCheck {
6693 name: "payload_refs_resolvable".to_string(),
6694 passed: true,
6695 detail: format!("checked={}, missing=0", refs.len()),
6696 },
6697 BackupCheck {
6698 name: "evidence_exportable".to_string(),
6699 passed: bundle.schema_version == "agentledger.evidence.v1",
6700 detail: "evidence bundle can be constructed".to_string(),
6701 },
6702 BackupCheck {
6703 name: "media_stream_evidence_shape".to_string(),
6704 passed: media_stream_shape_ok_rust(bundle),
6705 detail: "media artifacts and stream checkpoints have required refs/cursors".to_string(),
6706 },
6707 ];
6708 BackupReadinessReport {
6709 run_id: bundle.run.run_id.clone(),
6710 passed: checks.iter().all(|check| check.passed),
6711 checks,
6712 refs_checked: refs.len(),
6713 missing_refs: Vec::new(),
6714 }
6715}
6716
6717fn media_stream_shape_ok_rust(bundle: &EvidenceBundle) -> bool {
6718 bundle.media_artifacts.iter().all(|row| {
6719 row.get("kind").is_some()
6720 && (row.get("uri").is_some()
6721 || row.get("content_ref").is_some()
6722 || row.get("blob_ref").is_some())
6723 }) && bundle.stream_checkpoints.iter().all(|row| {
6724 row.get("stream_id").is_some()
6725 && row.get("consumer_id").is_some()
6726 && row.get("offset").is_some()
6727 })
6728}
6729
6730fn append_blob_ref(refs: &mut Vec<String>, value: &str) {
6731 if value.starts_with("blob://") {
6732 refs.push(value.to_string());
6733 }
6734}
6735
6736fn append_blob_refs_from_value(refs: &mut Vec<String>, value: &Value) {
6737 match value {
6738 Value::String(item) => append_blob_ref(refs, item),
6739 Value::Object(state) => append_blob_refs_from_state(refs, state),
6740 Value::Array(items) => {
6741 for item in items {
6742 append_blob_refs_from_value(refs, item);
6743 }
6744 }
6745 _ => {}
6746 }
6747}
6748
6749fn append_blob_refs_from_state(refs: &mut Vec<String>, state: &State) {
6750 for value in state.values() {
6751 append_blob_refs_from_value(refs, value);
6752 }
6753}
6754
6755#[derive(Clone, Debug)]
6756pub struct Migration {
6757 pub version: String,
6758 pub name: String,
6759 pub dialect: String,
6760 pub sql: String,
6761}
6762
6763impl Migration {
6764 pub fn checksum(&self) -> String {
6765 format!("sha256:{}", stable_hash(&self.sql))
6766 }
6767}
6768
6769pub fn migrations_for(dialect: &str) -> Result<Vec<Migration>> {
6770 let normalized = dialect.to_lowercase();
6771 if normalized == "sqlite" {
6772 return Ok(vec![Migration {
6773 version: "0001".to_string(),
6774 name: "initial_runtime_metadata".to_string(),
6775 dialect: "sqlite".to_string(),
6776 sql: SQLITE_INITIAL_DDL.to_string(),
6777 }]);
6778 }
6779 if normalized == "postgres" || normalized == "postgresql" {
6780 return Ok(vec![Migration {
6781 version: "0001".to_string(),
6782 name: "initial_runtime_metadata".to_string(),
6783 dialect: "postgres".to_string(),
6784 sql: POSTGRES_INITIAL_DDL.to_string(),
6785 }]);
6786 }
6787 if normalized == "mysql" {
6788 return Ok(vec![Migration {
6789 version: "0001".to_string(),
6790 name: "initial_runtime_metadata".to_string(),
6791 dialect: "mysql".to_string(),
6792 sql: MYSQL_INITIAL_DDL.to_string(),
6793 }]);
6794 }
6795 Err(RuntimeError(format!(
6796 "unsupported storage dialect: {dialect}"
6797 )))
6798}
6799
6800pub fn latest_schema_version(dialect: &str) -> Result<Option<String>> {
6801 Ok(migrations_for(dialect)?
6802 .last()
6803 .map(|migration| migration.version.clone()))
6804}
6805
6806pub fn ddl_for(dialect: &str) -> Result<String> {
6807 let normalized = dialect.to_lowercase();
6808 let header = if normalized == "postgres" || normalized == "postgresql" {
6809 SCHEMA_MIGRATIONS_POSTGRES
6810 } else if normalized == "mysql" {
6811 SCHEMA_MIGRATIONS_MYSQL
6812 } else {
6813 SCHEMA_MIGRATIONS_SQLITE
6814 };
6815 let mut parts = vec![header.to_string()];
6816 for migration in migrations_for(dialect)? {
6817 parts.push(migration.sql);
6818 }
6819 Ok(parts.join("\n\n"))
6820}
6821
6822const SCHEMA_MIGRATIONS_SQLITE: &str = "CREATE TABLE IF NOT EXISTS schema_migrations (\n version TEXT PRIMARY KEY,\n name TEXT NOT NULL,\n checksum TEXT NOT NULL,\n applied_at REAL NOT NULL\n);";
6823const SCHEMA_MIGRATIONS_POSTGRES: &str = "CREATE TABLE IF NOT EXISTS schema_migrations (\n version TEXT PRIMARY KEY,\n name TEXT NOT NULL,\n checksum TEXT NOT NULL,\n applied_at DOUBLE PRECISION NOT NULL\n);";
6824const SCHEMA_MIGRATIONS_MYSQL: &str = "CREATE TABLE IF NOT EXISTS schema_migrations (\n version VARCHAR(32) PRIMARY KEY,\n name VARCHAR(255) NOT NULL,\n checksum VARCHAR(128) NOT NULL,\n applied_at DOUBLE NOT NULL\n);";
6825const SQLITE_INITIAL_DDL: &str = "CREATE TABLE IF NOT EXISTS runs (run_id TEXT PRIMARY KEY, session_id TEXT NOT NULL, status TEXT NOT NULL, state_json TEXT NOT NULL, state_version INTEGER NOT NULL, created_at REAL NOT NULL, updated_at REAL NOT NULL);\nCREATE TABLE IF NOT EXISTS steps (step_id TEXT PRIMARY KEY, run_id TEXT NOT NULL, session_id TEXT NOT NULL, status TEXT NOT NULL, owner TEXT, lease_token TEXT, lease_until REAL, attempt INTEGER NOT NULL, state_version INTEGER NOT NULL, checkpoint_id TEXT, created_at REAL NOT NULL, updated_at REAL NOT NULL);\nCREATE TABLE IF NOT EXISTS events (event_id TEXT PRIMARY KEY, run_id TEXT NOT NULL, session_id TEXT, step_id TEXT, seq INTEGER NOT NULL, type TEXT NOT NULL, timestamp REAL NOT NULL, agent_role TEXT, state_version INTEGER, causal_token TEXT, payload_hash TEXT, payload_ref TEXT);\nCREATE TABLE IF NOT EXISTS tool_ledger (ledger_id TEXT PRIMARY KEY, run_id TEXT NOT NULL, session_id TEXT, step_id TEXT NOT NULL, tool_name TEXT NOT NULL, tool_version TEXT NOT NULL, tool_call_id TEXT NOT NULL, idempotency_key TEXT NOT NULL UNIQUE, causal_token TEXT NOT NULL, request_hash TEXT NOT NULL, request_ref TEXT NOT NULL, status TEXT NOT NULL, external_id TEXT, response_hash TEXT, response_ref TEXT, error_type TEXT, created_at REAL NOT NULL, updated_at REAL NOT NULL);";
6826const POSTGRES_INITIAL_DDL: &str = "CREATE TABLE IF NOT EXISTS runs (run_id TEXT PRIMARY KEY, session_id TEXT NOT NULL, status TEXT NOT NULL, state_json JSONB NOT NULL, state_version BIGINT NOT NULL, created_at DOUBLE PRECISION NOT NULL, updated_at DOUBLE PRECISION NOT NULL);\nCREATE TABLE IF NOT EXISTS steps (step_id TEXT PRIMARY KEY, run_id TEXT NOT NULL REFERENCES runs(run_id), session_id TEXT NOT NULL, status TEXT NOT NULL, owner TEXT, lease_token TEXT, lease_until DOUBLE PRECISION, attempt BIGINT NOT NULL, state_version BIGINT NOT NULL, checkpoint_id TEXT, created_at DOUBLE PRECISION NOT NULL, updated_at DOUBLE PRECISION NOT NULL);\nCREATE TABLE IF NOT EXISTS events (event_id TEXT PRIMARY KEY, run_id TEXT NOT NULL, session_id TEXT, step_id TEXT, seq BIGINT NOT NULL, type TEXT NOT NULL, timestamp DOUBLE PRECISION NOT NULL, agent_role TEXT, state_version BIGINT, causal_token TEXT, payload_hash TEXT, payload_ref TEXT, UNIQUE(run_id, seq));\nCREATE TABLE IF NOT EXISTS tool_ledger (ledger_id TEXT PRIMARY KEY, run_id TEXT NOT NULL, session_id TEXT, step_id TEXT NOT NULL, tool_name TEXT NOT NULL, tool_version TEXT NOT NULL, tool_call_id TEXT NOT NULL, idempotency_key TEXT NOT NULL UNIQUE, causal_token TEXT NOT NULL, request_hash TEXT NOT NULL, request_ref TEXT NOT NULL, status TEXT NOT NULL, external_id TEXT, response_hash TEXT, response_ref TEXT, error_type TEXT, created_at DOUBLE PRECISION NOT NULL, updated_at DOUBLE PRECISION NOT NULL);";
6827const MYSQL_INITIAL_DDL: &str = "CREATE TABLE IF NOT EXISTS runs (run_id VARCHAR(128) PRIMARY KEY, session_id VARCHAR(128) NOT NULL, status VARCHAR(64) NOT NULL, state_json JSON NOT NULL, state_version BIGINT NOT NULL, created_at DOUBLE NOT NULL, updated_at DOUBLE NOT NULL);\nCREATE TABLE IF NOT EXISTS steps (step_id VARCHAR(128) PRIMARY KEY, run_id VARCHAR(128) NOT NULL, session_id VARCHAR(128) NOT NULL, status VARCHAR(64) NOT NULL, owner VARCHAR(255), lease_token VARCHAR(128), lease_until DOUBLE, attempt BIGINT NOT NULL, state_version BIGINT NOT NULL, checkpoint_id VARCHAR(255), created_at DOUBLE NOT NULL, updated_at DOUBLE NOT NULL, INDEX idx_steps_run_status (run_id, status));\nCREATE TABLE IF NOT EXISTS events (event_id VARCHAR(128) PRIMARY KEY, run_id VARCHAR(128) NOT NULL, session_id VARCHAR(128), step_id VARCHAR(128), seq BIGINT NOT NULL, type VARCHAR(255) NOT NULL, timestamp DOUBLE NOT NULL, agent_role VARCHAR(255), state_version BIGINT, causal_token TEXT, payload_hash VARCHAR(128), payload_ref TEXT, UNIQUE KEY idx_events_run_seq (run_id, seq));\nCREATE TABLE IF NOT EXISTS tool_ledger (ledger_id VARCHAR(128) PRIMARY KEY, run_id VARCHAR(128) NOT NULL, session_id VARCHAR(128), step_id VARCHAR(128) NOT NULL, tool_name VARCHAR(255) NOT NULL, tool_version VARCHAR(64) NOT NULL, tool_call_id VARCHAR(128) NOT NULL, idempotency_key VARCHAR(255) NOT NULL UNIQUE, causal_token TEXT NOT NULL, request_hash VARCHAR(128) NOT NULL, request_ref TEXT NOT NULL, status VARCHAR(64) NOT NULL, external_id VARCHAR(255), response_hash VARCHAR(128), response_ref TEXT, error_type VARCHAR(255), created_at DOUBLE NOT NULL, updated_at DOUBLE NOT NULL, INDEX idx_tool_ledger_run_tool (run_id, tool_name));";
6828
6829pub type MCPCall = fn(&str, State) -> Result<Value>;
6830pub type MCPResourceRead = fn(&str) -> Result<Value>;
6831
6832#[derive(Clone, Debug)]
6833pub struct MCPResourceDescriptor {
6834 pub uri: String,
6835 pub name: String,
6836 pub mime_type: String,
6837}
6838
6839impl MCPResourceDescriptor {
6840 pub fn to_state(&self) -> State {
6841 state(&[
6842 ("uri", self.uri.clone().into()),
6843 ("name", self.name.clone().into()),
6844 ("mimeType", self.mime_type.clone().into()),
6845 ])
6846 }
6847}
6848
6849pub struct InMemoryMCPToolServer {
6850 tools: HashMap<String, (State, MCPCall)>,
6851}
6852
6853impl InMemoryMCPToolServer {
6854 pub fn new() -> Self {
6855 Self {
6856 tools: HashMap::new(),
6857 }
6858 }
6859 pub fn add_tool(&mut self, descriptor: State, handler: MCPCall) {
6860 if let Some(Value::String(name)) = descriptor.get("name") {
6861 self.tools.insert(name.clone(), (descriptor, handler));
6862 }
6863 }
6864 pub fn list_tools(&self) -> Vec<State> {
6865 let mut names: Vec<_> = self.tools.keys().cloned().collect();
6866 names.sort();
6867 names
6868 .into_iter()
6869 .filter_map(|name| self.tools.get(&name).map(|entry| entry.0.clone()))
6870 .collect()
6871 }
6872 pub fn call_tool(&self, name: &str, args: State) -> Result<Value> {
6873 let (_, handler) = self
6874 .tools
6875 .get(name)
6876 .ok_or_else(|| RuntimeError(format!("MCP tool not found: {name}")))?;
6877 handler(name, args)
6878 }
6879}
6880
6881pub struct InMemoryMCPContextServer {
6882 resources: HashMap<String, (MCPResourceDescriptor, MCPResourceRead)>,
6883}
6884
6885impl InMemoryMCPContextServer {
6886 pub fn new() -> Self {
6887 Self {
6888 resources: HashMap::new(),
6889 }
6890 }
6891 pub fn add_resource(
6892 &mut self,
6893 uri: &str,
6894 name: &str,
6895 mime_type: &str,
6896 reader: MCPResourceRead,
6897 ) {
6898 self.resources.insert(
6899 uri.to_string(),
6900 (
6901 MCPResourceDescriptor {
6902 uri: uri.to_string(),
6903 name: name.to_string(),
6904 mime_type: if mime_type.is_empty() {
6905 "application/json".to_string()
6906 } else {
6907 mime_type.to_string()
6908 },
6909 },
6910 reader,
6911 ),
6912 );
6913 }
6914 pub fn list_resources(&self) -> Vec<State> {
6915 let mut uris: Vec<_> = self.resources.keys().cloned().collect();
6916 uris.sort();
6917 uris.into_iter()
6918 .filter_map(|uri| self.resources.get(&uri).map(|entry| entry.0.to_state()))
6919 .collect()
6920 }
6921 pub fn read_resource(&self, uri: &str) -> Result<State> {
6922 let (descriptor, reader) = self
6923 .resources
6924 .get(uri)
6925 .ok_or_else(|| RuntimeError(format!("MCP resource not found: {uri}")))?;
6926 Ok(state(&[
6927 ("resource", Value::Object(descriptor.to_state())),
6928 ("content", reader(uri)?),
6929 ]))
6930 }
6931}
6932
6933pub struct MCPToolAdapter {
6934 pub client_call: MCPCall,
6935}
6936
6937impl MCPToolAdapter {
6938 pub fn tool_spec_from_descriptor(&self, descriptor: &State) -> ToolSpec {
6939 let name = string_field(descriptor, "name", "");
6940 let version = string_field(descriptor, "version", "v1");
6941 let annotations = match descriptor.get("annotations") {
6942 Some(Value::Object(state)) => state.clone(),
6943 _ => State::new(),
6944 };
6945 let side_effect = string_field(&annotations, "side_effect", "none");
6946 let risk_level = string_field(&annotations, "risk_level", "low");
6947 let idempotency_required = match annotations.get("idempotency_required") {
6948 Some(Value::Bool(value)) => *value,
6949 _ => side_effect != "none",
6950 };
6951 let approval_required = match annotations.get("approval_required") {
6952 Some(Value::Bool(value)) => *value,
6953 _ => false,
6954 };
6955 let sandbox_required = match annotations.get("sandbox_required") {
6956 Some(Value::Bool(value)) => *value,
6957 _ => false,
6958 };
6959 let sandbox_executor = string_field(&annotations, "sandbox_executor", "");
6960 let sandbox_policy = match annotations.get("sandbox_policy") {
6961 Some(Value::Object(state)) => state.clone(),
6962 _ => State::new(),
6963 };
6964 let client_call = self.client_call;
6965 let tool_name = name.clone();
6966 let mut spec = ToolSpec::new(&name, Box::new(move |args| client_call(&tool_name, args)));
6967 spec.version = version;
6968 spec.side_effect = side_effect;
6969 spec.risk_level = risk_level;
6970 spec.idempotency_required = idempotency_required;
6971 spec.approval_required = approval_required;
6972 spec.sandbox_required = sandbox_required;
6973 spec.sandbox_executor = sandbox_executor;
6974 spec.sandbox_policy = sandbox_policy;
6975 spec.input_schema = descriptor
6976 .get("inputSchema")
6977 .or_else(|| descriptor.get("input_schema"))
6978 .cloned();
6979 spec.output_schema = descriptor
6980 .get("outputSchema")
6981 .or_else(|| descriptor.get("output_schema"))
6982 .cloned();
6983 spec
6984 }
6985}
6986
6987pub struct MCPContextAdapter {
6988 pub resource_read: MCPResourceRead,
6989}
6990
6991impl MCPContextAdapter {
6992 pub fn read_tool_spec(&self, name: &str, risk_level: &str) -> ToolSpec {
6993 let tool_name = if name.is_empty() {
6994 "mcp.context.read"
6995 } else {
6996 name
6997 };
6998 let risk = if risk_level.is_empty() {
6999 "low"
7000 } else {
7001 risk_level
7002 };
7003 let reader = self.resource_read;
7004 let mut spec = ToolSpec::new(
7005 tool_name,
7006 Box::new(move |args| match args.get("uri") {
7007 Some(Value::String(uri)) => reader(uri),
7008 _ => Err(RuntimeError("uri is required".to_string())),
7009 }),
7010 );
7011 spec.risk_level = risk.to_string();
7012 spec.side_effect = "none".to_string();
7013 spec.input_schema = Some(Value::Object(state(&[("type", "object".into())])));
7014 spec.output_schema = Some(Value::Object(state(&[("type", "object".into())])));
7015 spec
7016 }
7017}
7018
7019fn string_field(state: &State, key: &str, fallback: &str) -> String {
7020 match state.get(key) {
7021 Some(Value::String(value)) => value.clone(),
7022 _ => fallback.to_string(),
7023 }
7024}
7025
7026pub type FrameworkAgentFunc = fn(&mut AgentContext, State) -> Result<Option<Value>>;
7027
7028pub struct FunctionAdapter {
7029 pub func: FrameworkAgentFunc,
7030 pub role: String,
7031 pub name: String,
7032}
7033
7034impl FunctionAdapter {
7035 pub fn new(func: FrameworkAgentFunc, role: &str) -> Self {
7036 Self {
7037 func,
7038 role: if role.is_empty() {
7039 "Agent".to_string()
7040 } else {
7041 role.to_string()
7042 },
7043 name: "function".to_string(),
7044 }
7045 }
7046 pub fn map_run_spec(&self) -> State {
7047 state(&[
7048 ("adapter", self.name.clone().into()),
7049 ("role", self.role.clone().into()),
7050 ])
7051 }
7052 pub fn run(&self, ctx: &mut AgentContext, state_value: State, output_key: &str) -> Result<()> {
7053 if let Some(result) = (self.func)(ctx, state_value)? {
7054 if !output_key.is_empty() {
7055 ctx.write_state(output_key, result);
7056 }
7057 }
7058 Ok(())
7059 }
7060}
7061
7062pub type MethodHandler = fn(State) -> Result<Value>;
7063
7064pub struct MethodFrameworkAdapter {
7065 pub target_name: String,
7066 pub role: String,
7067 pub method_candidates: Vec<String>,
7068 pub methods: HashMap<String, MethodHandler>,
7069 pub output_key: String,
7070}
7071
7072impl MethodFrameworkAdapter {
7073 pub fn new(
7074 target_name: &str,
7075 role: &str,
7076 method_candidates: Vec<String>,
7077 methods: HashMap<String, MethodHandler>,
7078 output_key: &str,
7079 ) -> Self {
7080 Self {
7081 target_name: target_name.to_string(),
7082 role: if role.is_empty() {
7083 "FrameworkAgent".to_string()
7084 } else {
7085 role.to_string()
7086 },
7087 method_candidates,
7088 methods,
7089 output_key: if output_key.is_empty() {
7090 "output".to_string()
7091 } else {
7092 output_key.to_string()
7093 },
7094 }
7095 }
7096 pub fn map_run_spec(&self) -> State {
7097 state(&[
7098 ("adapter", "method-framework".into()),
7099 ("role", self.role.clone().into()),
7100 ("target", self.target_name.clone().into()),
7101 (
7102 "methods",
7103 Value::Array(
7104 self.method_candidates
7105 .iter()
7106 .map(|item| Value::String(item.clone()))
7107 .collect(),
7108 ),
7109 ),
7110 ])
7111 }
7112 pub fn run(&self, ctx: &mut AgentContext, state_value: State) -> Result<()> {
7113 for name in &self.method_candidates {
7114 if let Some(handler) = self.methods.get(name) {
7115 let result = handler(state_value)?;
7116 if !self.output_key.is_empty() {
7117 ctx.write_state(&self.output_key, result);
7118 }
7119 return Ok(());
7120 }
7121 }
7122 Err(RuntimeError(
7123 "target does not expose any candidate method".to_string(),
7124 ))
7125 }
7126}
7127
7128#[derive(Clone, Debug)]
7129pub struct BoundaryLintRule {
7130 pub rule_id: String,
7131 pub pattern: String,
7132 pub category: String,
7133 pub message: String,
7134 pub suggestion: String,
7135 pub prefix: bool,
7136}
7137
7138#[derive(Clone, Debug)]
7139pub struct BoundaryLintFinding {
7140 pub path: String,
7141 pub line: usize,
7142 pub column: usize,
7143 pub rule_id: String,
7144 pub severity: String,
7145 pub callee: String,
7146 pub category: String,
7147 pub message: String,
7148 pub suggestion: String,
7149}
7150
7151#[derive(Clone, Debug)]
7152pub struct BoundaryLintReport {
7153 pub passed: bool,
7154 pub scanned_files: Vec<String>,
7155 pub finding_count: usize,
7156 pub findings: Vec<BoundaryLintFinding>,
7157}
7158
7159pub fn default_boundary_rules() -> Vec<BoundaryLintRule> {
7160 vec![
7161 BoundaryLintRule { rule_id: "direct-shell-os-system".into(), pattern: "os.system".into(), category: "shell".into(), message: "direct shell execution bypasses ToolGateway, policy, ledger, sandbox, and audit".into(), suggestion: "wrap shell execution as a runtime-managed tool".into(), prefix: false },
7162 BoundaryLintRule { rule_id: "direct-shell-subprocess".into(), pattern: "subprocess.".into(), category: "shell".into(), message: "direct subprocess execution bypasses ToolGateway, policy, ledger, sandbox, and audit".into(), suggestion: "wrap command execution as a runtime-managed tool".into(), prefix: true },
7163 BoundaryLintRule { rule_id: "direct-http-requests".into(), pattern: "requests.".into(), category: "network".into(), message: "direct HTTP calls bypass ToolGateway, policy, ledger, budget, replay, and audit".into(), suggestion: "register the HTTP/API call as a runtime-managed tool".into(), prefix: true },
7164 BoundaryLintRule { rule_id: "direct-http-httpx".into(), pattern: "httpx.".into(), category: "network".into(), message: "direct HTTP calls bypass ToolGateway, policy, ledger, budget, replay, and audit".into(), suggestion: "register the HTTP/API call as a runtime-managed tool".into(), prefix: true },
7165 BoundaryLintRule { rule_id: "direct-openai-sdk".into(), pattern: "openai.".into(), category: "model".into(), message: "direct model SDK usage bypasses model provider archives, replay, budget, and attribution".into(), suggestion: "call models through the runtime model boundary".into(), prefix: true },
7166 BoundaryLintRule { rule_id: "direct-anthropic-sdk".into(), pattern: "anthropic.".into(), category: "model".into(), message: "direct model SDK usage bypasses model provider archives, replay, budget, and attribution".into(), suggestion: "call models through the runtime model boundary".into(), prefix: true },
7167 ]
7168}
7169
7170pub fn scan_boundary_source(
7171 path: &str,
7172 source: &str,
7173 rules: Option<Vec<BoundaryLintRule>>,
7174) -> BoundaryLintReport {
7175 let rules = rules.unwrap_or_else(default_boundary_rules);
7176 let lines: Vec<&str> = source.split('\n').collect();
7177 let mut findings = Vec::new();
7178 for (i, line) in lines.iter().enumerate() {
7179 let previous = if i > 0 { lines[i - 1] } else { "" };
7180 if line.contains("agentledger: ignore-boundary")
7181 || previous.contains("agentledger: ignore-next-line")
7182 {
7183 continue;
7184 }
7185 for rule in &rules {
7186 if let Some(index) = line.find(&rule.pattern) {
7187 let mut callee = rule.pattern.clone();
7188 if rule.prefix {
7189 let mut end = index + rule.pattern.len();
7190 while end < line.len() {
7191 let ch = line.as_bytes()[end] as char;
7192 if ch.is_ascii_alphanumeric() || ch == '_' || ch == '.' {
7193 end += 1;
7194 } else {
7195 break;
7196 }
7197 }
7198 callee = line[index..end].to_string();
7199 }
7200 findings.push(BoundaryLintFinding {
7201 path: path.into(),
7202 line: i + 1,
7203 column: index + 1,
7204 rule_id: rule.rule_id.clone(),
7205 severity: "error".into(),
7206 callee,
7207 category: rule.category.clone(),
7208 message: rule.message.clone(),
7209 suggestion: rule.suggestion.clone(),
7210 });
7211 break;
7212 }
7213 }
7214 }
7215 BoundaryLintReport {
7216 passed: findings.is_empty(),
7217 scanned_files: vec![path.into()],
7218 finding_count: findings.len(),
7219 findings,
7220 }
7221}
7222
7223#[derive(Clone, Debug)]
7224pub struct RecoverySummary {
7225 pub recovered_steps: usize,
7226}
7227
7228#[derive(Clone, Debug)]
7229pub struct SchedulerStepStatus {
7230 pub step_id: String,
7231 pub status: String,
7232 pub owner: Option<String>,
7233 pub attempt: u64,
7234 pub lease_until: Option<f64>,
7235 pub last_error_type: Option<String>,
7236}
7237
7238#[derive(Clone, Debug)]
7239pub struct SchedulerStatus {
7240 pub run_id: String,
7241 pub run_status: String,
7242 pub state_version: u64,
7243 pub steps: Vec<SchedulerStepStatus>,
7244 pub cost_summary: CostSummary,
7245}
7246
7247pub struct RuntimeScheduler;
7248
7249impl RuntimeScheduler {
7250 pub fn recover_expired_leases(store: &mut MemoryStore) -> RecoverySummary {
7251 RecoverySummary {
7252 recovered_steps: store.recover_expired_leases(),
7253 }
7254 }
7255
7256 pub fn cancel_run(store: &mut MemoryStore, run_id: &str, reason: &str) -> Result<usize> {
7257 store.cancel_run(run_id, reason)
7258 }
7259
7260 pub fn status(store: &MemoryStore, run_id: &str) -> Result<SchedulerStatus> {
7261 let run = store.run(run_id)?;
7262 let steps = store
7263 .steps(run_id)
7264 .into_iter()
7265 .map(|step| SchedulerStepStatus {
7266 step_id: step.step_id,
7267 status: step.status,
7268 owner: step.owner,
7269 attempt: step.attempt,
7270 lease_until: step.lease_until,
7271 last_error_type: step.last_error_type,
7272 })
7273 .collect();
7274 Ok(SchedulerStatus {
7275 run_id: run_id.to_string(),
7276 run_status: run.status,
7277 state_version: run.state_version,
7278 steps,
7279 cost_summary: store.cost_summary(run_id),
7280 })
7281 }
7282}
7283
7284#[derive(Clone, Debug)]
7285pub struct ReviewCheck {
7286 pub name: String,
7287 pub passed: bool,
7288 pub severity: String,
7289 pub detail: String,
7290}
7291
7292#[derive(Clone, Debug)]
7293pub struct AdversarialReviewReport {
7294 pub passed: bool,
7295 pub run_id: Option<String>,
7296 pub checks: Vec<ReviewCheck>,
7297 pub metadata: State,
7298}
7299
7300pub fn adversarial_review(
7301 bundle: &EvidenceBundle,
7302 max_total_usd: Option<f64>,
7303) -> AdversarialReviewReport {
7304 let mut checks = vec![
7305 review_check(
7306 "no_failed_steps",
7307 !bundle
7308 .events
7309 .iter()
7310 .any(|event| event.event_type == "step_failed"),
7311 "blocker",
7312 "no step is in failed status",
7313 ),
7314 review_check(
7315 "no_pending_verification",
7316 !bundle
7317 .tool_ledger
7318 .iter()
7319 .any(|row| row.status == "PENDING_VERIFICATION"),
7320 "blocker",
7321 "no side effect is pending verification",
7322 ),
7323 review_check(
7324 "no_pending_approvals",
7325 !bundle.approvals.iter().any(|row| row.status == "PENDING"),
7326 "blocker",
7327 "no approval request is still pending",
7328 ),
7329 review_check(
7330 "completed_steps_have_completion_events",
7331 completed_steps_have_events_rust(&bundle.steps, &bundle.events),
7332 "blocker",
7333 "completed steps have step_completed events",
7334 ),
7335 review_check(
7336 "ledger_statuses_known",
7337 ledger_statuses_known_rust(&bundle.tool_ledger),
7338 "blocker",
7339 "Tool Ledger rows use known statuses",
7340 ),
7341 review_check(
7342 "event_sequence_contiguous",
7343 event_sequence_contiguous_rust(&bundle.events),
7344 "blocker",
7345 "event sequence has no gaps",
7346 ),
7347 review_check(
7348 "artifacts_have_blob_refs",
7349 bundle
7350 .artifacts
7351 .iter()
7352 .all(|row| !row.blob_ref.is_empty() && !row.blob_hash.is_empty()),
7353 "warning",
7354 "artifacts have blob refs and hashes",
7355 ),
7356 review_check(
7357 "media_artifacts_have_refs",
7358 bundle
7359 .media_artifacts
7360 .iter()
7361 .all(media_artifact_has_ref_rust),
7362 "blocker",
7363 "media artifacts have kind and durable refs",
7364 ),
7365 review_check(
7366 "stream_checkpoints_have_offsets",
7367 bundle
7368 .stream_checkpoints
7369 .iter()
7370 .all(stream_checkpoint_has_offset_rust),
7371 "blocker",
7372 "stream checkpoints have stream, consumer, and offset",
7373 ),
7374 review_check(
7375 "high_risk_approvals_decided",
7376 high_risk_approvals_decided_rust(&bundle.approvals),
7377 "blocker",
7378 "high-risk approval requests are decided",
7379 ),
7380 review_check(
7381 "no_blocking_failure_events",
7382 !bundle.events.iter().any(|event| {
7383 matches!(
7384 event.event_type.as_str(),
7385 "error_raised" | "step_failed" | "tool_call_failed" | "tool_call_blocked"
7386 )
7387 }),
7388 "warning",
7389 "no blocking failure events are present",
7390 ),
7391 ];
7392 if let Some(limit) = max_total_usd {
7393 checks.push(review_check(
7394 "max_total_usd",
7395 bundle.cost_summary.total_usd <= limit,
7396 "blocker",
7397 "cost limit check",
7398 ));
7399 }
7400 let passed = checks
7401 .iter()
7402 .all(|check| check.severity != "blocker" || check.passed);
7403 let mut metadata = State::new();
7404 metadata.insert(
7405 "event_count".into(),
7406 Value::Number(bundle.events.len() as f64),
7407 );
7408 metadata.insert(
7409 "tool_ledger_count".into(),
7410 Value::Number(bundle.tool_ledger.len() as f64),
7411 );
7412 metadata.insert(
7413 "approval_count".into(),
7414 Value::Number(bundle.approvals.len() as f64),
7415 );
7416 metadata.insert(
7417 "artifact_count".into(),
7418 Value::Number(bundle.artifacts.len() as f64),
7419 );
7420 metadata.insert(
7421 "media_artifact_count".into(),
7422 Value::Number(bundle.media_artifacts.len() as f64),
7423 );
7424 metadata.insert(
7425 "stream_checkpoint_count".into(),
7426 Value::Number(bundle.stream_checkpoints.len() as f64),
7427 );
7428 AdversarialReviewReport {
7429 passed,
7430 run_id: Some(bundle.run.run_id.clone()),
7431 checks,
7432 metadata,
7433 }
7434}
7435
7436fn review_check(name: &str, passed: bool, severity: &str, detail: &str) -> ReviewCheck {
7437 ReviewCheck {
7438 name: name.into(),
7439 passed,
7440 severity: severity.into(),
7441 detail: detail.into(),
7442 }
7443}
7444
7445fn completed_steps_have_events_rust(steps: &[Step], events: &[Event]) -> bool {
7446 steps.iter().all(|step| {
7447 step.status != "completed"
7448 || events.iter().any(|event| {
7449 event.event_type == "step_completed"
7450 && event.step_id.as_deref() == Some(step.step_id.as_str())
7451 })
7452 })
7453}
7454
7455fn ledger_statuses_known_rust(rows: &[ToolLedgerEntry]) -> bool {
7456 rows.iter().all(|row| {
7457 matches!(
7458 row.status.as_str(),
7459 "SUCCEEDED"
7460 | "FAILED_NO_EFFECT"
7461 | "PENDING_VERIFICATION"
7462 | "COMPENSATED"
7463 | "RUNNING"
7464 | "RESERVED"
7465 )
7466 })
7467}
7468fn event_sequence_contiguous_rust(events: &[Event]) -> bool {
7469 events
7470 .iter()
7471 .enumerate()
7472 .all(|(index, event)| event.seq == (index as u64) + 1)
7473}
7474fn state_has_key(row: &State, key: &str) -> bool {
7475 !matches!(row.get(key), None | Some(Value::Null))
7476}
7477fn media_artifact_has_ref_rust(row: &State) -> bool {
7478 state_has_key(row, "kind")
7479 && (state_has_key(row, "uri")
7480 || state_has_key(row, "content_ref")
7481 || state_has_key(row, "blob_ref"))
7482}
7483fn stream_checkpoint_has_offset_rust(row: &State) -> bool {
7484 state_has_key(row, "stream_id")
7485 && state_has_key(row, "consumer_id")
7486 && state_has_key(row, "offset")
7487}
7488fn high_risk_approvals_decided_rust(rows: &[ApprovalRequest]) -> bool {
7489 rows.iter().all(|row| {
7490 !matches!(
7491 row.risk_level.as_str(),
7492 "high" | "destructive" | "sensitive"
7493 ) || matches!(row.status.as_str(), "APPROVED" | "DENIED")
7494 })
7495}
7496
7497#[derive(Clone, Debug)]
7498pub struct EvidenceCheck {
7499 pub name: String,
7500 pub passed: bool,
7501 pub detail: String,
7502}
7503
7504#[derive(Clone, Debug)]
7505pub struct EvidenceCheckReport {
7506 pub passed: bool,
7507 pub checks: Vec<EvidenceCheck>,
7508 pub metadata: State,
7509}
7510
7511pub fn evaluate_evidence(
7512 bundle: &EvidenceBundle,
7513 max_total_usd: Option<f64>,
7514) -> EvidenceCheckReport {
7515 let mut checks = vec![
7516 evidence_check(
7517 "no_failed_steps",
7518 !bundle
7519 .events
7520 .iter()
7521 .any(|event| event.event_type == "step_failed"),
7522 "all steps completed or remain non-failed",
7523 ),
7524 evidence_check(
7525 "no_pending_verification",
7526 !bundle
7527 .tool_ledger
7528 .iter()
7529 .any(|row| row.status == "PENDING_VERIFICATION"),
7530 "no side effect is waiting for human/external verification",
7531 ),
7532 evidence_check(
7533 "completed_steps_have_events",
7534 completed_steps_have_events_rust(&bundle.steps, &bundle.events),
7535 "each completed step has a step_completed event",
7536 ),
7537 evidence_check(
7538 "managed_side_effects_are_ledgered",
7539 ledger_statuses_known_rust(&bundle.tool_ledger),
7540 "every ledger row has a known status",
7541 ),
7542 evidence_check(
7543 "media_artifacts_have_refs",
7544 bundle
7545 .media_artifacts
7546 .iter()
7547 .all(media_artifact_has_ref_rust),
7548 "media artifacts have kind and durable refs",
7549 ),
7550 evidence_check(
7551 "stream_checkpoints_have_offsets",
7552 bundle
7553 .stream_checkpoints
7554 .iter()
7555 .all(stream_checkpoint_has_offset_rust),
7556 "stream checkpoints have stream, consumer, and offset",
7557 ),
7558 ];
7559 if let Some(limit) = max_total_usd {
7560 checks.push(evidence_check(
7561 "max_total_usd",
7562 bundle.cost_summary.total_usd <= limit,
7563 "cost limit check",
7564 ));
7565 }
7566 EvidenceCheckReport {
7567 passed: checks.iter().all(|check| check.passed),
7568 checks,
7569 metadata: State::new(),
7570 }
7571}
7572
7573pub fn evaluate_evidence_regression(
7574 golden: &EvidenceBundle,
7575 current: &EvidenceBundle,
7576 max_total_usd_delta: Option<f64>,
7577) -> EvidenceCheckReport {
7578 let diff = diff_evidence(golden, current);
7579 let mut checks = vec![
7580 evidence_check(
7581 "final_state_regression",
7582 diff.final_state_changed_count == 0,
7583 "final state regression check",
7584 ),
7585 evidence_check(
7586 "event_type_regression",
7587 diff.event_types_changed_count == 0,
7588 "event type regression check",
7589 ),
7590 evidence_check(
7591 "tool_ledger_status_regression",
7592 true,
7593 "tool ledger status regression check",
7594 ),
7595 evidence_check(
7596 "media_artifact_regression",
7597 diff.media_artifacts_changed_count == 0,
7598 "media artifact regression check",
7599 ),
7600 evidence_check(
7601 "stream_checkpoint_regression",
7602 diff.stream_checkpoints_changed_count == 0,
7603 "stream checkpoint regression check",
7604 ),
7605 ];
7606 if let Some(limit) = max_total_usd_delta {
7607 let delta = current.cost_summary.total_usd - golden.cost_summary.total_usd;
7608 checks.push(evidence_check(
7609 "max_total_usd_delta",
7610 delta <= limit,
7611 "cost delta limit check",
7612 ));
7613 }
7614 EvidenceCheckReport {
7615 passed: checks.iter().all(|check| check.passed),
7616 checks,
7617 metadata: State::new(),
7618 }
7619}
7620
7621fn evidence_check(name: &str, passed: bool, detail: &str) -> EvidenceCheck {
7622 EvidenceCheck {
7623 name: name.into(),
7624 passed,
7625 detail: detail.into(),
7626 }
7627}
7628
7629#[derive(Clone, Debug)]
7630pub struct FailureInjectionCheck {
7631 pub name: String,
7632 pub passed: bool,
7633 pub detail: String,
7634 pub run_id: Option<String>,
7635}
7636
7637#[derive(Clone, Debug)]
7638pub struct FailureInjectionReport {
7639 pub passed: bool,
7640 pub checks: Vec<FailureInjectionCheck>,
7641}
7642
7643pub fn run_failure_injection_suite() -> FailureInjectionReport {
7644 let checks = vec![
7645 failure_retry_exhaustion(),
7646 failure_lease_fencing(),
7647 failure_cancellation_fencing(),
7648 failure_side_effect_idempotency(),
7649 ];
7650 FailureInjectionReport {
7651 passed: checks.iter().all(|check| check.passed),
7652 checks,
7653 }
7654}
7655fn failure_check(
7656 name: &str,
7657 passed: bool,
7658 detail: String,
7659 run_id: String,
7660) -> FailureInjectionCheck {
7661 FailureInjectionCheck {
7662 name: name.into(),
7663 passed,
7664 detail,
7665 run_id: Some(run_id),
7666 }
7667}
7668fn failure_retry_exhaustion() -> FailureInjectionCheck {
7669 let mut runtime = Runtime::new();
7670 let (run_id, _) = runtime.create_run(State::new());
7671 let _ = runtime.run_once(
7672 &run_id,
7673 "retry-1",
7674 "FailureInjector",
7675 60.0,
7676 |_ctx, _state| Err(RuntimeError("retryable".into())),
7677 );
7678 let _ = runtime.run_once(
7679 &run_id,
7680 "retry-2",
7681 "FailureInjector",
7682 60.0,
7683 |_ctx, _state| Err(RuntimeError("final failure".into())),
7684 );
7685 let status = runtime
7686 .store
7687 .run(&run_id)
7688 .map(|run| run.status)
7689 .unwrap_or_else(|_| "missing".into());
7690 failure_check(
7691 "retry_exhaustion",
7692 status == "failed",
7693 format!("run_status={status}"),
7694 run_id,
7695 )
7696}
7697fn failure_lease_fencing() -> FailureInjectionCheck {
7698 let mut store = MemoryStore::new();
7699 let (run_id, step_id) = store.create_run(State::new());
7700 let claim = store.claim_step("stale-worker", &run_id, 0.0).unwrap();
7701 let recovered = store.recover_expired_leases();
7702 let stale_rejected = store
7703 .commit_state_patch(&run_id, &step_id, &claim.lease_token, 0, State::new())
7704 .is_err();
7705 let fresh = store.claim_step("fresh-worker", &run_id, 60.0).unwrap();
7706 let passed = recovered == 1 && stale_rejected && fresh.attempt == 2;
7707 failure_check(
7708 "lease_fencing",
7709 passed,
7710 format!("recovered_steps={recovered} stale_rejected={stale_rejected}"),
7711 run_id,
7712 )
7713}
7714fn failure_cancellation_fencing() -> FailureInjectionCheck {
7715 let mut store = MemoryStore::new();
7716 let (run_id, step_id) = store.create_run(State::new());
7717 let claim = store.claim_step("stale-worker", &run_id, 60.0).unwrap();
7718 let cancelled = store.cancel_run(&run_id, "failure injection").unwrap();
7719 let stale_rejected = store
7720 .commit_state_patch(&run_id, &step_id, &claim.lease_token, 0, State::new())
7721 .is_err();
7722 let fresh = store.claim_step("fresh-worker", &run_id, 60.0).is_err();
7723 let status = store.run(&run_id).map(|run| run.status).unwrap_or_default();
7724 let passed = cancelled == 1 && stale_rejected && fresh && status == "cancelled";
7725 failure_check(
7726 "cancellation_fencing",
7727 passed,
7728 format!("cancelled_steps={cancelled} stale_rejected={stale_rejected}"),
7729 run_id,
7730 )
7731}
7732fn failure_side_effect_idempotency() -> FailureInjectionCheck {
7733 use std::sync::{Arc, Mutex};
7734 let calls = Arc::new(Mutex::new(0usize));
7735 let calls_for_tool = Arc::clone(&calls);
7736 let mut runtime = Runtime::new();
7737 runtime.register_tool(
7738 ToolSpec::new(
7739 "external.create",
7740 Box::new(move |_args| {
7741 let mut guard = calls_for_tool.lock().unwrap();
7742 *guard += 1;
7743 Ok(Value::Object(state(&[(
7744 "id",
7745 Value::String("EXT-1".into()),
7746 )])))
7747 }),
7748 )
7749 .side_effect("external")
7750 .idempotency_required(true),
7751 );
7752 let (run_id, _) = runtime.create_run(State::new());
7753 let ctx = failure_claim_context(&mut runtime, &run_id, "worker-1", "FailureInjector");
7754 let _ = runtime.call_tool(
7755 &ctx,
7756 "external.create",
7757 state(&[("title", Value::String("once".into()))]),
7758 );
7759 runtime
7760 .store
7761 .mark_retry(&run_id, &ctx.step_id, "RetryableAgentError", "retryable");
7762 let ctx2 = failure_claim_context(&mut runtime, &run_id, "worker-2", "FailureInjector");
7763 let _ = runtime.call_tool(
7764 &ctx2,
7765 "external.create",
7766 state(&[("title", Value::String("once".into()))]),
7767 );
7768 let count = *calls.lock().unwrap();
7769 failure_check(
7770 "side_effect_idempotency",
7771 count == 1,
7772 format!("external_call_count={count}"),
7773 run_id,
7774 )
7775}
7776
7777fn failure_claim_context(
7778 runtime: &mut Runtime,
7779 run_id: &str,
7780 worker_id: &str,
7781 agent_role: &str,
7782) -> AgentContext {
7783 let claim = runtime.store.claim_step(worker_id, run_id, 60.0).unwrap();
7784 AgentContext {
7785 run_id: claim.run_id,
7786 session_id: claim.session_id,
7787 step_id: claim.step_id,
7788 agent_role: agent_role.to_string(),
7789 lease_token: claim.lease_token,
7790 attempt: claim.attempt,
7791 state_version: claim.state_version,
7792 pending_patch: State::new(),
7793 }
7794}
7795
7796#[derive(Clone, Debug)]
7797pub struct ShadowReport {
7798 pub source_run_id: String,
7799 pub shadow_run_id: String,
7800 pub ok: bool,
7801 pub state_diff: State,
7802}
7803
7804pub fn diff_states(source: &State, shadow: &State) -> State {
7805 let mut changed = State::new();
7806 for key in source.keys().chain(shadow.keys()) {
7807 if changed.contains_key(key) {
7808 continue;
7809 }
7810 if source.get(key) != shadow.get(key) {
7811 changed.insert(
7812 key.clone(),
7813 Value::Object(state(&[
7814 ("source", source.get(key).cloned().unwrap_or_default()),
7815 ("shadow", shadow.get(key).cloned().unwrap_or_default()),
7816 ])),
7817 );
7818 }
7819 }
7820 state(&[
7821 ("changed", Value::Object(changed.clone())),
7822 ("changed_count", Value::Number(changed.len() as f64)),
7823 ])
7824}
7825
7826pub fn shadow_report(
7827 source_run_id: &str,
7828 shadow_run_id: &str,
7829 ok: bool,
7830 source_state: &State,
7831 shadow_state: &State,
7832) -> ShadowReport {
7833 ShadowReport {
7834 source_run_id: source_run_id.into(),
7835 shadow_run_id: shadow_run_id.into(),
7836 ok,
7837 state_diff: diff_states(source_state, shadow_state),
7838 }
7839}
7840
7841pub fn builtin_golden_names() -> Vec<String> {
7842 vec![
7843 "media-stream-checkpoint".into(),
7844 "minimal-success".into(),
7845 "tool-ledger-success".into(),
7846 ]
7847}
7848
7849pub fn builtin_golden_evidence(name: &str) -> Result<EvidenceBundle> {
7850 match name {
7851 "minimal-success" => golden_minimal_success(),
7852 "tool-ledger-success" => golden_tool_ledger_success(),
7853 "media-stream-checkpoint" => golden_media_stream_checkpoint(),
7854 _ => Err(RuntimeError(format!(
7855 "unknown built-in golden case: {name}"
7856 ))),
7857 }
7858}
7859
7860pub fn golden_regression(golden: &EvidenceBundle, current: &EvidenceBundle) -> EvidenceCheckReport {
7861 evaluate_evidence_regression(golden, current, None)
7862}
7863
7864fn golden_minimal_success() -> Result<EvidenceBundle> {
7865 let mut runtime = Runtime::new();
7866 let (run_id, _) = runtime.create_run(State::new());
7867 runtime.run_once(
7868 &run_id,
7869 "golden-worker",
7870 "GoldenAgent",
7871 60.0,
7872 |ctx, _state| {
7873 ctx.write_state("answer", Value::String("ok".into()));
7874 Ok(())
7875 },
7876 )?;
7877 export_evidence(&runtime.store, &run_id)
7878}
7879fn golden_tool_ledger_success() -> Result<EvidenceBundle> {
7880 let mut runtime = Runtime::new();
7881 runtime.register_tool(
7882 ToolSpec::new(
7883 "github.create_issue",
7884 Box::new(|_args| {
7885 Ok(Value::Object(state(&[(
7886 "issue_id",
7887 Value::String("ISSUE-1".into()),
7888 )])))
7889 }),
7890 )
7891 .side_effect("external"),
7892 );
7893 let (run_id, _) = runtime.create_run(State::new());
7894 runtime.run_once(
7895 &run_id,
7896 "golden-worker",
7897 "ExecutorAgent",
7898 60.0,
7899 |ctx, _state| {
7900 ctx.write_state("issue_id", Value::String("ISSUE-1".into()));
7901 Ok(())
7902 },
7903 )?;
7904 export_evidence(&runtime.store, &run_id)
7905}
7906fn golden_media_stream_checkpoint() -> Result<EvidenceBundle> {
7907 let mut runtime = Runtime::new();
7908 let (run_id, _) = runtime.create_run(State::new());
7909 runtime.run_once(
7910 &run_id,
7911 "golden-worker",
7912 "MediaAgent",
7913 60.0,
7914 |ctx, _state| {
7915 ctx.write_state("processed_offset", Value::Number(42.0));
7916 Ok(())
7917 },
7918 )?;
7919 runtime.store.create_artifact(
7920 &run_id,
7921 None,
7922 "golden-video-frame",
7923 State::new(),
7924 state(&[(
7925 "agentledger_media",
7926 Value::Object(state(&[
7927 ("kind", Value::String("frame".into())),
7928 ("uri", Value::String("file://golden-frame.jpg".into())),
7929 ])),
7930 )]),
7931 );
7932 runtime.store.create_artifact(
7933 &run_id,
7934 None,
7935 "golden-stream-checkpoint",
7936 State::new(),
7937 state(&[(
7938 "agentledger_stream",
7939 Value::Object(state(&[
7940 ("stream_id", Value::String("stream-golden".into())),
7941 ("consumer_id", Value::String("consumer-golden".into())),
7942 ("offset", Value::Number(42.0)),
7943 ])),
7944 )]),
7945 );
7946 export_evidence(&runtime.store, &run_id)
7947}
7948
7949#[derive(Clone, Debug)]
7950pub struct TimeTravelFrame {
7951 pub seq: u64,
7952 pub event_id: String,
7953 pub event_type: String,
7954 pub step_id: Option<String>,
7955 pub agent_role: Option<String>,
7956 pub state_version: Option<u64>,
7957 pub timestamp: f64,
7958 pub state_changed: bool,
7959 pub changed_keys: Vec<String>,
7960 pub patch: Option<State>,
7961 pub state_after: Option<State>,
7962}
7963
7964#[derive(Clone, Debug)]
7965pub struct TimeTravelReport {
7966 pub run_id: String,
7967 pub at_seq: Option<u64>,
7968 pub event_count: usize,
7969 pub timeline: Vec<TimeTravelFrame>,
7970 pub state_at_seq: State,
7971 pub selected_event: Option<TimeTravelFrame>,
7972}
7973
7974pub fn time_travel(
7975 bundle: &EvidenceBundle,
7976 at_seq: Option<u64>,
7977 include_states: bool,
7978) -> TimeTravelReport {
7979 let mut current = State::new();
7980 let mut state_at_seq = State::new();
7981 let mut selected_event = None;
7982 let mut timeline = Vec::new();
7983 for event in &bundle.events {
7984 let before = current.clone();
7985 let patch = patch_for_time_travel_event(event);
7986 if let Some(patch_value) = &patch {
7987 for (key, value) in patch_value {
7988 current.insert(key.clone(), value.clone());
7989 }
7990 }
7991 let diff = diff_states(&before, ¤t);
7992 let changed_keys = match diff.get("changed") {
7993 Some(Value::Object(obj)) => obj.keys().cloned().collect(),
7994 _ => Vec::new(),
7995 };
7996 let frame = TimeTravelFrame {
7997 seq: event.seq,
7998 event_id: event.event_id.clone(),
7999 event_type: event.event_type.clone(),
8000 step_id: event.step_id.clone(),
8001 agent_role: event.agent_role.clone(),
8002 state_version: event.state_version,
8003 timestamp: event.timestamp,
8004 state_changed: diff.get("changed_count") != Some(&Value::Number(0.0)),
8005 changed_keys,
8006 patch,
8007 state_after: if include_states {
8008 Some(current.clone())
8009 } else {
8010 None
8011 },
8012 };
8013 if at_seq.is_some_and(|seq| event.seq <= seq) {
8014 state_at_seq = current.clone();
8015 selected_event = Some(frame.clone());
8016 }
8017 timeline.push(frame);
8018 }
8019 if at_seq.is_none() {
8020 state_at_seq = current.clone();
8021 }
8022 TimeTravelReport {
8023 run_id: bundle.run.run_id.clone(),
8024 at_seq,
8025 event_count: timeline.len(),
8026 timeline,
8027 state_at_seq,
8028 selected_event,
8029 }
8030}
8031
8032fn patch_for_time_travel_event(event: &Event) -> Option<State> {
8033 if event.event_type == "run_created" {
8034 if let Some(Value::Object(obj)) = event.payload.get("initial_state") {
8035 return Some(obj.clone());
8036 }
8037 return Some(State::new());
8038 }
8039 if event.event_type == "state_committed"
8040 || event.event_type == "state_patch_committed"
8041 || event.event_type == "system_state_patch_applied"
8042 {
8043 if let Some(Value::Object(obj)) = event.payload.get("patch") {
8044 return Some(obj.clone());
8045 }
8046 return Some(State::new());
8047 }
8048 None
8049}
8050
8051pub fn time_travel_html(report: &TimeTravelReport) -> String {
8052 let rows = report
8053 .timeline
8054 .iter()
8055 .map(|frame| {
8056 format!(
8057 "<tr><td>{}</td><td>{}</td><td>{}</td></tr>",
8058 frame.seq,
8059 frame.event_type,
8060 frame.changed_keys.join(", ")
8061 )
8062 })
8063 .collect::<Vec<_>>()
8064 .join("\n");
8065 format!("<!doctype html><html><head><meta charset=\"utf-8\"><title>AgentLedger Time Travel Report</title></head><body><h1>AgentLedger Time Travel Report</h1><p>Run <code>{}</code></p><table>{}</table><h2>State At Selected Point</h2><pre>{:?}</pre><h2>Selected Event</h2><pre>{:?}</pre></body></html>", report.run_id, rows, report.state_at_seq, report.selected_event)
8066}
8067
8068#[derive(Debug, Clone, PartialEq, Eq)]
8069pub struct OptionalAdapterCapability {
8070 pub name: String,
8071 pub category: String,
8072 pub core_imports_heavy_sdks: bool,
8073 pub adapter_is_optional: bool,
8074 pub fail_closed_without_adapter: bool,
8075 pub contract_surface: Vec<String>,
8076}
8077
8078pub fn optional_adapter_capabilities() -> Vec<OptionalAdapterCapability> {
8079 fn item(name: &str, category: &str, surface: &[&str]) -> OptionalAdapterCapability {
8080 OptionalAdapterCapability {
8081 name: name.to_string(),
8082 category: category.to_string(),
8083 core_imports_heavy_sdks: false,
8084 adapter_is_optional: true,
8085 fail_closed_without_adapter: true,
8086 contract_surface: surface.iter().map(|s| s.to_string()).collect(),
8087 }
8088 }
8089 vec![
8090 item("postgres", "storage", &["ddl_for", "migrations_for", "state_store"]),
8091 item("mysql", "storage", &["ddl_for", "migrations_for", "state_store"]),
8092 item("s3", "blobstore", &["put_json", "get_json", "content_address"]),
8093 item("docker", "sandbox", &["sandbox_policy", "sandbox_result", "tool_gateway"]),
8094 item("e2b", "sandbox", &["sandbox_policy", "sandbox_result", "tool_gateway"]),
8095 item("bubblewrap", "sandbox", &["sandbox_policy", "sandbox_result", "tool_gateway"]),
8096 item("kubernetes", "sandbox", &["sandbox_policy", "sandbox_result", "tool_gateway"]),
8097 item("gvisor", "sandbox", &["sandbox_policy", "sandbox_result", "tool_gateway"]),
8098 item("firecracker", "sandbox", &["sandbox_policy", "sandbox_result", "tool_gateway"]),
8099 item("langgraph", "framework", &["framework_adapter", "checkpoint_contract"]),
8100 item("langchain", "framework", &["framework_adapter"]),
8101 item("crewai", "framework", &["framework_adapter"]),
8102 item("autogen", "framework", &["framework_adapter"]),
8103 item("openai-agents-sdk", "framework", &["framework_adapter"]),
8104 item("llamaindex", "framework", &["framework_adapter"]),
8105 item("semantic-kernel", "framework", &["framework_adapter"]),
8106 item("mcp-transport", "mcp", &["mcp_tool_descriptor", "mcp_resource_descriptor"]),
8107 item("langfuse", "observability", &["evidence_bundle", "trace_payload", "correlation_ids"]),
8108 item("shadow-runner", "shadow", &["evidence_bundle", "tool_ledger", "state_diff"]),
8109 ]
8110}
8111
8112pub trait SqlExecutor {
8113 fn exec(&mut self, sql: &str, params: &[Value]) -> Result<()>;
8114}
8115
8116pub struct PostgresAdapter<C: SqlExecutor> {
8117 pub schema: String,
8118 pub client: C,
8119}
8120
8121impl<C: SqlExecutor> PostgresAdapter<C> {
8122 pub fn new(client: C, schema: &str) -> Self {
8123 Self { schema: if schema.is_empty() { "agentledger".to_string() } else { schema.to_string() }, client }
8124 }
8125 pub fn migration_plan(&self) -> Result<Vec<Migration>> { migrations_for("postgres") }
8126 pub fn apply_migrations(&mut self) -> Result<()> {
8127 self.client.exec(&ddl_for("postgres")?, &[])?;
8128 for migration in self.migration_plan()? {
8129 self.client.exec(
8130 "INSERT INTO schema_migrations(version, name, checksum) VALUES ($1, $2, $3) ON CONFLICT (version) DO NOTHING",
8131 &[Value::String(migration.version), Value::String(migration.name), Value::String(stable_hash(&migration.sql))],
8132 )?;
8133 }
8134 Ok(())
8135 }
8136}
8137
8138pub struct MySQLAdapter<C: SqlExecutor> {
8139 pub database: String,
8140 pub client: C,
8141}
8142
8143impl<C: SqlExecutor> MySQLAdapter<C> {
8144 pub fn new(client: C, database: &str) -> Self {
8145 Self { database: if database.is_empty() { "agentledger".to_string() } else { database.to_string() }, client }
8146 }
8147 pub fn migration_plan(&self) -> Result<Vec<Migration>> { migrations_for("mysql") }
8148 pub fn apply_migrations(&mut self) -> Result<()> {
8149 self.client.exec(&ddl_for("mysql")?, &[])?;
8150 for migration in self.migration_plan()? {
8151 self.client.exec(
8152 "INSERT INTO schema_migrations(version, name, checksum, applied_at) VALUES (?, ?, ?, UNIX_TIMESTAMP()) ON DUPLICATE KEY UPDATE version=version",
8153 &[Value::String(migration.version), Value::String(migration.name), Value::String(stable_hash(&migration.sql))],
8154 )?;
8155 }
8156 Ok(())
8157 }
8158}
8159
8160pub trait ObjectClient {
8161 fn put_object(&mut self, bucket: &str, key: &str, body: &[u8], content_type: &str, metadata: State) -> Result<()>;
8162 fn get_object(&mut self, bucket: &str, key: &str) -> Result<Vec<u8>>;
8163}
8164
8165pub struct S3BlobStoreAdapter<C: ObjectClient> {
8166 pub bucket: String,
8167 pub prefix: String,
8168 pub client: C,
8169}
8170
8171impl<C: ObjectClient> S3BlobStoreAdapter<C> {
8172 pub fn new(client: C, bucket: &str, prefix: &str) -> Self {
8173 Self { bucket: bucket.to_string(), prefix: if prefix.is_empty() { "agentledger/blobs".to_string() } else { prefix.trim_matches('/').to_string() }, client }
8174 }
8175 pub fn put_json(&mut self, value: &Value) -> Result<(String, String)> {
8176 let encoded = encode_value(value);
8177 let digest = stable_hash(&encoded);
8178 let key = format!("{}/sha256/{}.json", self.prefix, digest);
8179 let mut metadata = State::new();
8180 metadata.insert("agentledger-digest".to_string(), Value::String(format!("sha256:{digest}")));
8181 self.client.put_object(&self.bucket, &key, encoded.as_bytes(), "application/json", metadata)?;
8182 Ok((format!("sha256:{digest}"), format!("s3://{}/{}", self.bucket, key)))
8183 }
8184 pub fn get_json(&mut self, reference: &str) -> Result<Value> {
8185 let prefix = format!("s3://{}/", self.bucket);
8186 if !reference.starts_with(&prefix) || reference.contains("..") {
8187 return Err(RuntimeError(format!("unsupported s3 blob ref: {reference}")));
8188 }
8189 let key = &reference[prefix.len()..];
8190 let body = self.client.get_object(&self.bucket, key)?;
8191 let text = String::from_utf8(body).map_err(|err| RuntimeError(err.to_string()))?;
8192 decode_value(&text)
8193 }
8194}
8195
8196pub trait OtlpClient { fn post_json(&mut self, endpoint: &str, payload: &str, content_type: &str) -> Result<()>; }
8197pub struct OtlpTransport<C: OtlpClient> { pub endpoint: String, pub client: C }
8198impl<C: OtlpClient> OtlpTransport<C> { pub fn export(&mut self, payload: &str) -> Result<()> { self.client.post_json(&self.endpoint, payload, "application/json") } }
8199
8200pub struct DockerSandboxAdapter { pub image: String }
8201impl DockerSandboxAdapter {
8202 pub fn manifest(&self, policy: &State, command: Vec<String>) -> State {
8203 let mut out = State::new();
8204 out.insert("backend".to_string(), Value::String("docker".to_string()));
8205 out.insert("image".to_string(), Value::String(if self.image.is_empty() { "python:3.11-slim".to_string() } else { self.image.clone() }));
8206 let network = match policy.get("network") { Some(Value::String(value)) if value != "deny" => value.clone(), _ => "none".to_string() };
8207 out.insert("network".to_string(), Value::String(network));
8208 out.insert("read_only_root".to_string(), Value::Bool(true));
8209 out.insert("requires_explicit_execution".to_string(), Value::Bool(true));
8210 out.insert("command".to_string(), Value::Array(command.into_iter().map(Value::String).collect()));
8211 out
8212 }
8213}
8214
8215pub struct DockerSandboxExecutor {
8216 pub image: String,
8217 pub binary: String,
8218 pub allow_command_execution: bool,
8219 pub allow_shell: bool,
8220 pub shell: String,
8221 pub memory: String,
8222 pub cpus: String,
8223}
8224
8225impl DockerSandboxExecutor {
8226 pub fn new(image: &str, allow_command_execution: bool) -> Self {
8227 Self {
8228 image: image.to_string(),
8229 binary: "docker".to_string(),
8230 allow_command_execution,
8231 allow_shell: false,
8232 shell: "/bin/sh".to_string(),
8233 memory: String::new(),
8234 cpus: String::new(),
8235 }
8236 }
8237
8238 pub fn with_binary(mut self, binary: &str) -> Self {
8239 self.binary = binary.to_string();
8240 self
8241 }
8242
8243 fn extract_command(&self, args: &State) -> std::result::Result<Vec<String>, String> {
8244 let raw = args.get("_sandbox_command").or_else(|| args.get("command"));
8245 match raw {
8246 Some(Value::String(command)) => {
8247 if !self.allow_shell {
8248 Err("string commands require allow_shell=true; pass argv list in `_sandbox_command` instead".to_string())
8249 } else {
8250 let shell = if self.shell.is_empty() { "/bin/sh" } else { &self.shell };
8251 Ok(vec![shell.to_string(), "-lc".to_string(), command.clone()])
8252 }
8253 }
8254 Some(Value::Array(items)) => {
8255 let mut command = Vec::new();
8256 for item in items {
8257 match item {
8258 Value::String(value) if !value.is_empty() => command.push(value.clone()),
8259 _ => return Err("_sandbox_command must be a non-empty string array".to_string()),
8260 }
8261 }
8262 if command.is_empty() {
8263 Err("_sandbox_command must be a non-empty string array".to_string())
8264 } else {
8265 Ok(command)
8266 }
8267 }
8268 _ => Err("external sandbox tools require a command-style `_sandbox_command` arg".to_string()),
8269 }
8270 }
8271
8272 fn docker_argv(&self, policy: &SandboxPolicy, command: &[String]) -> Vec<String> {
8273 let image = if self.image.is_empty() { "python:3.11-slim" } else { &self.image };
8274 let network = if policy.network == "deny" || policy.network.is_empty() { "none" } else { &policy.network };
8275 let mut argv = vec![
8276 self.binary.clone(),
8277 "run".to_string(),
8278 "--rm".to_string(),
8279 "--network".to_string(),
8280 network.to_string(),
8281 "--read-only".to_string(),
8282 ];
8283 if !self.memory.is_empty() {
8284 argv.extend(["--memory".to_string(), self.memory.clone()]);
8285 }
8286 if !self.cpus.is_empty() {
8287 argv.extend(["--cpus".to_string(), self.cpus.clone()]);
8288 }
8289 argv.push(image.to_string());
8290 argv.extend(command.iter().cloned());
8291 argv
8292 }
8293
8294 fn result_error(policy: &SandboxPolicy, manifest: State, error_type: &str, error: String) -> SandboxResult {
8295 let mut metadata = State::new();
8296 metadata.insert("executor".to_string(), Value::String(policy.executor.clone()));
8297 metadata.insert("isolation_level".to_string(), Value::String("container".to_string()));
8298 metadata.insert("manifest".to_string(), Value::Object(manifest));
8299 metadata.insert("error_type".to_string(), Value::String(error_type.to_string()));
8300 SandboxResult { ok: false, output: Value::Null, error: Some(error), metadata }
8301 }
8302}
8303
8304impl SandboxExecutor for DockerSandboxExecutor {
8305 fn run_tool(&self, args: State, policy: &SandboxPolicy) -> SandboxResult {
8306 let command = match self.extract_command(&args) {
8307 Ok(command) => command,
8308 Err(error) => return Self::result_error(policy, State::new(), "InvalidSandboxCommand", error),
8309 };
8310 let mut policy_state = State::new();
8311 policy_state.insert("network".to_string(), Value::String(policy.network.clone()));
8312 let manifest = (DockerSandboxAdapter { image: self.image.clone() }).manifest(&policy_state, command.clone());
8313 if !self.allow_command_execution {
8314 return Self::result_error(policy, manifest, "SandboxAdapterNotInstalled", "command execution is not enabled for this executor".to_string());
8315 }
8316 let argv = self.docker_argv(policy, &command);
8317 let mut cmd = Command::new(&argv[0]);
8318 cmd.args(&argv[1..]).stdout(Stdio::piped()).stderr(Stdio::piped());
8319 let output = match cmd.output() {
8320 Ok(output) => output,
8321 Err(error) => return Self::result_error(policy, manifest, "SandboxBinaryMissing", error.to_string()),
8322 };
8323 let mut value = State::new();
8324 value.insert("stdout".to_string(), Value::String(String::from_utf8_lossy(&output.stdout).to_string()));
8325 value.insert("stderr".to_string(), Value::String(String::from_utf8_lossy(&output.stderr).to_string()));
8326 value.insert("returncode".to_string(), Value::Number(output.status.code().unwrap_or(-1) as f64));
8327 let mut metadata = State::new();
8328 metadata.insert("executor".to_string(), Value::String(policy.executor.clone()));
8329 metadata.insert("isolation_level".to_string(), Value::String("container".to_string()));
8330 metadata.insert("manifest".to_string(), Value::Object(manifest));
8331 metadata.insert("executed".to_string(), Value::Bool(true));
8332 if !output.status.success() {
8333 metadata.insert("error_type".to_string(), Value::String("SandboxCommandFailed".to_string()));
8334 return SandboxResult {
8335 ok: false,
8336 output: Value::Object(value),
8337 error: Some(format!("sandbox command exited with {}", output.status.code().unwrap_or(-1))),
8338 metadata,
8339 };
8340 }
8341 SandboxResult { ok: true, output: Value::Object(value), error: None, metadata }
8342 }
8343}
8344
8345pub mod adapters {
8346 pub mod postgres {
8347 pub const PACKAGE_NAME: &str = "agentledger-postgres";
8348 pub const FEATURE: &str = "adapter-postgres";
8349 pub use crate::{migrations_for, Migration, PostgresAdapter, SqlExecutor};
8350 }
8351
8352 pub mod mysql {
8353 pub const PACKAGE_NAME: &str = "agentledger-mysql";
8354 pub const FEATURE: &str = "adapter-mysql";
8355 pub use crate::{migrations_for, Migration, MySQLAdapter, SqlExecutor};
8356 }
8357
8358 pub mod s3 {
8359 pub const PACKAGE_NAME: &str = "agentledger-s3";
8360 pub const FEATURE: &str = "adapter-s3";
8361 pub use crate::{ObjectClient, S3BlobStoreAdapter};
8362 }
8363
8364 pub mod mcp {
8365 pub const PACKAGE_NAME: &str = "agentledger-mcp";
8366 pub const FEATURE: &str = "adapter-mcp";
8367 pub use crate::{
8368 InMemoryMCPContextServer, InMemoryMCPToolServer, MCPCall, MCPContextAdapter,
8369 MCPResourceDescriptor, MCPResourceRead, MCPToolAdapter,
8370 };
8371 }
8372
8373 pub mod otel {
8374 pub const PACKAGE_NAME: &str = "agentledger-otel";
8375 pub const FEATURE: &str = "adapter-otel";
8376 pub use crate::{OtlpClient, OtlpTransport};
8377 }
8378
8379 pub mod langfuse {
8380 pub const PACKAGE_NAME: &str = "agentledger-langfuse";
8381 pub const FEATURE: &str = "adapter-langfuse";
8382 pub const CATEGORY: &str = "observability";
8383 }
8384
8385 pub mod docker {
8386 pub const PACKAGE_NAME: &str = "agentledger-sandbox-docker";
8387 pub const FEATURE: &str = "adapter-docker";
8388 pub use crate::{DockerSandboxAdapter, DockerSandboxExecutor, State, Value};
8389 }
8390
8391 pub mod framework {
8392 pub const PACKAGE_NAME: &str = "agentledger-framework";
8393 pub const FEATURE: &str = "adapter-framework";
8394 pub use crate::{FunctionAdapter, MethodFrameworkAdapter};
8395 }
8396}