1use std::collections::HashMap;
2use std::fmt;
3use std::fs;
4use std::path::{Path, PathBuf};
5use std::process::{Command, Stdio};
6use std::sync::atomic::{AtomicU64, Ordering};
7use std::time::{SystemTime, UNIX_EPOCH};
8
9static ID_COUNTER: AtomicU64 = AtomicU64::new(1);
10
11pub type State = HashMap<String, Value>;
12
13#[derive(Clone, Debug, PartialEq)]
14pub enum Value {
15 Null,
16 Bool(bool),
17 Number(f64),
18 String(String),
19 Object(State),
20 Array(Vec<Value>),
21}
22
23impl Default for Value {
24 fn default() -> Self {
25 Value::Null
26 }
27}
28
29impl From<&str> for Value {
30 fn from(value: &str) -> Self {
31 Value::String(value.to_string())
32 }
33}
34
35impl From<String> for Value {
36 fn from(value: String) -> Self {
37 Value::String(value)
38 }
39}
40
41impl From<bool> for Value {
42 fn from(value: bool) -> Self {
43 Value::Bool(value)
44 }
45}
46
47impl From<i64> for Value {
48 fn from(value: i64) -> Self {
49 Value::Number(value as f64)
50 }
51}
52
53impl From<f64> for Value {
54 fn from(value: f64) -> Self {
55 Value::Number(value)
56 }
57}
58
59#[derive(Clone, Debug)]
60pub struct Run {
61 pub run_id: String,
62 pub session_id: String,
63 pub status: String,
64 pub state: State,
65 pub state_version: u64,
66 pub created_at: f64,
67 pub updated_at: f64,
68}
69
70#[derive(Clone, Debug)]
71pub struct Step {
72 pub step_id: String,
73 pub run_id: String,
74 pub session_id: String,
75 pub status: String,
76 pub owner: Option<String>,
77 pub lease_token: Option<String>,
78 pub lease_until: Option<f64>,
79 pub attempt: u64,
80 pub state_version: u64,
81 pub checkpoint_id: Option<String>,
82 pub last_error_type: Option<String>,
83 pub last_error: Option<String>,
84 pub cancelled_at: Option<f64>,
85 pub created_at: f64,
86 pub updated_at: f64,
87}
88
89#[derive(Clone, Debug)]
90pub struct StepClaim {
91 pub run_id: String,
92 pub session_id: String,
93 pub step_id: String,
94 pub attempt: u64,
95 pub lease_token: String,
96 pub state_version: u64,
97 pub lease_until: f64,
98}
99
100#[derive(Clone, Debug)]
101pub struct Event {
102 pub event_id: String,
103 pub run_id: String,
104 pub session_id: Option<String>,
105 pub step_id: Option<String>,
106 pub seq: u64,
107 pub event_type: String,
108 pub timestamp: f64,
109 pub agent_role: Option<String>,
110 pub state_version: Option<u64>,
111 pub causal_token: Option<String>,
112 pub payload_hash: String,
113 pub payload_ref: String,
114 pub payload: State,
115}
116
117#[derive(Clone, Debug)]
118pub struct ToolLedgerEntry {
119 pub ledger_id: String,
120 pub run_id: String,
121 pub session_id: String,
122 pub step_id: String,
123 pub tool_name: String,
124 pub tool_version: String,
125 pub tool_call_id: String,
126 pub idempotency_key: String,
127 pub causal_token: String,
128 pub request_hash: String,
129 pub request_ref: String,
130 pub status: String,
131 pub external_id: Option<String>,
132 pub response_hash: Option<String>,
133 pub response_ref: Option<String>,
134 pub error_type: Option<String>,
135 pub response: Option<Value>,
136 pub created_at: f64,
137 pub updated_at: f64,
138}
139
140#[derive(Clone, Debug)]
141pub struct ApprovalRequest {
142 pub approval_id: String,
143 pub approval_key: String,
144 pub run_id: String,
145 pub session_id: String,
146 pub step_id: String,
147 pub tool_name: String,
148 pub risk_level: String,
149 pub status: String,
150 pub reason: String,
151 pub request_hash: String,
152 pub request_ref: String,
153 pub requested_by: String,
154 pub approved_by: Option<String>,
155 pub decision_reason: Option<String>,
156 pub created_at: f64,
157 pub updated_at: f64,
158}
159
160#[derive(Clone, Debug)]
161pub struct CostRecord {
162 pub cost_id: String,
163 pub run_id: String,
164 pub session_id: String,
165 pub step_id: String,
166 pub category: String,
167 pub name: String,
168 pub amount: f64,
169 pub unit: String,
170 pub metadata: State,
171 pub created_at: f64,
172}
173
174#[derive(Clone, Debug, Default, PartialEq)]
175pub struct CostSummary {
176 pub tool_calls: f64,
177 pub model_tokens: f64,
178 pub total_usd: f64,
179 pub by_category: HashMap<String, f64>,
180}
181
182pub const MEDIA_SCHEMA_VERSION: &str = "agentledger.media.v0";
183pub const STREAM_SCHEMA_VERSION: &str = "agentledger.stream.v0";
184
185#[derive(Clone, Debug)]
186pub struct LocalBlobStore {
187 root: PathBuf,
188}
189
190impl LocalBlobStore {
191 pub fn open(root: impl AsRef<Path>) -> Result<Self> {
192 fs::create_dir_all(root.as_ref()).map_err(|err| RuntimeError(err.to_string()))?;
193 Ok(Self {
194 root: root.as_ref().to_path_buf(),
195 })
196 }
197
198 pub fn put_json(&self, value: &Value) -> Result<(String, String)> {
199 let encoded = encode_value(value);
200 let digest = stable_hash(&encoded);
201 let dir = self.root.join("sha256");
202 fs::create_dir_all(&dir).map_err(|err| RuntimeError(err.to_string()))?;
203 let path = dir.join(format!("{digest}.json"));
204 if !path.exists() {
205 let tmp = path.with_extension("json.tmp");
206 fs::write(&tmp, encoded).map_err(|err| RuntimeError(err.to_string()))?;
207 fs::rename(&tmp, &path).map_err(|err| RuntimeError(err.to_string()))?;
208 }
209 Ok((
210 format!("sha256:{digest}"),
211 format!("blob://sha256/{digest}.json"),
212 ))
213 }
214
215 pub fn get_json(&self, reference: &str) -> Result<Value> {
216 let prefix = "blob://sha256/";
217 if !reference.starts_with(prefix) {
218 return Err(RuntimeError(format!("unsupported blob ref: {reference}")));
219 }
220 let name = &reference[prefix.len()..];
221 if !name.ends_with(".json")
222 || name.contains("..")
223 || name.contains('/')
224 || name.contains('\\')
225 {
226 return Err(RuntimeError(format!("unsupported blob ref: {reference}")));
227 }
228 let body = fs::read_to_string(self.root.join("sha256").join(name))
229 .map_err(|err| RuntimeError(err.to_string()))?;
230 decode_value(&body)
231 }
232}
233
234#[derive(Clone, Debug)]
235pub struct Artifact {
236 pub artifact_id: String,
237 pub run_id: String,
238 pub step_id: Option<String>,
239 pub name: String,
240 pub blob_hash: String,
241 pub blob_ref: String,
242 pub metadata: State,
243 pub created_at: f64,
244}
245
246#[derive(Clone, Debug, Default)]
247pub struct MediaArtifactOptions {
248 pub uri: Option<String>,
249 pub content_ref: Option<String>,
250 pub media_metadata: State,
251 pub lineage: State,
252 pub derived_outputs: State,
253 pub metadata: State,
254}
255
256#[derive(Clone, Debug, Default)]
257pub struct StreamChunkRef {
258 pub stream_id: String,
259 pub chunk_id: String,
260 pub offset: Value,
261 pub content_ref: Option<String>,
262 pub content_hash: Option<String>,
263 pub sequence: Option<f64>,
264 pub event_time: Option<f64>,
265 pub metadata: State,
266}
267
268#[derive(Clone, Debug, Default)]
269pub struct StreamCheckpointOptions {
270 pub stream_id: String,
271 pub consumer_id: String,
272 pub offset: Value,
273 pub watermark: Option<Value>,
274 pub chunk: Option<StreamChunkRef>,
275 pub partial_result_ref: Option<String>,
276 pub backpressure: State,
277 pub metadata: State,
278}
279
280#[derive(Clone, Debug, Default)]
281pub struct BudgetLimits {
282 pub max_tool_calls: Option<f64>,
283 pub max_model_tokens: Option<f64>,
284 pub max_total_usd: Option<f64>,
285}
286
287#[derive(Debug, Clone)]
288pub struct RuntimeError(pub String);
289
290impl fmt::Display for RuntimeError {
291 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
292 write!(f, "{}", self.0)
293 }
294}
295
296impl std::error::Error for RuntimeError {}
297
298pub type Result<T> = std::result::Result<T, RuntimeError>;
299pub type ToolFunc = Box<dyn Fn(State) -> Result<Value> + Send + Sync>;
300pub type AgentFunc = fn(&mut AgentContext, State) -> Result<()>;
301
302pub struct SandboxPolicy {
303 pub tool_name: String,
304 pub run_id: String,
305 pub step_id: String,
306 pub executor: String,
307 pub network: String,
308 pub filesystem: String,
309 pub timeout_seconds: u64,
310 pub extra: State,
311}
312
313pub struct SandboxResult {
314 pub ok: bool,
315 pub output: Value,
316 pub error: Option<String>,
317 pub metadata: State,
318}
319
320pub trait SandboxExecutor {
321 fn run_tool(&self, args: State, policy: &SandboxPolicy) -> SandboxResult;
322}
323
324pub struct DisabledSandboxExecutor;
325
326impl SandboxExecutor for DisabledSandboxExecutor {
327 fn run_tool(&self, _args: State, policy: &SandboxPolicy) -> SandboxResult {
328 let mut metadata = State::new();
329 metadata.insert("executor".to_string(), Value::String(policy.executor.clone()));
330 metadata.insert("isolation_level".to_string(), Value::String("none".to_string()));
331 metadata.insert("fail_closed".to_string(), Value::Bool(true));
332 SandboxResult {
333 ok: false,
334 output: Value::Null,
335 error: Some(format!("sandbox executor \"{}\" is disabled", policy.executor)),
336 metadata,
337 }
338 }
339}
340
341pub struct ToolSpec {
342 pub name: String,
343 pub version: String,
344 pub side_effect: String,
345 pub risk_level: String,
346 pub idempotency_required: bool,
347 pub approval_required: bool,
348 pub sandbox_required: bool,
349 pub sandbox_executor: String,
350 pub sandbox_policy: State,
351 pub input_schema: Option<Value>,
352 pub output_schema: Option<Value>,
353 pub func: ToolFunc,
354}
355
356impl ToolSpec {
357 pub fn new(name: &str, func: ToolFunc) -> Self {
358 Self {
359 name: name.to_string(),
360 version: "v1".to_string(),
361 side_effect: "none".to_string(),
362 risk_level: "low".to_string(),
363 idempotency_required: false,
364 approval_required: false,
365 sandbox_required: false,
366 sandbox_executor: String::new(),
367 sandbox_policy: State::new(),
368 input_schema: None,
369 output_schema: None,
370 func,
371 }
372 }
373
374 pub fn side_effect(mut self, side_effect: &str) -> Self {
375 self.side_effect = side_effect.to_string();
376 self
377 }
378
379 pub fn risk_level(mut self, risk_level: &str) -> Self {
380 self.risk_level = risk_level.to_string();
381 self
382 }
383
384 pub fn idempotency_required(mut self, required: bool) -> Self {
385 self.idempotency_required = required;
386 self
387 }
388
389 pub fn approval_required(mut self, required: bool) -> Self {
390 self.approval_required = required;
391 self
392 }
393
394 pub fn sandbox_required(mut self, required: bool) -> Self {
395 self.sandbox_required = required;
396 self
397 }
398
399 pub fn sandbox_executor(mut self, executor: &str) -> Self {
400 self.sandbox_executor = executor.to_string();
401 self
402 }
403
404 pub fn sandbox_policy(mut self, policy: State) -> Self {
405 self.sandbox_policy = policy;
406 self
407 }
408
409 pub fn input_schema(mut self, schema: Value) -> Self {
410 self.input_schema = Some(schema);
411 self
412 }
413
414 pub fn output_schema(mut self, schema: Value) -> Self {
415 self.output_schema = Some(schema);
416 self
417 }
418}
419
420#[derive(Default)]
421pub struct MemoryStore {
422 runs: HashMap<String, Run>,
423 steps: HashMap<String, Step>,
424 events: HashMap<String, Vec<Event>>,
425 tool_ledger: HashMap<String, ToolLedgerEntry>,
426 approval_requests: HashMap<String, ApprovalRequest>,
427 cost_records: HashMap<String, Vec<CostRecord>>,
428 artifacts: HashMap<String, Vec<Artifact>>,
429}
430
431impl MemoryStore {
432 pub fn new() -> Self {
433 Self::default()
434 }
435
436 pub fn save_to_path(&self, path: impl AsRef<Path>) -> Result<()> {
437 fs::write(path, encode_store(self)).map_err(|err| RuntimeError(err.to_string()))
438 }
439
440 pub fn load_from_path(path: impl AsRef<Path>) -> Result<Self> {
441 let body = fs::read_to_string(path).map_err(|err| RuntimeError(err.to_string()))?;
442 decode_store(&body)
443 }
444
445 pub fn create_run(&mut self, initial_state: State) -> (String, String) {
446 let run_id = new_id("run");
447 let session_id = new_id("sess");
448 let step_id = new_id("step");
449 let now = now_seconds();
450 self.runs.insert(
451 run_id.clone(),
452 Run {
453 run_id: run_id.clone(),
454 session_id: session_id.clone(),
455 status: "pending".to_string(),
456 state: initial_state.clone(),
457 state_version: 0,
458 created_at: now,
459 updated_at: now,
460 },
461 );
462 self.steps.insert(
463 step_id.clone(),
464 Step {
465 step_id: step_id.clone(),
466 run_id: run_id.clone(),
467 session_id: session_id.clone(),
468 status: "pending".to_string(),
469 owner: None,
470 lease_token: None,
471 lease_until: None,
472 attempt: 0,
473 state_version: 0,
474 checkpoint_id: None,
475 last_error_type: None,
476 last_error: None,
477 cancelled_at: None,
478 created_at: now,
479 updated_at: now,
480 },
481 );
482 let mut payload = State::new();
483 payload.insert("initial_state".to_string(), Value::Object(initial_state));
484 self.append_event(
485 &run_id,
486 Some(&session_id),
487 None,
488 "run_created",
489 payload,
490 None,
491 None,
492 None,
493 );
494 let mut step_payload = State::new();
495 step_payload.insert("step_id".to_string(), Value::String(step_id.clone()));
496 self.append_event(
497 &run_id,
498 Some(&session_id),
499 Some(&step_id),
500 "step_created",
501 step_payload,
502 None,
503 None,
504 None,
505 );
506 (run_id, step_id)
507 }
508
509 pub fn claim_step(
510 &mut self,
511 worker_id: &str,
512 run_id: &str,
513 lease_seconds: f64,
514 ) -> Result<StepClaim> {
515 let mut candidates: Vec<Step> = self
516 .steps
517 .values()
518 .filter(|step| {
519 (run_id.is_empty() || step.run_id == run_id)
520 && (step.status == "pending" || step.status == "retry_scheduled")
521 })
522 .cloned()
523 .collect();
524 candidates.sort_by(|a, b| {
525 a.created_at
526 .partial_cmp(&b.created_at)
527 .unwrap()
528 .then(a.step_id.cmp(&b.step_id))
529 });
530 let candidate = candidates
531 .first()
532 .ok_or_else(|| RuntimeError("agentledger: no runnable step".to_string()))?;
533 let now = now_seconds();
534 let step = self.steps.get_mut(&candidate.step_id).unwrap();
535 step.status = "running".to_string();
536 step.owner = Some(worker_id.to_string());
537 step.lease_token = Some(new_id("lease"));
538 step.lease_until = Some(now + lease_seconds);
539 step.attempt += 1;
540 step.updated_at = now;
541 let claim = StepClaim {
542 run_id: step.run_id.clone(),
543 session_id: step.session_id.clone(),
544 step_id: step.step_id.clone(),
545 attempt: step.attempt,
546 lease_token: step.lease_token.clone().unwrap(),
547 state_version: step.state_version,
548 lease_until: step.lease_until.unwrap(),
549 };
550 let run = self.runs.get_mut(&claim.run_id).unwrap();
551 run.status = "running".to_string();
552 run.updated_at = now;
553 let mut payload = State::new();
554 payload.insert(
555 "worker_id".to_string(),
556 Value::String(worker_id.to_string()),
557 );
558 payload.insert(
559 "lease_token".to_string(),
560 Value::String(claim.lease_token.clone()),
561 );
562 payload.insert("attempt".to_string(), Value::Number(claim.attempt as f64));
563 self.append_event(
564 &claim.run_id,
565 Some(&claim.session_id),
566 Some(&claim.step_id),
567 "step_claimed",
568 payload,
569 None,
570 None,
571 None,
572 );
573 Ok(claim)
574 }
575
576 pub fn load_state(&self, run_id: &str) -> Result<(State, u64, String)> {
577 let run = self
578 .runs
579 .get(run_id)
580 .ok_or_else(|| RuntimeError(format!("run not found: {run_id}")))?;
581 Ok((run.state.clone(), run.state_version, run.session_id.clone()))
582 }
583
584 pub fn recover_expired_leases(&mut self) -> usize {
585 let now = now_seconds();
586 let ids: Vec<String> = self.steps.keys().cloned().collect();
587 let mut recovered = 0;
588 for step_id in ids {
589 let should_recover = self
590 .steps
591 .get(&step_id)
592 .map(|step| {
593 step.status == "running" && step.lease_until.is_some_and(|until| until <= now)
594 })
595 .unwrap_or(false);
596 if !should_recover {
597 continue;
598 }
599 let (run_id, session_id, previous_owner, attempt) = {
600 let step = self.steps.get_mut(&step_id).unwrap();
601 let previous_owner = step.owner.clone().unwrap_or_default();
602 step.status = "retry_scheduled".to_string();
603 step.owner = None;
604 step.lease_token = None;
605 step.lease_until = None;
606 step.updated_at = now;
607 (
608 step.run_id.clone(),
609 step.session_id.clone(),
610 previous_owner,
611 step.attempt,
612 )
613 };
614 let run = self.runs.get_mut(&run_id).unwrap();
615 run.status = "retry_scheduled".to_string();
616 run.updated_at = now;
617 recovered += 1;
618 let mut payload = State::new();
619 payload.insert("previous_owner".to_string(), Value::String(previous_owner));
620 payload.insert("attempt".to_string(), Value::Number(attempt as f64));
621 self.append_event(
622 &run_id,
623 Some(&session_id),
624 Some(&step_id),
625 "lease_expired",
626 payload,
627 None,
628 None,
629 None,
630 );
631 let mut retry_payload = State::new();
632 retry_payload.insert("step_id".to_string(), Value::String(step_id.clone()));
633 retry_payload.insert(
634 "reason".to_string(),
635 Value::String("lease_expired".to_string()),
636 );
637 self.append_event(
638 &run_id,
639 Some(&session_id),
640 Some(&step_id),
641 "step_retry_scheduled",
642 retry_payload,
643 None,
644 None,
645 None,
646 );
647 }
648 recovered
649 }
650
651 pub fn cancel_run(&mut self, run_id: &str, reason: &str) -> Result<usize> {
652 let session_id = self
653 .runs
654 .get(run_id)
655 .ok_or_else(|| RuntimeError(format!("run not found: {run_id}")))?
656 .session_id
657 .clone();
658 let mut payload = State::new();
659 payload.insert("reason".to_string(), Value::String(reason.to_string()));
660 self.append_event(
661 run_id,
662 Some(&session_id),
663 None,
664 "run_cancel_requested",
665 payload,
666 None,
667 None,
668 None,
669 );
670 let ids: Vec<String> = self.steps.keys().cloned().collect();
671 let now = now_seconds();
672 let mut cancelled = 0;
673 for step_id in ids {
674 let should_cancel = self
675 .steps
676 .get(&step_id)
677 .map(|step| {
678 step.run_id == run_id
679 && !["completed", "failed", "cancelled"].contains(&step.status.as_str())
680 })
681 .unwrap_or(false);
682 if !should_cancel {
683 continue;
684 }
685 let step_session = {
686 let step = self.steps.get_mut(&step_id).unwrap();
687 step.status = "cancelled".to_string();
688 step.owner = None;
689 step.lease_token = None;
690 step.lease_until = None;
691 step.cancelled_at = Some(now);
692 step.updated_at = now;
693 step.session_id.clone()
694 };
695 cancelled += 1;
696 let mut step_payload = State::new();
697 step_payload.insert("reason".to_string(), Value::String(reason.to_string()));
698 self.append_event(
699 run_id,
700 Some(&step_session),
701 Some(&step_id),
702 "step_cancelled",
703 step_payload,
704 None,
705 None,
706 None,
707 );
708 }
709 let run = self.runs.get_mut(run_id).unwrap();
710 run.status = "cancelled".to_string();
711 run.updated_at = now;
712 let mut done_payload = State::new();
713 done_payload.insert("reason".to_string(), Value::String(reason.to_string()));
714 done_payload.insert(
715 "cancelled_steps".to_string(),
716 Value::Number(cancelled as f64),
717 );
718 self.append_event(
719 run_id,
720 Some(&session_id),
721 None,
722 "run_cancelled",
723 done_payload,
724 None,
725 None,
726 None,
727 );
728 Ok(cancelled)
729 }
730
731 pub fn commit_state_patch(
732 &mut self,
733 run_id: &str,
734 step_id: &str,
735 lease_token: &str,
736 base_version: u64,
737 patch: State,
738 ) -> Result<u64> {
739 self.validate_lease(step_id, lease_token)?;
740 let run = self
741 .runs
742 .get(run_id)
743 .ok_or_else(|| RuntimeError(format!("run not found: {run_id}")))?;
744 if run.state_version != base_version {
745 return Err(RuntimeError(format!(
746 "state version conflict: expected {base_version}, got {}",
747 run.state_version
748 )));
749 }
750 let now = now_seconds();
751 let new_version = base_version + 1;
752 let session_id = self.steps.get(step_id).unwrap().session_id.clone();
753 let run = self.runs.get_mut(run_id).unwrap();
754 run.state = merge_patch(&run.state, &patch);
755 run.state_version = new_version;
756 run.status = "completed".to_string();
757 run.updated_at = now;
758 let step = self.steps.get_mut(step_id).unwrap();
759 step.status = "completed".to_string();
760 step.state_version = new_version;
761 step.checkpoint_id = Some(format!("ckpt:{run_id}:{step_id}:{}", step.attempt));
762 step.updated_at = now;
763 let mut payload = State::new();
764 payload.insert("patch".to_string(), Value::Object(patch));
765 payload.insert(
766 "state_version".to_string(),
767 Value::Number(new_version as f64),
768 );
769 self.append_event(
770 run_id,
771 Some(&session_id),
772 Some(step_id),
773 "state_patch_committed",
774 payload,
775 None,
776 Some(new_version),
777 None,
778 );
779 let mut complete_payload = State::new();
780 complete_payload.insert("step_id".to_string(), Value::String(step_id.to_string()));
781 self.append_event(
782 run_id,
783 Some(&session_id),
784 Some(step_id),
785 "step_completed",
786 complete_payload,
787 None,
788 Some(new_version),
789 None,
790 );
791 Ok(new_version)
792 }
793
794 pub fn mark_retry(&mut self, run_id: &str, step_id: &str, error_type: &str, message: &str) {
795 let now = now_seconds();
796 let session_id = {
797 let step = self.steps.get_mut(step_id).unwrap();
798 step.status = "retry_scheduled".to_string();
799 step.owner = None;
800 step.lease_token = None;
801 step.lease_until = None;
802 step.last_error_type = Some(error_type.to_string());
803 step.last_error = Some(message.to_string());
804 step.updated_at = now;
805 step.session_id.clone()
806 };
807 let run = self.runs.get_mut(run_id).unwrap();
808 run.status = "retry_scheduled".to_string();
809 run.updated_at = now;
810 let mut classified = State::new();
811 classified.insert("error".to_string(), Value::String(message.to_string()));
812 classified.insert(
813 "error_type".to_string(),
814 Value::String(error_type.to_string()),
815 );
816 classified.insert("retryable".to_string(), Value::Bool(true));
817 classified.insert("source".to_string(), Value::String("agent".to_string()));
818 self.append_event(
819 run_id,
820 Some(&session_id),
821 Some(step_id),
822 "failure_classified",
823 classified,
824 None,
825 None,
826 None,
827 );
828 let mut payload = State::new();
829 payload.insert("error".to_string(), Value::String(message.to_string()));
830 payload.insert(
831 "error_type".to_string(),
832 Value::String(error_type.to_string()),
833 );
834 self.append_event(
835 run_id,
836 Some(&session_id),
837 Some(step_id),
838 "error_raised",
839 payload,
840 None,
841 None,
842 None,
843 );
844 let mut retry_payload = State::new();
845 retry_payload.insert("step_id".to_string(), Value::String(step_id.to_string()));
846 self.append_event(
847 run_id,
848 Some(&session_id),
849 Some(step_id),
850 "step_retry_scheduled",
851 retry_payload,
852 None,
853 None,
854 None,
855 );
856 }
857
858 pub fn mark_waiting_human(
859 &mut self,
860 run_id: &str,
861 step_id: &str,
862 reason: &str,
863 approval_id: &str,
864 ) {
865 let now = now_seconds();
866 let session_id = {
867 let step = self.steps.get_mut(step_id).unwrap();
868 step.status = "waiting_human".to_string();
869 step.owner = None;
870 step.lease_token = None;
871 step.lease_until = None;
872 step.last_error_type = Some("ApprovalRequired".to_string());
873 step.last_error = Some(reason.to_string());
874 step.updated_at = now;
875 step.session_id.clone()
876 };
877 let run = self.runs.get_mut(run_id).unwrap();
878 run.status = "waiting_human".to_string();
879 run.updated_at = now;
880 let mut payload = State::new();
881 payload.insert("reason".to_string(), Value::String(reason.to_string()));
882 payload.insert(
883 "approval_id".to_string(),
884 Value::String(approval_id.to_string()),
885 );
886 self.append_event(
887 run_id,
888 Some(&session_id),
889 Some(step_id),
890 "step_waiting_human",
891 payload,
892 None,
893 None,
894 None,
895 );
896 }
897
898 pub fn mark_failed(&mut self, run_id: &str, step_id: &str, error_type: &str, message: &str) {
899 let now = now_seconds();
900 let session_id = {
901 let step = self.steps.get_mut(step_id).unwrap();
902 step.status = "failed".to_string();
903 step.owner = None;
904 step.lease_token = None;
905 step.lease_until = None;
906 step.last_error_type = Some(error_type.to_string());
907 step.last_error = Some(message.to_string());
908 step.updated_at = now;
909 step.session_id.clone()
910 };
911 let run = self.runs.get_mut(run_id).unwrap();
912 run.status = "failed".to_string();
913 run.updated_at = now;
914 let mut classified = State::new();
915 classified.insert("error".to_string(), Value::String(message.to_string()));
916 classified.insert(
917 "error_type".to_string(),
918 Value::String(error_type.to_string()),
919 );
920 classified.insert("retryable".to_string(), Value::Bool(false));
921 classified.insert(
922 "source".to_string(),
923 Value::String(failure_source(error_type).to_string()),
924 );
925 self.append_event(
926 run_id,
927 Some(&session_id),
928 Some(step_id),
929 "failure_classified",
930 classified,
931 None,
932 None,
933 None,
934 );
935 let mut error = State::new();
936 error.insert("error".to_string(), Value::String(message.to_string()));
937 error.insert(
938 "error_type".to_string(),
939 Value::String(error_type.to_string()),
940 );
941 self.append_event(
942 run_id,
943 Some(&session_id),
944 Some(step_id),
945 "error_raised",
946 error,
947 None,
948 None,
949 None,
950 );
951 let mut payload = State::new();
952 payload.insert("step_id".to_string(), Value::String(step_id.to_string()));
953 payload.insert(
954 "error_type".to_string(),
955 Value::String(error_type.to_string()),
956 );
957 self.append_event(
958 run_id,
959 Some(&session_id),
960 Some(step_id),
961 "step_failed",
962 payload,
963 None,
964 None,
965 None,
966 );
967 }
968
969 pub fn append_event(
970 &mut self,
971 run_id: &str,
972 session_id: Option<&str>,
973 step_id: Option<&str>,
974 event_type: &str,
975 payload: State,
976 agent_role: Option<&str>,
977 state_version: Option<u64>,
978 causal_token: Option<&str>,
979 ) -> Event {
980 let events = self.events.entry(run_id.to_string()).or_default();
981 let payload_ref = format_state(&payload);
982 let event = Event {
983 event_id: new_id("evt"),
984 run_id: run_id.to_string(),
985 session_id: session_id.map(str::to_string),
986 step_id: step_id.map(str::to_string),
987 seq: events.len() as u64 + 1,
988 event_type: event_type.to_string(),
989 timestamp: now_seconds(),
990 agent_role: agent_role.map(str::to_string),
991 state_version,
992 causal_token: causal_token.map(str::to_string),
993 payload_hash: stable_hash(&payload_ref),
994 payload_ref,
995 payload,
996 };
997 events.push(event.clone());
998 event
999 }
1000
1001 pub fn reserve_ledger(&mut self, key: &str, entry: ToolLedgerEntry) -> Option<ToolLedgerEntry> {
1002 if let Some(existing) = self.tool_ledger.get(key) {
1003 return Some(existing.clone());
1004 }
1005 self.tool_ledger.insert(key.to_string(), entry);
1006 None
1007 }
1008
1009 pub fn update_ledger(
1010 &mut self,
1011 key: &str,
1012 status: &str,
1013 response: Option<Value>,
1014 error_type: Option<String>,
1015 ) {
1016 if let Some(entry) = self.tool_ledger.get_mut(key) {
1017 entry.status = status.to_string();
1018 entry.response = response.clone();
1019 entry.response_ref = response.as_ref().map(format_value);
1020 entry.response_hash = entry.response_ref.as_ref().map(|value| stable_hash(value));
1021 entry.error_type = error_type;
1022 entry.updated_at = now_seconds();
1023 }
1024 }
1025
1026 pub fn request_approval(
1027 &mut self,
1028 approval_key: &str,
1029 run_id: &str,
1030 session_id: &str,
1031 step_id: &str,
1032 tool_name: &str,
1033 risk_level: &str,
1034 reason: &str,
1035 request_hash: &str,
1036 request_ref: &str,
1037 requested_by: &str,
1038 ) -> ApprovalRequest {
1039 if let Some(existing) = self.approval_requests.get(approval_key) {
1040 return existing.clone();
1041 }
1042 let now = now_seconds();
1043 let approval = ApprovalRequest {
1044 approval_id: new_id("approval"),
1045 approval_key: approval_key.to_string(),
1046 run_id: run_id.to_string(),
1047 session_id: session_id.to_string(),
1048 step_id: step_id.to_string(),
1049 tool_name: tool_name.to_string(),
1050 risk_level: risk_level.to_string(),
1051 status: "PENDING".to_string(),
1052 reason: reason.to_string(),
1053 request_hash: request_hash.to_string(),
1054 request_ref: request_ref.to_string(),
1055 requested_by: requested_by.to_string(),
1056 approved_by: None,
1057 decision_reason: None,
1058 created_at: now,
1059 updated_at: now,
1060 };
1061 self.approval_requests
1062 .insert(approval_key.to_string(), approval.clone());
1063 approval
1064 }
1065
1066 pub fn approval_for_key(&self, approval_key: &str) -> Option<ApprovalRequest> {
1067 self.approval_requests.get(approval_key).cloned()
1068 }
1069
1070 pub fn approval_requests(&self, run_id: &str) -> Vec<ApprovalRequest> {
1071 self.approval_requests
1072 .values()
1073 .filter(|item| run_id.is_empty() || item.run_id == run_id)
1074 .cloned()
1075 .collect()
1076 }
1077
1078 pub fn approve_request(
1079 &mut self,
1080 approval_id: &str,
1081 approver: &str,
1082 reason: &str,
1083 ) -> Result<ApprovalRequest> {
1084 self.decide_approval(approval_id, "APPROVED", approver, reason)
1085 }
1086
1087 pub fn deny_request(
1088 &mut self,
1089 approval_id: &str,
1090 approver: &str,
1091 reason: &str,
1092 ) -> Result<ApprovalRequest> {
1093 self.decide_approval(approval_id, "DENIED", approver, reason)
1094 }
1095
1096 fn decide_approval(
1097 &mut self,
1098 approval_id: &str,
1099 status: &str,
1100 approver: &str,
1101 reason: &str,
1102 ) -> Result<ApprovalRequest> {
1103 let key = self
1104 .approval_requests
1105 .iter()
1106 .find(|(_, item)| item.approval_id == approval_id)
1107 .map(|(key, _)| key.clone())
1108 .ok_or_else(|| RuntimeError(format!("approval not found: {approval_id}")))?;
1109 let mut approval = self.approval_requests.get(&key).unwrap().clone();
1110 approval.status = status.to_string();
1111 approval.approved_by = Some(approver.to_string());
1112 approval.decision_reason = Some(reason.to_string());
1113 approval.updated_at = now_seconds();
1114 self.approval_requests.insert(key, approval.clone());
1115 let mut payload = State::new();
1116 payload.insert(
1117 "approval_id".to_string(),
1118 Value::String(approval_id.to_string()),
1119 );
1120 payload.insert(
1121 "tool".to_string(),
1122 Value::String(approval.tool_name.clone()),
1123 );
1124 payload.insert("status".to_string(), Value::String(status.to_string()));
1125 self.append_event(
1126 &approval.run_id,
1127 Some(&approval.session_id),
1128 Some(&approval.step_id),
1129 "tool_approval_decided",
1130 payload,
1131 None,
1132 None,
1133 None,
1134 );
1135 if self
1136 .steps
1137 .get(&approval.step_id)
1138 .map(|step| step.status.as_str())
1139 == Some("waiting_human")
1140 {
1141 if status == "APPROVED" {
1142 let step = self.steps.get_mut(&approval.step_id).unwrap();
1143 step.status = "pending".to_string();
1144 step.owner = None;
1145 step.lease_token = None;
1146 step.lease_until = None;
1147 step.updated_at = now_seconds();
1148 let run = self.runs.get_mut(&approval.run_id).unwrap();
1149 run.status = "pending".to_string();
1150 let mut retry = State::new();
1151 retry.insert(
1152 "step_id".to_string(),
1153 Value::String(approval.step_id.clone()),
1154 );
1155 retry.insert(
1156 "reason".to_string(),
1157 Value::String("approval_granted".to_string()),
1158 );
1159 self.append_event(
1160 &approval.run_id,
1161 Some(&approval.session_id),
1162 Some(&approval.step_id),
1163 "step_retry_scheduled",
1164 retry,
1165 None,
1166 None,
1167 None,
1168 );
1169 } else if status == "DENIED" {
1170 self.mark_failed(
1171 &approval.run_id,
1172 &approval.step_id,
1173 "ApprovalDenied",
1174 reason,
1175 );
1176 }
1177 }
1178 Ok(approval)
1179 }
1180
1181 pub fn record_cost(
1182 &mut self,
1183 run_id: &str,
1184 session_id: &str,
1185 step_id: &str,
1186 category: &str,
1187 name: &str,
1188 amount: f64,
1189 unit: &str,
1190 metadata: State,
1191 ) -> String {
1192 let cost_id = new_id("cost");
1193 let record = CostRecord {
1194 cost_id: cost_id.clone(),
1195 run_id: run_id.to_string(),
1196 session_id: session_id.to_string(),
1197 step_id: step_id.to_string(),
1198 category: category.to_string(),
1199 name: name.to_string(),
1200 amount,
1201 unit: unit.to_string(),
1202 metadata,
1203 created_at: now_seconds(),
1204 };
1205 self.cost_records
1206 .entry(run_id.to_string())
1207 .or_default()
1208 .push(record);
1209 let mut payload = State::new();
1210 payload.insert("cost_id".to_string(), Value::String(cost_id.clone()));
1211 payload.insert("category".to_string(), Value::String(category.to_string()));
1212 payload.insert("name".to_string(), Value::String(name.to_string()));
1213 payload.insert("amount".to_string(), Value::Number(amount));
1214 payload.insert("unit".to_string(), Value::String(unit.to_string()));
1215 self.append_event(
1216 run_id,
1217 Some(session_id),
1218 Some(step_id),
1219 "cost_recorded",
1220 payload,
1221 None,
1222 None,
1223 None,
1224 );
1225 cost_id
1226 }
1227
1228 pub fn cost_records(&self, run_id: &str) -> Vec<CostRecord> {
1229 self.cost_records.get(run_id).cloned().unwrap_or_default()
1230 }
1231
1232 pub fn cost_summary(&self, run_id: &str) -> CostSummary {
1233 let mut summary = CostSummary::default();
1234 for record in self.cost_records(run_id) {
1235 add_cost(&mut summary, &record);
1236 }
1237 summary
1238 }
1239
1240 pub fn create_artifact(
1241 &mut self,
1242 run_id: &str,
1243 step_id: Option<&str>,
1244 name: &str,
1245 content: State,
1246 metadata: State,
1247 ) -> Artifact {
1248 let blob_ref = format_state(&content);
1249 let artifact = Artifact {
1250 artifact_id: new_id("art"),
1251 run_id: run_id.to_string(),
1252 step_id: step_id.map(str::to_string),
1253 name: name.to_string(),
1254 blob_hash: format!("sha256:{}", stable_hash(&blob_ref)),
1255 blob_ref,
1256 metadata,
1257 created_at: now_seconds(),
1258 };
1259 self.artifacts
1260 .entry(run_id.to_string())
1261 .or_default()
1262 .push(artifact.clone());
1263 artifact
1264 }
1265
1266 pub fn artifacts(&self, run_id: &str) -> Vec<Artifact> {
1267 let mut items = self.artifacts.get(run_id).cloned().unwrap_or_default();
1268 items.sort_by(|a, b| {
1269 a.created_at
1270 .partial_cmp(&b.created_at)
1271 .unwrap()
1272 .then(a.artifact_id.cmp(&b.artifact_id))
1273 });
1274 items
1275 }
1276
1277 pub fn validate_lease(&self, step_id: &str, lease_token: &str) -> Result<()> {
1278 let step = self
1279 .steps
1280 .get(step_id)
1281 .ok_or_else(|| RuntimeError(format!("step not found: {step_id}")))?;
1282 if step.status != "running" || step.lease_token.as_deref() != Some(lease_token) {
1283 return Err(RuntimeError("invalid or stale lease token".to_string()));
1284 }
1285 if step.lease_until.is_some_and(|until| until <= now_seconds()) {
1286 return Err(RuntimeError("lease expired".to_string()));
1287 }
1288 Ok(())
1289 }
1290
1291 pub fn final_state(&self, run_id: &str) -> Result<State> {
1292 Ok(self.load_state(run_id)?.0)
1293 }
1294
1295 pub fn run(&self, run_id: &str) -> Result<Run> {
1296 self.runs
1297 .get(run_id)
1298 .cloned()
1299 .ok_or_else(|| RuntimeError(format!("run not found: {run_id}")))
1300 }
1301
1302 pub fn steps(&self, run_id: &str) -> Vec<Step> {
1303 self.steps
1304 .values()
1305 .filter(|step| step.run_id == run_id)
1306 .cloned()
1307 .collect()
1308 }
1309
1310 pub fn events(&self, run_id: &str) -> Vec<Event> {
1311 self.events.get(run_id).cloned().unwrap_or_default()
1312 }
1313
1314 pub fn ledger(&self, run_id: &str) -> Vec<ToolLedgerEntry> {
1315 self.tool_ledger
1316 .values()
1317 .filter(|entry| entry.run_id == run_id)
1318 .cloned()
1319 .collect()
1320 }
1321}
1322
1323pub struct ToolRegistry {
1324 tools: HashMap<String, ToolSpec>,
1325}
1326
1327impl ToolRegistry {
1328 pub fn new() -> Self {
1329 Self {
1330 tools: HashMap::new(),
1331 }
1332 }
1333
1334 pub fn register(&mut self, spec: ToolSpec) {
1335 self.tools.insert(spec.name.clone(), spec);
1336 }
1337
1338 pub fn get(&self, name: &str) -> Result<&ToolSpec> {
1339 self.tools
1340 .get(name)
1341 .ok_or_else(|| RuntimeError(format!("tool not registered: {name}")))
1342 }
1343}
1344
1345impl Default for ToolRegistry {
1346 fn default() -> Self {
1347 Self::new()
1348 }
1349}
1350
1351pub struct Runtime {
1352 pub store: MemoryStore,
1353 pub registry: ToolRegistry,
1354 pub budget: BudgetLimits,
1355 pub sandbox: Box<dyn SandboxExecutor>,
1356}
1357
1358impl Runtime {
1359 pub fn new() -> Self {
1360 Self {
1361 store: MemoryStore::new(),
1362 registry: ToolRegistry::new(),
1363 budget: BudgetLimits::default(),
1364 sandbox: Box::new(DisabledSandboxExecutor),
1365 }
1366 }
1367
1368 pub fn register_tool(&mut self, spec: ToolSpec) {
1369 self.registry.register(spec);
1370 }
1371
1372 pub fn set_budget(&mut self, budget: BudgetLimits) {
1373 self.budget = budget;
1374 }
1375
1376 pub fn set_sandbox(&mut self, sandbox: Box<dyn SandboxExecutor>) {
1377 self.sandbox = sandbox;
1378 }
1379
1380 pub fn create_run(&mut self, initial_state: State) -> (String, String) {
1381 self.store.create_run(initial_state)
1382 }
1383
1384 pub fn run_once(
1385 &mut self,
1386 run_id: &str,
1387 worker_id: &str,
1388 agent_role: &str,
1389 lease_seconds: f64,
1390 agent: AgentFunc,
1391 ) -> Result<bool> {
1392 let claim = match self.store.claim_step(worker_id, run_id, lease_seconds) {
1393 Ok(claim) => claim,
1394 Err(err) if err.0.contains("no runnable step") => return Ok(false),
1395 Err(err) => return Err(err),
1396 };
1397 let (state, version, session_id) = self.store.load_state(&claim.run_id)?;
1398 let mut payload = State::new();
1399 payload.insert(
1400 "agent_role".to_string(),
1401 Value::String(agent_role.to_string()),
1402 );
1403 payload.insert("attempt".to_string(), Value::Number(claim.attempt as f64));
1404 self.store.append_event(
1405 &claim.run_id,
1406 Some(&session_id),
1407 Some(&claim.step_id),
1408 "agent_started",
1409 payload,
1410 Some(agent_role),
1411 Some(version),
1412 None,
1413 );
1414 let mut ctx = AgentContext {
1415 run_id: claim.run_id.clone(),
1416 session_id,
1417 step_id: claim.step_id.clone(),
1418 agent_role: agent_role.to_string(),
1419 lease_token: claim.lease_token.clone(),
1420 attempt: claim.attempt,
1421 state_version: version,
1422 pending_patch: State::new(),
1423 };
1424 match agent(&mut ctx, state) {
1425 Ok(()) => {
1426 self.store.commit_state_patch(
1427 &claim.run_id,
1428 &claim.step_id,
1429 &claim.lease_token,
1430 version,
1431 ctx.pending_patch,
1432 )?;
1433 Ok(true)
1434 }
1435 Err(err) if err.0 == "retryable" => {
1436 self.store.mark_retry(
1437 &claim.run_id,
1438 &claim.step_id,
1439 "RetryableAgentError",
1440 "retryable",
1441 );
1442 Ok(false)
1443 }
1444 Err(err) if err.0.starts_with("approval required:") => {
1445 let approval_id = err.0.trim_start_matches("approval required:");
1446 self.store
1447 .mark_waiting_human(&claim.run_id, &claim.step_id, &err.0, approval_id);
1448 Ok(false)
1449 }
1450 Err(err) => {
1451 self.store.mark_failed(
1452 &claim.run_id,
1453 &claim.step_id,
1454 classify_runtime_error(&err.0),
1455 &err.0,
1456 );
1457 Err(err)
1458 }
1459 }
1460 }
1461
1462 pub fn call_tool(&mut self, ctx: &AgentContext, tool_name: &str, args: State) -> Result<Value> {
1463 let (
1464 version,
1465 side_effect,
1466 risk_level,
1467 idempotency_required,
1468 approval_required,
1469 sandbox_required,
1470 sandbox_executor,
1471 sandbox_policy,
1472 input_schema,
1473 output_schema,
1474 ) = {
1475 let spec = self.registry.get(tool_name)?;
1476 (
1477 spec.version.clone(),
1478 spec.side_effect.clone(),
1479 spec.risk_level.clone(),
1480 spec.idempotency_required,
1481 spec.approval_required,
1482 spec.sandbox_required,
1483 spec.sandbox_executor.clone(),
1484 spec.sandbox_policy.clone(),
1485 spec.input_schema.clone(),
1486 spec.output_schema.clone(),
1487 )
1488 };
1489 let mut request = State::new();
1490 request.insert("tool".to_string(), Value::String(tool_name.to_string()));
1491 request.insert("args".to_string(), Value::Object(args.clone()));
1492 let request_ref = format_state(&request);
1493 let request_hash = stable_hash(&request_ref);
1494 let causal_token = format!(
1495 "{}:{}:{}:{}",
1496 ctx.run_id, ctx.step_id, ctx.attempt, ctx.state_version
1497 );
1498 let idempotency_key = format!(
1499 "{}:{}:{}:{}",
1500 ctx.run_id, ctx.step_id, tool_name, request_hash
1501 );
1502 let managed = side_effect != "none" || idempotency_required;
1503 self.store.append_event(
1504 &ctx.run_id,
1505 Some(&ctx.session_id),
1506 Some(&ctx.step_id),
1507 "tool_call_requested",
1508 request,
1509 Some(&ctx.agent_role),
1510 Some(ctx.state_version),
1511 Some(&causal_token),
1512 );
1513 if let Some(schema) = input_schema.as_ref() {
1514 if let Err(err) = validate_tool_schema(schema, &Value::Object(args.clone()), "$arg") {
1515 let mut failed = State::new();
1516 failed.insert("tool".to_string(), Value::String(tool_name.to_string()));
1517 failed.insert("error".to_string(), Value::String(err.0.clone()));
1518 failed.insert(
1519 "phase".to_string(),
1520 Value::String("input_validation".to_string()),
1521 );
1522 self.store.append_event(
1523 &ctx.run_id,
1524 Some(&ctx.session_id),
1525 Some(&ctx.step_id),
1526 "tool_call_failed",
1527 failed,
1528 Some(&ctx.agent_role),
1529 Some(ctx.state_version),
1530 Some(&causal_token),
1531 );
1532 return Err(err);
1533 }
1534 }
1535
1536 let mut allowed = !is_high_risk(&risk_level);
1537 let mut reason = if allowed {
1538 "default allow for low/medium risk in local runtime".to_string()
1539 } else {
1540 "high-risk tool denied by default".to_string()
1541 };
1542 if let Some(approval) = self.store.approval_for_key(&idempotency_key) {
1543 if approval.status == "DENIED" {
1544 reason = format!("approval denied for tool {tool_name}");
1545 self.record_permission(ctx, tool_name, false, &reason, &causal_token);
1546 return Err(RuntimeError(reason));
1547 }
1548 if approval.status == "APPROVED" {
1549 allowed = true;
1550 reason = format!(
1551 "approved by {}",
1552 approval
1553 .approved_by
1554 .unwrap_or_else(|| "operator".to_string())
1555 );
1556 }
1557 } else if approval_required {
1558 let approval = self.store.request_approval(
1559 &idempotency_key,
1560 &ctx.run_id,
1561 &ctx.session_id,
1562 &ctx.step_id,
1563 tool_name,
1564 &risk_level,
1565 "tool requires approval",
1566 &request_hash,
1567 &request_ref,
1568 &ctx.agent_role,
1569 );
1570 self.record_permission(ctx, tool_name, false, "approval required", &causal_token);
1571 let mut payload = State::new();
1572 payload.insert("tool".to_string(), Value::String(tool_name.to_string()));
1573 payload.insert(
1574 "approval_id".to_string(),
1575 Value::String(approval.approval_id.clone()),
1576 );
1577 payload.insert(
1578 "approval_key".to_string(),
1579 Value::String(idempotency_key.clone()),
1580 );
1581 payload.insert("risk_level".to_string(), Value::String(risk_level));
1582 self.store.append_event(
1583 &ctx.run_id,
1584 Some(&ctx.session_id),
1585 Some(&ctx.step_id),
1586 "tool_approval_required",
1587 payload,
1588 Some(&ctx.agent_role),
1589 Some(ctx.state_version),
1590 Some(&causal_token),
1591 );
1592 return Err(RuntimeError(format!(
1593 "approval required:{}",
1594 approval.approval_id
1595 )));
1596 }
1597 self.record_permission(ctx, tool_name, allowed, &reason, &causal_token);
1598 if !allowed {
1599 return Err(RuntimeError(reason));
1600 }
1601 if let Some(max) = self.budget.max_tool_calls {
1602 let used = self.store.cost_summary(&ctx.run_id).tool_calls;
1603 if used >= max {
1604 let message = format!("tool call budget exceeded: {used}/{max}");
1605 let mut payload = State::new();
1606 payload.insert("category".to_string(), Value::String("tool".to_string()));
1607 payload.insert("tool".to_string(), Value::String(tool_name.to_string()));
1608 payload.insert("error".to_string(), Value::String(message.clone()));
1609 self.store.append_event(
1610 &ctx.run_id,
1611 Some(&ctx.session_id),
1612 Some(&ctx.step_id),
1613 "budget_check_failed",
1614 payload,
1615 Some(&ctx.agent_role),
1616 Some(ctx.state_version),
1617 Some(&causal_token),
1618 );
1619 return Err(RuntimeError(message));
1620 }
1621 }
1622 if managed {
1623 let entry = ToolLedgerEntry {
1624 ledger_id: new_id("ledger"),
1625 run_id: ctx.run_id.clone(),
1626 session_id: ctx.session_id.clone(),
1627 step_id: ctx.step_id.clone(),
1628 tool_name: tool_name.to_string(),
1629 tool_version: version,
1630 tool_call_id: new_id("toolcall"),
1631 idempotency_key: idempotency_key.clone(),
1632 causal_token: causal_token.clone(),
1633 request_hash: request_hash.clone(),
1634 request_ref: request_ref.clone(),
1635 status: "RESERVED".to_string(),
1636 external_id: None,
1637 response_hash: None,
1638 response_ref: None,
1639 error_type: None,
1640 response: None,
1641 created_at: now_seconds(),
1642 updated_at: now_seconds(),
1643 };
1644 if let Some(existing) = self.store.reserve_ledger(&idempotency_key, entry) {
1645 if existing.status == "SUCCEEDED" {
1646 let mut payload = State::new();
1647 payload.insert("tool".to_string(), Value::String(tool_name.to_string()));
1648 payload.insert("replayed_from_ledger".to_string(), Value::Bool(true));
1649 self.store.append_event(
1650 &ctx.run_id,
1651 Some(&ctx.session_id),
1652 Some(&ctx.step_id),
1653 "tool_call_completed",
1654 payload,
1655 Some(&ctx.agent_role),
1656 Some(ctx.state_version),
1657 Some(&causal_token),
1658 );
1659 self.store.record_cost(
1660 &ctx.run_id,
1661 &ctx.session_id,
1662 &ctx.step_id,
1663 "tool",
1664 tool_name,
1665 1.0,
1666 "call",
1667 State::new(),
1668 );
1669 return existing
1670 .response
1671 .ok_or_else(|| RuntimeError("missing ledger response".to_string()));
1672 }
1673 return Err(RuntimeError(
1674 "tool side effect already in progress".to_string(),
1675 ));
1676 }
1677 self.store
1678 .update_ledger(&idempotency_key, "RUNNING", None, None);
1679 }
1680 if sandbox_required {
1681 let executor = if sandbox_executor.is_empty() { "default".to_string() } else { sandbox_executor.clone() };
1682 let network = match sandbox_policy.get("network") {
1683 Some(Value::String(value)) => value.clone(),
1684 _ => "deny".to_string(),
1685 };
1686 let filesystem = match sandbox_policy.get("filesystem") {
1687 Some(Value::String(value)) => value.clone(),
1688 _ => "read-only".to_string(),
1689 };
1690 let timeout_seconds = match sandbox_policy.get("timeout_seconds") {
1691 Some(Value::Number(value)) if *value > 0.0 => *value as u64,
1692 _ => 30,
1693 };
1694 let policy = SandboxPolicy {
1695 tool_name: tool_name.to_string(),
1696 run_id: ctx.run_id.clone(),
1697 step_id: ctx.step_id.clone(),
1698 executor: executor.clone(),
1699 network,
1700 filesystem,
1701 timeout_seconds,
1702 extra: sandbox_policy.clone(),
1703 };
1704 let mut sandbox_payload = State::new();
1705 sandbox_payload.insert("tool_name".to_string(), Value::String(tool_name.to_string()));
1706 sandbox_payload.insert("executor".to_string(), Value::String(executor));
1707 sandbox_payload.insert("network".to_string(), Value::String(policy.network.clone()));
1708 sandbox_payload.insert("filesystem".to_string(), Value::String(policy.filesystem.clone()));
1709 sandbox_payload.insert("timeout_seconds".to_string(), Value::Number(policy.timeout_seconds as f64));
1710 self.store.append_event(
1711 &ctx.run_id,
1712 Some(&ctx.session_id),
1713 Some(&ctx.step_id),
1714 "sandbox_started",
1715 sandbox_payload,
1716 Some(&ctx.agent_role),
1717 Some(ctx.state_version),
1718 Some(&causal_token),
1719 );
1720 let result = self.sandbox.run_tool(args, &policy);
1721 let mut completed = State::new();
1722 completed.insert("ok".to_string(), Value::Bool(result.ok));
1723 completed.insert("metadata".to_string(), Value::Object(result.metadata.clone()));
1724 if let Some(error) = &result.error {
1725 completed.insert("error".to_string(), Value::String(error.clone()));
1726 }
1727 self.store.append_event(
1728 &ctx.run_id,
1729 Some(&ctx.session_id),
1730 Some(&ctx.step_id),
1731 "sandbox_completed",
1732 completed,
1733 Some(&ctx.agent_role),
1734 Some(ctx.state_version),
1735 Some(&causal_token),
1736 );
1737 if !result.ok {
1738 let message = result.error.unwrap_or_else(|| "sandboxed tool failed".to_string());
1739 let mut failed = State::new();
1740 failed.insert("tool".to_string(), Value::String(tool_name.to_string()));
1741 failed.insert("error".to_string(), Value::String(message.clone()));
1742 self.store.append_event(
1743 &ctx.run_id,
1744 Some(&ctx.session_id),
1745 Some(&ctx.step_id),
1746 "tool_call_failed",
1747 failed,
1748 Some(&ctx.agent_role),
1749 Some(ctx.state_version),
1750 Some(&causal_token),
1751 );
1752 return Err(RuntimeError(message));
1753 }
1754 let value = result.output;
1755 if let Some(schema) = output_schema.as_ref() {
1756 validate_tool_schema(schema, &value, "$result")?;
1757 }
1758 if managed {
1759 self.store.update_ledger(&idempotency_key, "SUCCEEDED", Some(value.clone()), None);
1760 }
1761 let mut payload = State::new();
1762 payload.insert("tool".to_string(), Value::String(tool_name.to_string()));
1763 payload.insert("idempotency_key".to_string(), Value::String(idempotency_key));
1764 self.store.append_event(
1765 &ctx.run_id,
1766 Some(&ctx.session_id),
1767 Some(&ctx.step_id),
1768 "tool_call_completed",
1769 payload,
1770 Some(&ctx.agent_role),
1771 Some(ctx.state_version),
1772 Some(&causal_token),
1773 );
1774 self.store.record_cost(
1775 &ctx.run_id,
1776 &ctx.session_id,
1777 &ctx.step_id,
1778 "tool",
1779 tool_name,
1780 1.0,
1781 "call",
1782 State::new(),
1783 );
1784 return Ok(value);
1785 }
1786 let result = {
1787 let spec = self.registry.get(tool_name)?;
1788 (spec.func)(args)
1789 };
1790 match result {
1791 Ok(value) => {
1792 if let Some(schema) = output_schema.as_ref() {
1793 if let Err(err) = validate_tool_schema(schema, &value, "$result") {
1794 if managed {
1795 self.store.update_ledger(
1796 &idempotency_key,
1797 "PENDING_VERIFICATION",
1798 None,
1799 Some("ToolOutputValidationError".to_string()),
1800 );
1801 }
1802 let mut failed = State::new();
1803 failed.insert("tool".to_string(), Value::String(tool_name.to_string()));
1804 failed.insert("error".to_string(), Value::String(err.0.clone()));
1805 failed.insert(
1806 "phase".to_string(),
1807 Value::String("output_validation".to_string()),
1808 );
1809 self.store.append_event(
1810 &ctx.run_id,
1811 Some(&ctx.session_id),
1812 Some(&ctx.step_id),
1813 "tool_call_failed",
1814 failed,
1815 Some(&ctx.agent_role),
1816 Some(ctx.state_version),
1817 Some(&causal_token),
1818 );
1819 return Err(err);
1820 }
1821 }
1822 if managed {
1823 self.store.update_ledger(
1824 &idempotency_key,
1825 "SUCCEEDED",
1826 Some(value.clone()),
1827 None,
1828 );
1829 }
1830 let mut payload = State::new();
1831 payload.insert("tool".to_string(), Value::String(tool_name.to_string()));
1832 payload.insert(
1833 "idempotency_key".to_string(),
1834 Value::String(idempotency_key),
1835 );
1836 self.store.append_event(
1837 &ctx.run_id,
1838 Some(&ctx.session_id),
1839 Some(&ctx.step_id),
1840 "tool_call_completed",
1841 payload,
1842 Some(&ctx.agent_role),
1843 Some(ctx.state_version),
1844 Some(&causal_token),
1845 );
1846 self.store.record_cost(
1847 &ctx.run_id,
1848 &ctx.session_id,
1849 &ctx.step_id,
1850 "tool",
1851 tool_name,
1852 1.0,
1853 "call",
1854 State::new(),
1855 );
1856 Ok(value)
1857 }
1858 Err(err) => {
1859 if managed {
1860 self.store.update_ledger(
1861 &idempotency_key,
1862 "PENDING_VERIFICATION",
1863 None,
1864 Some("RuntimeError".to_string()),
1865 );
1866 }
1867 let mut payload = State::new();
1868 payload.insert("tool".to_string(), Value::String(tool_name.to_string()));
1869 payload.insert("error".to_string(), Value::String(err.0.clone()));
1870 self.store.append_event(
1871 &ctx.run_id,
1872 Some(&ctx.session_id),
1873 Some(&ctx.step_id),
1874 "tool_call_failed",
1875 payload,
1876 Some(&ctx.agent_role),
1877 Some(ctx.state_version),
1878 Some(&causal_token),
1879 );
1880 Err(err)
1881 }
1882 }
1883 }
1884
1885 fn record_permission(
1886 &mut self,
1887 ctx: &AgentContext,
1888 tool_name: &str,
1889 allowed: bool,
1890 reason: &str,
1891 causal_token: &str,
1892 ) {
1893 let mut permission = State::new();
1894 permission.insert("tool".to_string(), Value::String(tool_name.to_string()));
1895 permission.insert("allowed".to_string(), Value::Bool(allowed));
1896 permission.insert("reason".to_string(), Value::String(reason.to_string()));
1897 self.store.append_event(
1898 &ctx.run_id,
1899 Some(&ctx.session_id),
1900 Some(&ctx.step_id),
1901 "tool_permission_decided",
1902 permission,
1903 Some(&ctx.agent_role),
1904 Some(ctx.state_version),
1905 Some(causal_token),
1906 );
1907 }
1908
1909 pub fn create_artifact(
1910 &mut self,
1911 ctx: &AgentContext,
1912 name: &str,
1913 content: State,
1914 metadata: State,
1915 ) -> Result<String> {
1916 let artifact =
1917 self.store
1918 .create_artifact(&ctx.run_id, Some(&ctx.step_id), name, content, metadata);
1919 let mut payload = State::new();
1920 payload.insert(
1921 "artifact_id".to_string(),
1922 Value::String(artifact.artifact_id.clone()),
1923 );
1924 payload.insert("name".to_string(), Value::String(name.to_string()));
1925 self.store.append_event(
1926 &ctx.run_id,
1927 Some(&ctx.session_id),
1928 Some(&ctx.step_id),
1929 "artifact_created",
1930 payload,
1931 Some(&ctx.agent_role),
1932 Some(ctx.state_version),
1933 None,
1934 );
1935 Ok(artifact.artifact_id)
1936 }
1937
1938 pub fn create_media_artifact(
1939 &mut self,
1940 ctx: &AgentContext,
1941 name: &str,
1942 kind: &str,
1943 options: MediaArtifactOptions,
1944 ) -> Result<String> {
1945 if !is_media_kind(kind) {
1946 return Err(RuntimeError(format!("unsupported media kind: {kind}")));
1947 }
1948 let mut media_metadata = options.media_metadata.clone();
1949 media_metadata.insert(
1950 "schema_version".to_string(),
1951 Value::String(MEDIA_SCHEMA_VERSION.to_string()),
1952 );
1953 media_metadata.insert("kind".to_string(), Value::String(kind.to_string()));
1954 let mut content = State::new();
1955 content.insert(
1956 "schema_version".to_string(),
1957 Value::String(MEDIA_SCHEMA_VERSION.to_string()),
1958 );
1959 content.insert("kind".to_string(), Value::String(kind.to_string()));
1960 if let Some(uri) = &options.uri {
1961 content.insert("uri".to_string(), Value::String(uri.clone()));
1962 }
1963 if let Some(content_ref) = &options.content_ref {
1964 content.insert(
1965 "content_ref".to_string(),
1966 Value::String(content_ref.clone()),
1967 );
1968 }
1969 content.insert(
1970 "metadata".to_string(),
1971 Value::Object(media_metadata.clone()),
1972 );
1973 if !options.lineage.is_empty() {
1974 content.insert(
1975 "lineage".to_string(),
1976 Value::Object(options.lineage.clone()),
1977 );
1978 }
1979 if !options.derived_outputs.is_empty() {
1980 content.insert(
1981 "derived_outputs".to_string(),
1982 Value::Object(options.derived_outputs.clone()),
1983 );
1984 }
1985 let mut media = State::new();
1986 media.insert(
1987 "schema_version".to_string(),
1988 Value::String(MEDIA_SCHEMA_VERSION.to_string()),
1989 );
1990 media.insert("kind".to_string(), Value::String(kind.to_string()));
1991 if let Some(uri) = options.uri {
1992 media.insert("uri".to_string(), Value::String(uri));
1993 }
1994 if let Some(content_ref) = options.content_ref {
1995 media.insert("content_ref".to_string(), Value::String(content_ref));
1996 }
1997 media.insert("metadata".to_string(), Value::Object(media_metadata));
1998 media.insert("lineage".to_string(), Value::Object(options.lineage));
1999 let mut artifact_metadata = options.metadata;
2000 artifact_metadata.insert("agentledger_media".to_string(), Value::Object(media));
2001 self.create_artifact(ctx, name, content, artifact_metadata)
2002 }
2003
2004 pub fn create_stream_checkpoint(
2005 &mut self,
2006 ctx: &AgentContext,
2007 name: &str,
2008 options: StreamCheckpointOptions,
2009 ) -> Result<String> {
2010 if options.stream_id.is_empty() || options.consumer_id.is_empty() {
2011 return Err(RuntimeError(
2012 "stream_id and consumer_id are required".to_string(),
2013 ));
2014 }
2015 let chunk = options.chunk.map(stream_chunk_to_state);
2016 let mut content = State::new();
2017 content.insert(
2018 "schema_version".to_string(),
2019 Value::String(STREAM_SCHEMA_VERSION.to_string()),
2020 );
2021 content.insert(
2022 "stream_id".to_string(),
2023 Value::String(options.stream_id.clone()),
2024 );
2025 content.insert(
2026 "consumer_id".to_string(),
2027 Value::String(options.consumer_id.clone()),
2028 );
2029 content.insert("offset".to_string(), options.offset.clone());
2030 if let Some(watermark) = &options.watermark {
2031 content.insert("watermark".to_string(), watermark.clone());
2032 }
2033 if let Some(chunk_state) = &chunk {
2034 content.insert("chunk".to_string(), Value::Object(chunk_state.clone()));
2035 }
2036 if let Some(partial_ref) = &options.partial_result_ref {
2037 content.insert(
2038 "partial_result_ref".to_string(),
2039 Value::String(partial_ref.clone()),
2040 );
2041 }
2042 if !options.backpressure.is_empty() {
2043 content.insert(
2044 "backpressure".to_string(),
2045 Value::Object(options.backpressure.clone()),
2046 );
2047 }
2048 if !options.metadata.is_empty() {
2049 content.insert("metadata".to_string(), Value::Object(options.metadata));
2050 }
2051 let mut stream = State::new();
2052 stream.insert(
2053 "schema_version".to_string(),
2054 Value::String(STREAM_SCHEMA_VERSION.to_string()),
2055 );
2056 stream.insert("stream_id".to_string(), Value::String(options.stream_id));
2057 stream.insert(
2058 "consumer_id".to_string(),
2059 Value::String(options.consumer_id),
2060 );
2061 stream.insert("offset".to_string(), options.offset);
2062 if let Some(watermark) = options.watermark {
2063 stream.insert("watermark".to_string(), watermark);
2064 }
2065 if let Some(chunk_state) = chunk {
2066 stream.insert("chunk".to_string(), Value::Object(chunk_state));
2067 }
2068 if let Some(partial_ref) = options.partial_result_ref {
2069 stream.insert("partial_result_ref".to_string(), Value::String(partial_ref));
2070 }
2071 if !options.backpressure.is_empty() {
2072 stream.insert(
2073 "backpressure".to_string(),
2074 Value::Object(options.backpressure),
2075 );
2076 }
2077 let mut artifact_metadata = State::new();
2078 artifact_metadata.insert("agentledger_stream".to_string(), Value::Object(stream));
2079 self.create_artifact(ctx, name, content, artifact_metadata)
2080 }
2081
2082 pub fn record_model_call(
2083 &mut self,
2084 ctx: &AgentContext,
2085 model: &str,
2086 input_tokens: f64,
2087 output_tokens: f64,
2088 total_usd: f64,
2089 ) -> Result<()> {
2090 let tokens = input_tokens + output_tokens;
2091 if let Some(max) = self.budget.max_model_tokens {
2092 let used = self.store.cost_summary(&ctx.run_id).model_tokens;
2093 if used + tokens > max {
2094 let message = format!("model token budget exceeded: {used}+{tokens}/{max}");
2095 let mut payload = State::new();
2096 payload.insert("category".to_string(), Value::String("model".to_string()));
2097 payload.insert("model".to_string(), Value::String(model.to_string()));
2098 payload.insert("error".to_string(), Value::String(message.clone()));
2099 self.store.append_event(
2100 &ctx.run_id,
2101 Some(&ctx.session_id),
2102 Some(&ctx.step_id),
2103 "budget_check_failed",
2104 payload,
2105 Some(&ctx.agent_role),
2106 Some(ctx.state_version),
2107 None,
2108 );
2109 return Err(RuntimeError(message));
2110 }
2111 }
2112 let mut payload = State::new();
2113 payload.insert("model".to_string(), Value::String(model.to_string()));
2114 payload.insert("input_tokens".to_string(), Value::Number(input_tokens));
2115 payload.insert("output_tokens".to_string(), Value::Number(output_tokens));
2116 payload.insert("total_tokens".to_string(), Value::Number(tokens));
2117 payload.insert("total_usd".to_string(), Value::Number(total_usd));
2118 self.store.append_event(
2119 &ctx.run_id,
2120 Some(&ctx.session_id),
2121 Some(&ctx.step_id),
2122 "model_call_completed",
2123 payload,
2124 Some(&ctx.agent_role),
2125 Some(ctx.state_version),
2126 None,
2127 );
2128 if tokens > 0.0 {
2129 self.store.record_cost(
2130 &ctx.run_id,
2131 &ctx.session_id,
2132 &ctx.step_id,
2133 "model",
2134 model,
2135 tokens,
2136 "token",
2137 State::new(),
2138 );
2139 }
2140 if total_usd > 0.0 {
2141 self.store.record_cost(
2142 &ctx.run_id,
2143 &ctx.session_id,
2144 &ctx.step_id,
2145 "model",
2146 model,
2147 total_usd,
2148 "usd",
2149 State::new(),
2150 );
2151 }
2152 Ok(())
2153 }
2154}
2155
2156impl Default for Runtime {
2157 fn default() -> Self {
2158 Self::new()
2159 }
2160}
2161
2162#[derive(Clone, Debug)]
2163pub struct AgentContext {
2164 pub run_id: String,
2165 pub session_id: String,
2166 pub step_id: String,
2167 pub agent_role: String,
2168 pub lease_token: String,
2169 pub attempt: u64,
2170 pub state_version: u64,
2171 pub pending_patch: State,
2172}
2173
2174impl AgentContext {
2175 pub fn write_state(&mut self, key: &str, value: Value) {
2176 self.pending_patch.insert(key.to_string(), value);
2177 }
2178}
2179
2180#[derive(Clone, Debug)]
2181pub struct EvidenceBundle {
2182 pub schema_version: String,
2183 pub bundle_hash: String,
2184 pub run: Run,
2185 pub steps: Vec<Step>,
2186 pub events: Vec<Event>,
2187 pub tool_ledger: Vec<ToolLedgerEntry>,
2188 pub approvals: Vec<ApprovalRequest>,
2189 pub artifacts: Vec<Artifact>,
2190 pub media_artifacts: Vec<State>,
2191 pub stream_checkpoints: Vec<State>,
2192 pub cost_records: Vec<CostRecord>,
2193 pub cost_summary: CostSummary,
2194 pub summary: State,
2195 pub final_state: State,
2196}
2197
2198#[derive(Clone, Debug, Default)]
2199pub struct WorkerRunSummary {
2200 pub worker_id: String,
2201 pub run_id: Option<String>,
2202 pub iterations: u64,
2203 pub attempts: u64,
2204 pub succeeded_attempts: u64,
2205 pub recovered_leases: u64,
2206 pub final_status: Option<String>,
2207 pub stopped_reason: String,
2208}
2209
2210pub struct LocalWorker {
2211 pub worker_id: String,
2212 pub agent_role: String,
2213 pub lease_seconds: f64,
2214 pub recover_expired: bool,
2215}
2216
2217impl LocalWorker {
2218 pub fn new(worker_id: &str, agent_role: &str) -> Self {
2219 Self {
2220 worker_id: worker_id.to_string(),
2221 agent_role: agent_role.to_string(),
2222 lease_seconds: 60.0,
2223 recover_expired: true,
2224 }
2225 }
2226
2227 pub fn run_until_idle(
2228 &self,
2229 runtime: &mut Runtime,
2230 run_id: &str,
2231 max_iterations: u64,
2232 agent: AgentFunc,
2233 ) -> Result<WorkerRunSummary> {
2234 let mut summary = WorkerRunSummary {
2235 worker_id: self.worker_id.clone(),
2236 run_id: Some(run_id.to_string()),
2237 stopped_reason: "max_iterations".to_string(),
2238 ..WorkerRunSummary::default()
2239 };
2240 for index in 1..=max_iterations {
2241 summary.iterations = index;
2242 if self.recover_expired {
2243 summary.recovered_leases += runtime.store.recover_expired_leases() as u64;
2244 }
2245 let status = runtime.store.run(run_id)?.status;
2246 if is_terminal_status(&status) {
2247 summary.final_status = Some(status);
2248 summary.stopped_reason = "terminal_status".to_string();
2249 break;
2250 }
2251 let ok = runtime.run_once(
2252 run_id,
2253 &self.worker_id,
2254 &self.agent_role,
2255 self.lease_seconds,
2256 agent,
2257 )?;
2258 if !ok {
2259 summary.stopped_reason = "idle".to_string();
2260 break;
2261 }
2262 summary.attempts += 1;
2263 summary.succeeded_attempts += 1;
2264 }
2265 let status = runtime.store.run(run_id)?.status;
2266 summary.final_status = Some(status.clone());
2267 if is_terminal_status(&status) {
2268 summary.stopped_reason = "terminal_status".to_string();
2269 }
2270 Ok(summary)
2271 }
2272}
2273
2274#[derive(Clone, Debug, Default)]
2275pub struct WorkerServiceSummary {
2276 pub worker_id: String,
2277 pub run_id: Option<String>,
2278 pub loops: u64,
2279 pub attempts: u64,
2280 pub succeeded_attempts: u64,
2281 pub recovered_leases: u64,
2282 pub idle_polls: u64,
2283 pub stopped_reason: String,
2284 pub final_status: Option<String>,
2285 pub stop_requested: bool,
2286}
2287
2288pub struct WorkerService {
2289 pub worker: LocalWorker,
2290 pub stop_requested: bool,
2291 pub stop_reason: String,
2292}
2293
2294impl WorkerService {
2295 pub fn new(worker: LocalWorker) -> Self {
2296 Self {
2297 worker,
2298 stop_requested: false,
2299 stop_reason: "stop_requested".to_string(),
2300 }
2301 }
2302
2303 pub fn request_stop(&mut self, reason: &str) {
2304 self.stop_requested = true;
2305 self.stop_reason = reason.to_string();
2306 }
2307
2308 pub fn serve(
2309 &mut self,
2310 runtime: &mut Runtime,
2311 run_id: Option<&str>,
2312 max_loops: u64,
2313 max_idle_polls: Option<u64>,
2314 agent: AgentFunc,
2315 ) -> Result<WorkerServiceSummary> {
2316 let mut summary = WorkerServiceSummary {
2317 worker_id: self.worker.worker_id.clone(),
2318 run_id: run_id.map(str::to_string),
2319 stopped_reason: "max_loops".to_string(),
2320 ..WorkerServiceSummary::default()
2321 };
2322 while summary.loops < max_loops {
2323 if self.stop_requested {
2324 summary.stopped_reason = self.stop_reason.clone();
2325 summary.stop_requested = true;
2326 break;
2327 }
2328 summary.loops += 1;
2329 let Some(run_id) = run_id else {
2330 summary.idle_polls += 1;
2331 if max_idle_polls.is_some_and(|limit| summary.idle_polls >= limit) {
2332 summary.stopped_reason = "idle".to_string();
2333 break;
2334 }
2335 continue;
2336 };
2337 let run_summary = self.worker.run_until_idle(runtime, run_id, 1, agent)?;
2338 summary.attempts += run_summary.attempts;
2339 summary.succeeded_attempts += run_summary.succeeded_attempts;
2340 summary.recovered_leases += run_summary.recovered_leases;
2341 summary.final_status = run_summary.final_status.clone();
2342 if summary
2343 .final_status
2344 .as_deref()
2345 .is_some_and(is_terminal_status)
2346 {
2347 summary.stopped_reason = "terminal_status".to_string();
2348 break;
2349 }
2350 if run_summary.attempts == 0 {
2351 summary.idle_polls += 1;
2352 if max_idle_polls.is_some_and(|limit| summary.idle_polls >= limit) {
2353 summary.stopped_reason = "idle".to_string();
2354 break;
2355 }
2356 } else {
2357 summary.idle_polls = 0;
2358 }
2359 }
2360 Ok(summary)
2361 }
2362}
2363
2364fn is_terminal_status(status: &str) -> bool {
2365 matches!(status, "completed" | "failed" | "cancelled")
2366}
2367
2368pub fn export_evidence(store: &MemoryStore, run_id: &str) -> Result<EvidenceBundle> {
2369 let run = store.run(run_id)?;
2370 let steps = store.steps(run_id);
2371 let events = store.events(run_id);
2372 let tool_ledger = store.ledger(run_id);
2373 let approvals = store.approval_requests(run_id);
2374 let artifacts = store.artifacts(run_id);
2375 let media_artifacts = media_artifacts_from(&artifacts);
2376 let stream_checkpoints = stream_checkpoints_from(&artifacts);
2377 let cost_records = store.cost_records(run_id);
2378 let cost_summary = store.cost_summary(run_id);
2379 let summary = evidence_summary(
2380 &steps,
2381 &events,
2382 &tool_ledger,
2383 &approvals,
2384 &artifacts,
2385 &media_artifacts,
2386 &stream_checkpoints,
2387 &cost_records,
2388 &cost_summary,
2389 );
2390 let final_state = store.final_state(run_id)?;
2391 let basis = format!(
2392 "{}:{}:{}:{}:{}",
2393 run_id,
2394 steps.len(),
2395 events.len(),
2396 tool_ledger.len(),
2397 cost_records.len()
2398 );
2399 Ok(EvidenceBundle {
2400 schema_version: "agentledger.evidence.v1".to_string(),
2401 bundle_hash: stable_hash(&basis),
2402 run,
2403 steps,
2404 events,
2405 tool_ledger,
2406 approvals,
2407 artifacts,
2408 media_artifacts,
2409 stream_checkpoints,
2410 cost_records,
2411 cost_summary,
2412 summary,
2413 final_state,
2414 })
2415}
2416
2417fn evidence_summary(
2418 steps: &[Step],
2419 events: &[Event],
2420 ledger: &[ToolLedgerEntry],
2421 approvals: &[ApprovalRequest],
2422 artifacts: &[Artifact],
2423 media_artifacts: &[State],
2424 stream_checkpoints: &[State],
2425 cost_records: &[CostRecord],
2426 cost_summary: &CostSummary,
2427) -> State {
2428 let mut summary = State::new();
2429 summary.insert("event_count".into(), Value::Number(events.len() as f64));
2430 summary.insert("step_count".into(), Value::Number(steps.len() as f64));
2431 summary.insert(
2432 "tool_ledger_count".into(),
2433 Value::Number(ledger.len() as f64),
2434 );
2435 summary.insert(
2436 "approval_count".into(),
2437 Value::Number(approvals.len() as f64),
2438 );
2439 summary.insert(
2440 "artifact_count".into(),
2441 Value::Number(artifacts.len() as f64),
2442 );
2443 summary.insert(
2444 "media_artifact_count".into(),
2445 Value::Number(media_artifacts.len() as f64),
2446 );
2447 summary.insert(
2448 "stream_checkpoint_count".into(),
2449 Value::Number(stream_checkpoints.len() as f64),
2450 );
2451 summary.insert(
2452 "cost_record_count".into(),
2453 Value::Number(cost_records.len() as f64),
2454 );
2455 summary.insert(
2456 "has_failed_steps".into(),
2457 Value::Bool(steps.iter().any(|step| step.status == "failed")),
2458 );
2459 summary.insert(
2460 "has_pending_verification".into(),
2461 Value::Bool(
2462 ledger
2463 .iter()
2464 .any(|row| row.status == "PENDING_VERIFICATION"),
2465 ),
2466 );
2467 summary.insert(
2468 "has_pending_approvals".into(),
2469 Value::Bool(approvals.iter().any(|row| row.status == "PENDING")),
2470 );
2471 let mut cost = State::new();
2472 cost.insert("tool_calls".into(), Value::Number(cost_summary.tool_calls));
2473 cost.insert(
2474 "model_tokens".into(),
2475 Value::Number(cost_summary.model_tokens),
2476 );
2477 cost.insert("total_usd".into(), Value::Number(cost_summary.total_usd));
2478 summary.insert("cost_summary".into(), Value::Object(cost));
2479 summary
2480}
2481
2482#[derive(Clone, Debug)]
2483pub struct ReplaySummary {
2484 pub run_id: String,
2485 pub event_count: usize,
2486 pub tool_call_count: usize,
2487 pub final_state: State,
2488 pub event_hash: String,
2489 pub replay_safe: bool,
2490 pub artifact_count: usize,
2491 pub media_artifact_count: usize,
2492 pub stream_checkpoint_count: usize,
2493}
2494
2495pub fn replay(store: &MemoryStore, run_id: &str) -> Result<ReplaySummary> {
2496 let events = store.events(run_id);
2497 let tool_call_count = events
2498 .iter()
2499 .filter(|event| event.event_type.starts_with("tool_call_"))
2500 .count();
2501 let digest = events
2502 .iter()
2503 .map(|event| format!("{}:{}:{}", event.seq, event.event_type, event.payload_hash))
2504 .collect::<Vec<_>>()
2505 .join("|");
2506 let artifacts = store.artifacts(run_id);
2507 Ok(ReplaySummary {
2508 run_id: run_id.to_string(),
2509 event_count: events.len(),
2510 tool_call_count,
2511 final_state: store.final_state(run_id)?,
2512 event_hash: stable_hash(&digest),
2513 replay_safe: true,
2514 artifact_count: artifacts.len(),
2515 media_artifact_count: media_artifacts_from(&artifacts).len(),
2516 stream_checkpoint_count: stream_checkpoints_from(&artifacts).len(),
2517 })
2518}
2519
2520#[derive(Clone, Debug)]
2521pub struct CostAttributionReport {
2522 pub run_id: String,
2523 pub total: CostSummary,
2524 pub by_agent: HashMap<String, CostSummary>,
2525 pub by_step: HashMap<String, CostSummary>,
2526 pub by_name: HashMap<String, CostSummary>,
2527}
2528
2529pub fn cost_attribution(store: &MemoryStore, run_id: &str) -> CostAttributionReport {
2530 let mut step_roles = HashMap::new();
2531 for event in store.events(run_id) {
2532 if let (Some(step_id), Some(agent_role)) = (event.step_id, event.agent_role) {
2533 step_roles.insert(step_id, agent_role);
2534 }
2535 }
2536 let mut report = CostAttributionReport {
2537 run_id: run_id.to_string(),
2538 total: CostSummary::default(),
2539 by_agent: HashMap::new(),
2540 by_step: HashMap::new(),
2541 by_name: HashMap::new(),
2542 };
2543 for record in store.cost_records(run_id) {
2544 add_cost(&mut report.total, &record);
2545 let agent = step_roles
2546 .get(&record.step_id)
2547 .cloned()
2548 .unwrap_or_else(|| "<unknown>".to_string());
2549 add_cost(report.by_agent.entry(agent).or_default(), &record);
2550 add_cost(
2551 report.by_step.entry(record.step_id.clone()).or_default(),
2552 &record,
2553 );
2554 add_cost(
2555 report.by_name.entry(record.name.clone()).or_default(),
2556 &record,
2557 );
2558 }
2559 report
2560}
2561
2562#[derive(Clone, Debug)]
2563pub struct FailureAttributionReport {
2564 pub run_id: String,
2565 pub run_status: String,
2566 pub failed_steps: Vec<Step>,
2567 pub pending_verification: Vec<ToolLedgerEntry>,
2568 pub pending_approvals: Vec<ApprovalRequest>,
2569 pub failure_events: Vec<Event>,
2570 pub summary: State,
2571}
2572
2573pub fn failure_attribution(store: &MemoryStore, run_id: &str) -> Result<FailureAttributionReport> {
2574 let run = store.run(run_id)?;
2575 let failed_steps: Vec<Step> = store
2576 .steps(run_id)
2577 .into_iter()
2578 .filter(|step| step.status == "failed")
2579 .collect();
2580 let pending_verification: Vec<ToolLedgerEntry> = store
2581 .ledger(run_id)
2582 .into_iter()
2583 .filter(|entry| entry.status == "PENDING_VERIFICATION")
2584 .collect();
2585 let pending_approvals: Vec<ApprovalRequest> = store
2586 .approval_requests(run_id)
2587 .into_iter()
2588 .filter(|entry| entry.status == "PENDING")
2589 .collect();
2590 let failure_events: Vec<Event> = store
2591 .events(run_id)
2592 .into_iter()
2593 .filter(|event| is_failure_event(&event.event_type))
2594 .collect();
2595 let mut summary = State::new();
2596 summary.insert(
2597 "failed_step_count".to_string(),
2598 Value::Number(failed_steps.len() as f64),
2599 );
2600 summary.insert(
2601 "pending_verification_count".to_string(),
2602 Value::Number(pending_verification.len() as f64),
2603 );
2604 summary.insert(
2605 "pending_approval_count".to_string(),
2606 Value::Number(pending_approvals.len() as f64),
2607 );
2608 summary.insert(
2609 "failure_event_count".to_string(),
2610 Value::Number(failure_events.len() as f64),
2611 );
2612 Ok(FailureAttributionReport {
2613 run_id: run_id.to_string(),
2614 run_status: run.status,
2615 failed_steps,
2616 pending_verification,
2617 pending_approvals,
2618 failure_events,
2619 summary,
2620 })
2621}
2622
2623fn encode_store(store: &MemoryStore) -> String {
2624 let mut lines = vec!["AGENTLEDGER_RUST_STORE_V1".to_string()];
2625 let mut runs: Vec<_> = store.runs.values().collect();
2626 runs.sort_by(|a, b| a.run_id.cmp(&b.run_id));
2627 for run in runs {
2628 lines.push(join_fields(&[
2629 "R".to_string(),
2630 hex_encode(&run.run_id),
2631 hex_encode(&run.session_id),
2632 hex_encode(&run.status),
2633 run.state_version.to_string(),
2634 run.created_at.to_string(),
2635 run.updated_at.to_string(),
2636 encode_state(&run.state),
2637 ]));
2638 }
2639
2640 let mut steps: Vec<_> = store.steps.values().collect();
2641 steps.sort_by(|a, b| a.step_id.cmp(&b.step_id));
2642 for step in steps {
2643 lines.push(join_fields(&[
2644 "S".to_string(),
2645 hex_encode(&step.step_id),
2646 hex_encode(&step.run_id),
2647 hex_encode(&step.session_id),
2648 hex_encode(&step.status),
2649 encode_option_string(&step.owner),
2650 encode_option_string(&step.lease_token),
2651 encode_option_f64(step.lease_until),
2652 step.attempt.to_string(),
2653 step.state_version.to_string(),
2654 encode_option_string(&step.checkpoint_id),
2655 encode_option_string(&step.last_error_type),
2656 encode_option_string(&step.last_error),
2657 encode_option_f64(step.cancelled_at),
2658 step.created_at.to_string(),
2659 step.updated_at.to_string(),
2660 ]));
2661 }
2662
2663 let mut run_ids: Vec<_> = store.events.keys().collect();
2664 run_ids.sort();
2665 for run_id in run_ids {
2666 if let Some(events) = store.events.get(run_id) {
2667 for event in events {
2668 lines.push(join_fields(&[
2669 "E".to_string(),
2670 hex_encode(&event.event_id),
2671 hex_encode(&event.run_id),
2672 encode_option_string(&event.session_id),
2673 encode_option_string(&event.step_id),
2674 event.seq.to_string(),
2675 hex_encode(&event.event_type),
2676 event.timestamp.to_string(),
2677 encode_option_string(&event.agent_role),
2678 encode_option_u64(event.state_version),
2679 encode_option_string(&event.causal_token),
2680 hex_encode(&event.payload_hash),
2681 hex_encode(&event.payload_ref),
2682 encode_state(&event.payload),
2683 ]));
2684 }
2685 }
2686 }
2687
2688 let mut ledgers: Vec<_> = store.tool_ledger.values().collect();
2689 ledgers.sort_by(|a, b| a.idempotency_key.cmp(&b.idempotency_key));
2690 for entry in ledgers {
2691 lines.push(join_fields(&[
2692 "L".to_string(),
2693 hex_encode(&entry.ledger_id),
2694 hex_encode(&entry.run_id),
2695 hex_encode(&entry.session_id),
2696 hex_encode(&entry.step_id),
2697 hex_encode(&entry.tool_name),
2698 hex_encode(&entry.tool_version),
2699 hex_encode(&entry.tool_call_id),
2700 hex_encode(&entry.idempotency_key),
2701 hex_encode(&entry.causal_token),
2702 hex_encode(&entry.request_hash),
2703 hex_encode(&entry.request_ref),
2704 hex_encode(&entry.status),
2705 encode_option_string(&entry.external_id),
2706 encode_option_string(&entry.response_hash),
2707 encode_option_string(&entry.response_ref),
2708 encode_option_string(&entry.error_type),
2709 encode_option_value(&entry.response),
2710 entry.created_at.to_string(),
2711 entry.updated_at.to_string(),
2712 ]));
2713 }
2714
2715 let mut approvals: Vec<_> = store.approval_requests.values().collect();
2716 approvals.sort_by(|a, b| a.approval_key.cmp(&b.approval_key));
2717 for approval in approvals {
2718 lines.push(join_fields(&[
2719 "A".to_string(),
2720 hex_encode(&approval.approval_id),
2721 hex_encode(&approval.approval_key),
2722 hex_encode(&approval.run_id),
2723 hex_encode(&approval.session_id),
2724 hex_encode(&approval.step_id),
2725 hex_encode(&approval.tool_name),
2726 hex_encode(&approval.risk_level),
2727 hex_encode(&approval.status),
2728 hex_encode(&approval.reason),
2729 hex_encode(&approval.request_hash),
2730 hex_encode(&approval.request_ref),
2731 hex_encode(&approval.requested_by),
2732 encode_option_string(&approval.approved_by),
2733 encode_option_string(&approval.decision_reason),
2734 approval.created_at.to_string(),
2735 approval.updated_at.to_string(),
2736 ]));
2737 }
2738
2739 let mut cost_run_ids: Vec<_> = store.cost_records.keys().collect();
2740 cost_run_ids.sort();
2741 for run_id in cost_run_ids {
2742 if let Some(records) = store.cost_records.get(run_id) {
2743 for record in records {
2744 lines.push(join_fields(&[
2745 "C".to_string(),
2746 hex_encode(&record.cost_id),
2747 hex_encode(&record.run_id),
2748 hex_encode(&record.session_id),
2749 hex_encode(&record.step_id),
2750 hex_encode(&record.category),
2751 hex_encode(&record.name),
2752 record.amount.to_string(),
2753 hex_encode(&record.unit),
2754 encode_state(&record.metadata),
2755 record.created_at.to_string(),
2756 ]));
2757 }
2758 }
2759 }
2760
2761 let mut artifact_run_ids: Vec<_> = store.artifacts.keys().collect();
2762 artifact_run_ids.sort();
2763 for run_id in artifact_run_ids {
2764 if let Some(artifacts) = store.artifacts.get(run_id) {
2765 for artifact in artifacts {
2766 lines.push(join_fields(&[
2767 "F".to_string(),
2768 hex_encode(&artifact.artifact_id),
2769 hex_encode(&artifact.run_id),
2770 encode_option_string(&artifact.step_id),
2771 hex_encode(&artifact.name),
2772 hex_encode(&artifact.blob_hash),
2773 hex_encode(&artifact.blob_ref),
2774 encode_state(&artifact.metadata),
2775 artifact.created_at.to_string(),
2776 ]));
2777 }
2778 }
2779 }
2780
2781 lines.push(String::new());
2782 lines.join("\n")
2783}
2784
2785fn decode_store(body: &str) -> Result<MemoryStore> {
2786 let mut lines = body.lines();
2787 match lines.next() {
2788 Some("AGENTLEDGER_RUST_STORE_V1") => {}
2789 _ => {
2790 return Err(RuntimeError(
2791 "invalid Rust store snapshot header".to_string(),
2792 ))
2793 }
2794 }
2795 let mut store = MemoryStore::new();
2796 for line in lines.filter(|line| !line.trim().is_empty()) {
2797 let fields: Vec<&str> = line.split('\t').collect();
2798 let tag = fields.first().copied().unwrap_or_default();
2799 match tag {
2800 "R" => {
2801 require_len(tag, &fields, 8)?;
2802 let run = Run {
2803 run_id: hex_decode(fields[1])?,
2804 session_id: hex_decode(fields[2])?,
2805 status: hex_decode(fields[3])?,
2806 state_version: parse_u64(fields[4])?,
2807 created_at: parse_f64(fields[5])?,
2808 updated_at: parse_f64(fields[6])?,
2809 state: decode_state(fields[7])?,
2810 };
2811 store.runs.insert(run.run_id.clone(), run);
2812 }
2813 "S" => {
2814 require_len(tag, &fields, 16)?;
2815 let step = Step {
2816 step_id: hex_decode(fields[1])?,
2817 run_id: hex_decode(fields[2])?,
2818 session_id: hex_decode(fields[3])?,
2819 status: hex_decode(fields[4])?,
2820 owner: decode_option_string(fields[5])?,
2821 lease_token: decode_option_string(fields[6])?,
2822 lease_until: decode_option_f64(fields[7])?,
2823 attempt: parse_u64(fields[8])?,
2824 state_version: parse_u64(fields[9])?,
2825 checkpoint_id: decode_option_string(fields[10])?,
2826 last_error_type: decode_option_string(fields[11])?,
2827 last_error: decode_option_string(fields[12])?,
2828 cancelled_at: decode_option_f64(fields[13])?,
2829 created_at: parse_f64(fields[14])?,
2830 updated_at: parse_f64(fields[15])?,
2831 };
2832 store.steps.insert(step.step_id.clone(), step);
2833 }
2834 "E" => {
2835 require_len(tag, &fields, 14)?;
2836 let event = Event {
2837 event_id: hex_decode(fields[1])?,
2838 run_id: hex_decode(fields[2])?,
2839 session_id: decode_option_string(fields[3])?,
2840 step_id: decode_option_string(fields[4])?,
2841 seq: parse_u64(fields[5])?,
2842 event_type: hex_decode(fields[6])?,
2843 timestamp: parse_f64(fields[7])?,
2844 agent_role: decode_option_string(fields[8])?,
2845 state_version: decode_option_u64(fields[9])?,
2846 causal_token: decode_option_string(fields[10])?,
2847 payload_hash: hex_decode(fields[11])?,
2848 payload_ref: hex_decode(fields[12])?,
2849 payload: decode_state(fields[13])?,
2850 };
2851 store
2852 .events
2853 .entry(event.run_id.clone())
2854 .or_default()
2855 .push(event);
2856 }
2857 "L" => {
2858 require_len(tag, &fields, 20)?;
2859 let entry = ToolLedgerEntry {
2860 ledger_id: hex_decode(fields[1])?,
2861 run_id: hex_decode(fields[2])?,
2862 session_id: hex_decode(fields[3])?,
2863 step_id: hex_decode(fields[4])?,
2864 tool_name: hex_decode(fields[5])?,
2865 tool_version: hex_decode(fields[6])?,
2866 tool_call_id: hex_decode(fields[7])?,
2867 idempotency_key: hex_decode(fields[8])?,
2868 causal_token: hex_decode(fields[9])?,
2869 request_hash: hex_decode(fields[10])?,
2870 request_ref: hex_decode(fields[11])?,
2871 status: hex_decode(fields[12])?,
2872 external_id: decode_option_string(fields[13])?,
2873 response_hash: decode_option_string(fields[14])?,
2874 response_ref: decode_option_string(fields[15])?,
2875 error_type: decode_option_string(fields[16])?,
2876 response: decode_option_value(fields[17])?,
2877 created_at: parse_f64(fields[18])?,
2878 updated_at: parse_f64(fields[19])?,
2879 };
2880 store
2881 .tool_ledger
2882 .insert(entry.idempotency_key.clone(), entry);
2883 }
2884 "A" => {
2885 require_len(tag, &fields, 17)?;
2886 let approval = ApprovalRequest {
2887 approval_id: hex_decode(fields[1])?,
2888 approval_key: hex_decode(fields[2])?,
2889 run_id: hex_decode(fields[3])?,
2890 session_id: hex_decode(fields[4])?,
2891 step_id: hex_decode(fields[5])?,
2892 tool_name: hex_decode(fields[6])?,
2893 risk_level: hex_decode(fields[7])?,
2894 status: hex_decode(fields[8])?,
2895 reason: hex_decode(fields[9])?,
2896 request_hash: hex_decode(fields[10])?,
2897 request_ref: hex_decode(fields[11])?,
2898 requested_by: hex_decode(fields[12])?,
2899 approved_by: decode_option_string(fields[13])?,
2900 decision_reason: decode_option_string(fields[14])?,
2901 created_at: parse_f64(fields[15])?,
2902 updated_at: parse_f64(fields[16])?,
2903 };
2904 store
2905 .approval_requests
2906 .insert(approval.approval_key.clone(), approval);
2907 }
2908 "C" => {
2909 require_len(tag, &fields, 11)?;
2910 let record = CostRecord {
2911 cost_id: hex_decode(fields[1])?,
2912 run_id: hex_decode(fields[2])?,
2913 session_id: hex_decode(fields[3])?,
2914 step_id: hex_decode(fields[4])?,
2915 category: hex_decode(fields[5])?,
2916 name: hex_decode(fields[6])?,
2917 amount: parse_f64(fields[7])?,
2918 unit: hex_decode(fields[8])?,
2919 metadata: decode_state(fields[9])?,
2920 created_at: parse_f64(fields[10])?,
2921 };
2922 store
2923 .cost_records
2924 .entry(record.run_id.clone())
2925 .or_default()
2926 .push(record);
2927 }
2928 "F" => {
2929 require_len(tag, &fields, 9)?;
2930 let artifact = Artifact {
2931 artifact_id: hex_decode(fields[1])?,
2932 run_id: hex_decode(fields[2])?,
2933 step_id: decode_option_string(fields[3])?,
2934 name: hex_decode(fields[4])?,
2935 blob_hash: hex_decode(fields[5])?,
2936 blob_ref: hex_decode(fields[6])?,
2937 metadata: decode_state(fields[7])?,
2938 created_at: parse_f64(fields[8])?,
2939 };
2940 store
2941 .artifacts
2942 .entry(artifact.run_id.clone())
2943 .or_default()
2944 .push(artifact);
2945 }
2946 _ => {
2947 return Err(RuntimeError(format!(
2948 "unknown Rust store snapshot row: {tag}"
2949 )))
2950 }
2951 }
2952 }
2953 Ok(store)
2954}
2955
2956fn join_fields(fields: &[String]) -> String {
2957 fields.join("\t")
2958}
2959
2960fn require_len(tag: &str, fields: &[&str], expected: usize) -> Result<()> {
2961 if fields.len() != expected {
2962 return Err(RuntimeError(format!(
2963 "invalid {tag} row: expected {expected} fields, got {}",
2964 fields.len()
2965 )));
2966 }
2967 Ok(())
2968}
2969
2970fn encode_option_string(value: &Option<String>) -> String {
2971 value
2972 .as_ref()
2973 .map(|item| hex_encode(item))
2974 .unwrap_or_else(|| "-".to_string())
2975}
2976
2977fn decode_option_string(value: &str) -> Result<Option<String>> {
2978 if value == "-" {
2979 Ok(None)
2980 } else {
2981 Ok(Some(hex_decode(value)?))
2982 }
2983}
2984
2985fn encode_option_f64(value: Option<f64>) -> String {
2986 value
2987 .map(|item| item.to_string())
2988 .unwrap_or_else(|| "-".to_string())
2989}
2990
2991fn decode_option_f64(value: &str) -> Result<Option<f64>> {
2992 if value == "-" {
2993 Ok(None)
2994 } else {
2995 Ok(Some(parse_f64(value)?))
2996 }
2997}
2998
2999fn encode_option_u64(value: Option<u64>) -> String {
3000 value
3001 .map(|item| item.to_string())
3002 .unwrap_or_else(|| "-".to_string())
3003}
3004
3005fn decode_option_u64(value: &str) -> Result<Option<u64>> {
3006 if value == "-" {
3007 Ok(None)
3008 } else {
3009 Ok(Some(parse_u64(value)?))
3010 }
3011}
3012
3013fn encode_option_value(value: &Option<Value>) -> String {
3014 value
3015 .as_ref()
3016 .map(encode_value)
3017 .unwrap_or_else(|| "-".to_string())
3018}
3019
3020fn decode_option_value(value: &str) -> Result<Option<Value>> {
3021 if value == "-" {
3022 Ok(None)
3023 } else {
3024 Ok(Some(decode_value(value)?))
3025 }
3026}
3027
3028fn encode_state(state: &State) -> String {
3029 encode_value(&Value::Object(state.clone()))
3030}
3031
3032fn decode_state(encoded: &str) -> Result<State> {
3033 match decode_value(encoded)? {
3034 Value::Object(state) => Ok(state),
3035 _ => Err(RuntimeError("encoded state was not an object".to_string())),
3036 }
3037}
3038
3039fn encode_value(value: &Value) -> String {
3040 match value {
3041 Value::Null => "Z".to_string(),
3042 Value::Bool(true) => "T".to_string(),
3043 Value::Bool(false) => "F".to_string(),
3044 Value::Number(item) => format!("N{}:", hex_encode(&item.to_string())),
3045 Value::String(item) => format!("S{}:", hex_encode(item)),
3046 Value::Object(state) => {
3047 let mut keys: Vec<_> = state.keys().collect();
3048 keys.sort();
3049 let mut out = format!("O{}:", keys.len());
3050 for key in keys {
3051 out.push_str(&hex_encode(key));
3052 out.push(':');
3053 out.push_str(&encode_value(&state[key]));
3054 }
3055 out
3056 }
3057 Value::Array(values) => {
3058 let mut out = format!("A{}:", values.len());
3059 for value in values {
3060 out.push_str(&encode_value(value));
3061 }
3062 out
3063 }
3064 }
3065}
3066
3067fn decode_value(encoded: &str) -> Result<Value> {
3068 let (value, index) = parse_value(encoded, 0)?;
3069 if index != encoded.len() {
3070 return Err(RuntimeError("trailing bytes in encoded value".to_string()));
3071 }
3072 Ok(value)
3073}
3074
3075fn parse_value(input: &str, index: usize) -> Result<(Value, usize)> {
3076 let bytes = input.as_bytes();
3077 let tag = *bytes
3078 .get(index)
3079 .ok_or_else(|| RuntimeError("unexpected end of encoded value".to_string()))?
3080 as char;
3081 match tag {
3082 'Z' => Ok((Value::Null, index + 1)),
3083 'T' => Ok((Value::Bool(true), index + 1)),
3084 'F' => Ok((Value::Bool(false), index + 1)),
3085 'N' => {
3086 let (hex, next) = read_until_colon(input, index + 1)?;
3087 Ok((Value::Number(parse_f64(&hex_decode(hex)?)?), next))
3088 }
3089 'S' => {
3090 let (hex, next) = read_until_colon(input, index + 1)?;
3091 Ok((Value::String(hex_decode(hex)?), next))
3092 }
3093 'O' => {
3094 let (count_text, mut next) = read_until_colon(input, index + 1)?;
3095 let count = count_text
3096 .parse::<usize>()
3097 .map_err(|err| RuntimeError(err.to_string()))?;
3098 let mut state = State::new();
3099 for _ in 0..count {
3100 let (key_hex, after_key) = read_until_colon(input, next)?;
3101 let key = hex_decode(key_hex)?;
3102 let (value, after_value) = parse_value(input, after_key)?;
3103 state.insert(key, value);
3104 next = after_value;
3105 }
3106 Ok((Value::Object(state), next))
3107 }
3108 'A' => {
3109 let (count_text, mut next) = read_until_colon(input, index + 1)?;
3110 let count = count_text
3111 .parse::<usize>()
3112 .map_err(|err| RuntimeError(err.to_string()))?;
3113 let mut values = Vec::with_capacity(count);
3114 for _ in 0..count {
3115 let (value, after_value) = parse_value(input, next)?;
3116 values.push(value);
3117 next = after_value;
3118 }
3119 Ok((Value::Array(values), next))
3120 }
3121 _ => Err(RuntimeError(format!("unknown encoded value tag: {tag}"))),
3122 }
3123}
3124
3125fn read_until_colon(input: &str, index: usize) -> Result<(&str, usize)> {
3126 let rest = input
3127 .get(index..)
3128 .ok_or_else(|| RuntimeError("invalid encoded value index".to_string()))?;
3129 let offset = rest
3130 .find(':')
3131 .ok_or_else(|| RuntimeError("missing encoded value delimiter".to_string()))?;
3132 Ok((&rest[..offset], index + offset + 1))
3133}
3134
3135fn hex_encode(value: &str) -> String {
3136 value
3137 .as_bytes()
3138 .iter()
3139 .map(|byte| format!("{byte:02x}"))
3140 .collect()
3141}
3142
3143fn hex_decode(value: &str) -> Result<String> {
3144 if value.len() % 2 != 0 {
3145 return Err(RuntimeError("invalid hex string length".to_string()));
3146 }
3147 let mut bytes = Vec::with_capacity(value.len() / 2);
3148 for index in (0..value.len()).step_by(2) {
3149 let byte = u8::from_str_radix(&value[index..index + 2], 16)
3150 .map_err(|err| RuntimeError(err.to_string()))?;
3151 bytes.push(byte);
3152 }
3153 String::from_utf8(bytes).map_err(|err| RuntimeError(err.to_string()))
3154}
3155
3156fn parse_f64(value: &str) -> Result<f64> {
3157 value
3158 .parse::<f64>()
3159 .map_err(|err| RuntimeError(err.to_string()))
3160}
3161
3162fn parse_u64(value: &str) -> Result<u64> {
3163 value
3164 .parse::<u64>()
3165 .map_err(|err| RuntimeError(err.to_string()))
3166}
3167
3168fn merge_patch(base: &State, patch: &State) -> State {
3169 let mut out = base.clone();
3170 for (key, value) in patch {
3171 match value {
3172 Value::Null => {
3173 out.remove(key);
3174 }
3175 Value::Object(patch_map) => {
3176 if let Some(Value::Object(base_map)) = out.get(key) {
3177 out.insert(key.clone(), Value::Object(merge_patch(base_map, patch_map)));
3178 } else {
3179 out.insert(key.clone(), value.clone());
3180 }
3181 }
3182 _ => {
3183 out.insert(key.clone(), value.clone());
3184 }
3185 }
3186 }
3187 out
3188}
3189
3190fn add_cost(summary: &mut CostSummary, record: &CostRecord) {
3191 if (record.category == "tool" || record.category == "tool_shadow") && record.unit == "call" {
3192 summary.tool_calls += record.amount;
3193 }
3194 if record.category == "model" && record.unit == "token" {
3195 summary.model_tokens += record.amount;
3196 }
3197 if record.unit == "usd" {
3198 summary.total_usd += record.amount;
3199 }
3200 let key = format!("{}:{}", record.category, record.unit);
3201 *summary.by_category.entry(key).or_insert(0.0) += record.amount;
3202}
3203
3204pub fn validate_tool_schema(schema: &Value, value: &Value, path: &str) -> Result<()> {
3205 let schema = match schema {
3206 Value::Object(schema) => schema,
3207 _ => return Err(RuntimeError(format!("{path} schema must be object"))),
3208 };
3209 if let Some(expected) = schema.get("const") {
3210 if expected != value {
3211 return Err(RuntimeError(format!("{path} expected const")));
3212 }
3213 }
3214 if let Some(Value::Array(items)) = schema.get("enum") {
3215 if !items.iter().any(|item| item == value) {
3216 return Err(RuntimeError(format!("{path} value not in enum")));
3217 }
3218 }
3219 let Some(Value::String(kind)) = schema.get("type") else {
3220 return Ok(());
3221 };
3222 match kind.as_str() {
3223 "object" => {
3224 let Value::Object(object) = value else {
3225 return Err(RuntimeError(format!("{path} expected object")));
3226 };
3227 if let Some(Value::Array(required)) = schema.get("required") {
3228 for item in required {
3229 if let Value::String(key) = item {
3230 if !object.contains_key(key) {
3231 return Err(RuntimeError(format!("{path}.{key} is required")));
3232 }
3233 }
3234 }
3235 }
3236 let properties = match schema.get("properties") {
3237 Some(Value::Object(properties)) => properties,
3238 _ => return Ok(()),
3239 };
3240 for (key, child_schema) in properties {
3241 if let Some(child) = object.get(key) {
3242 validate_tool_schema(child_schema, child, &format!("{path}.{key}"))?;
3243 }
3244 }
3245 if schema.get("additionalProperties") == Some(&Value::Bool(false)) {
3246 for key in object.keys() {
3247 if !properties.contains_key(key) {
3248 return Err(RuntimeError(format!("{path}.{key} is not allowed")));
3249 }
3250 }
3251 }
3252 }
3253 "string" => {
3254 let Value::String(text) = value else {
3255 return Err(RuntimeError(format!("{path} expected string")));
3256 };
3257 if let Some(Value::Number(min)) = schema.get("minLength") {
3258 if (text.len() as f64) < *min {
3259 return Err(RuntimeError(format!("{path} shorter than minLength")));
3260 }
3261 }
3262 if let Some(Value::Number(max)) = schema.get("maxLength") {
3263 if (text.len() as f64) > *max {
3264 return Err(RuntimeError(format!("{path} longer than maxLength")));
3265 }
3266 }
3267 }
3268 "number" | "integer" => {
3269 let Value::Number(number) = value else {
3270 return Err(RuntimeError(format!("{path} expected number")));
3271 };
3272 if kind == "integer" && number.fract() != 0.0 {
3273 return Err(RuntimeError(format!("{path} expected integer")));
3274 }
3275 if let Some(Value::Number(min)) = schema.get("minimum") {
3276 if number < min {
3277 return Err(RuntimeError(format!("{path} below minimum")));
3278 }
3279 }
3280 if let Some(Value::Number(max)) = schema.get("maximum") {
3281 if number > max {
3282 return Err(RuntimeError(format!("{path} above maximum")));
3283 }
3284 }
3285 }
3286 "boolean" => {
3287 if !matches!(value, Value::Bool(_)) {
3288 return Err(RuntimeError(format!("{path} expected boolean")));
3289 }
3290 }
3291 _ => {}
3292 }
3293 Ok(())
3294}
3295
3296fn is_high_risk(risk: &str) -> bool {
3297 matches!(
3298 risk,
3299 "high" | "destructive" | "sensitive" | "financial_or_legal"
3300 )
3301}
3302
3303fn is_media_kind(kind: &str) -> bool {
3304 matches!(
3305 kind,
3306 "image"
3307 | "audio"
3308 | "video"
3309 | "frame"
3310 | "audio_segment"
3311 | "video_segment"
3312 | "transcript"
3313 | "embedding"
3314 | "derived"
3315 )
3316}
3317
3318fn stream_chunk_to_state(chunk: StreamChunkRef) -> State {
3319 let mut state = State::new();
3320 state.insert(
3321 "schema_version".to_string(),
3322 Value::String(STREAM_SCHEMA_VERSION.to_string()),
3323 );
3324 state.insert("stream_id".to_string(), Value::String(chunk.stream_id));
3325 state.insert("chunk_id".to_string(), Value::String(chunk.chunk_id));
3326 state.insert("offset".to_string(), chunk.offset);
3327 if let Some(content_ref) = chunk.content_ref {
3328 state.insert("content_ref".to_string(), Value::String(content_ref));
3329 }
3330 if let Some(content_hash) = chunk.content_hash {
3331 state.insert("content_hash".to_string(), Value::String(content_hash));
3332 }
3333 if let Some(sequence) = chunk.sequence {
3334 state.insert("sequence".to_string(), Value::Number(sequence));
3335 }
3336 if let Some(event_time) = chunk.event_time {
3337 state.insert("event_time".to_string(), Value::Number(event_time));
3338 }
3339 if !chunk.metadata.is_empty() {
3340 state.insert("metadata".to_string(), Value::Object(chunk.metadata));
3341 }
3342 state
3343}
3344
3345fn media_artifacts_from(artifacts: &[Artifact]) -> Vec<State> {
3346 artifacts
3347 .iter()
3348 .filter_map(
3349 |artifact| match artifact.metadata.get("agentledger_media") {
3350 Some(Value::Object(metadata)) => {
3351 let mut row = State::new();
3352 row.insert(
3353 "artifact_id".to_string(),
3354 Value::String(artifact.artifact_id.clone()),
3355 );
3356 row.insert("name".to_string(), Value::String(artifact.name.clone()));
3357 row.insert(
3358 "blob_hash".to_string(),
3359 Value::String(artifact.blob_hash.clone()),
3360 );
3361 row.insert(
3362 "blob_ref".to_string(),
3363 Value::String(artifact.blob_ref.clone()),
3364 );
3365 for key in ["kind", "uri", "content_ref", "metadata", "lineage"] {
3366 if let Some(value) = metadata.get(key) {
3367 row.insert(key.to_string(), value.clone());
3368 }
3369 }
3370 Some(row)
3371 }
3372 _ => None,
3373 },
3374 )
3375 .collect()
3376}
3377
3378fn stream_checkpoints_from(artifacts: &[Artifact]) -> Vec<State> {
3379 artifacts
3380 .iter()
3381 .filter_map(
3382 |artifact| match artifact.metadata.get("agentledger_stream") {
3383 Some(Value::Object(metadata)) => {
3384 let mut row = State::new();
3385 row.insert(
3386 "artifact_id".to_string(),
3387 Value::String(artifact.artifact_id.clone()),
3388 );
3389 row.insert("name".to_string(), Value::String(artifact.name.clone()));
3390 row.insert(
3391 "blob_hash".to_string(),
3392 Value::String(artifact.blob_hash.clone()),
3393 );
3394 row.insert(
3395 "blob_ref".to_string(),
3396 Value::String(artifact.blob_ref.clone()),
3397 );
3398 for key in [
3399 "stream_id",
3400 "consumer_id",
3401 "offset",
3402 "watermark",
3403 "chunk",
3404 "partial_result_ref",
3405 "backpressure",
3406 ] {
3407 if let Some(value) = metadata.get(key) {
3408 row.insert(key.to_string(), value.clone());
3409 }
3410 }
3411 Some(row)
3412 }
3413 _ => None,
3414 },
3415 )
3416 .collect()
3417}
3418
3419fn classify_runtime_error(message: &str) -> &'static str {
3420 if message.contains("budget exceeded") || message.contains("budget") {
3421 "BudgetExceededError"
3422 } else if message.contains("sandbox executor") {
3423 "SandboxUnavailableError"
3424 } else if message.contains("high-risk") || message.contains("denied") {
3425 "PermissionDeniedError"
3426 } else {
3427 "RuntimeError"
3428 }
3429}
3430
3431fn failure_source(error_type: &str) -> &'static str {
3432 match error_type {
3433 "BudgetExceededError" => "budget",
3434 "SandboxUnavailableError" => "sandbox",
3435 "PermissionDeniedError" | "ApprovalDenied" => "policy",
3436 _ => "agent",
3437 }
3438}
3439
3440fn is_failure_event(kind: &str) -> bool {
3441 matches!(
3442 kind,
3443 "failure_classified"
3444 | "error_raised"
3445 | "step_failed"
3446 | "step_retry_scheduled"
3447 | "step_waiting_human"
3448 | "lease_expired"
3449 | "run_cancel_requested"
3450 | "run_cancelled"
3451 | "tool_call_failed"
3452 | "tool_approval_required"
3453 | "budget_check_failed"
3454 )
3455}
3456
3457fn format_state(state: &State) -> String {
3458 let mut keys: Vec<&String> = state.keys().collect();
3459 keys.sort();
3460 keys.into_iter()
3461 .map(|key| format!("{}={}", key, format_value(&state[key])))
3462 .collect::<Vec<_>>()
3463 .join(",")
3464}
3465
3466fn format_value(value: &Value) -> String {
3467 match value {
3468 Value::Null => "null".to_string(),
3469 Value::Bool(value) => value.to_string(),
3470 Value::Number(value) => value.to_string(),
3471 Value::String(value) => format!("\"{}\"", value),
3472 Value::Object(value) => format!("{{{}}}", format_state(value)),
3473 Value::Array(values) => format!(
3474 "[{}]",
3475 values
3476 .iter()
3477 .map(format_value)
3478 .collect::<Vec<_>>()
3479 .join(",")
3480 ),
3481 }
3482}
3483
3484fn stable_hash(input: &str) -> String {
3485 let mut hash: u64 = 0xcbf29ce484222325;
3486 for byte in input.as_bytes() {
3487 hash ^= *byte as u64;
3488 hash = hash.wrapping_mul(0x100000001b3);
3489 }
3490 format!("{hash:016x}")
3491}
3492
3493fn new_id(prefix: &str) -> String {
3494 let value = ID_COUNTER.fetch_add(1, Ordering::Relaxed);
3495 format!("{prefix}_{value:016x}")
3496}
3497
3498fn now_seconds() -> f64 {
3499 SystemTime::now()
3500 .duration_since(UNIX_EPOCH)
3501 .unwrap()
3502 .as_secs_f64()
3503}
3504
3505#[cfg(test)]
3506mod tests {
3507 use super::*;
3508
3509 fn state(items: &[(&str, Value)]) -> State {
3510 items
3511 .iter()
3512 .map(|(key, value)| ((*key).to_string(), value.clone()))
3513 .collect()
3514 }
3515
3516 fn event_exists(events: &[Event], event_type: &str) -> bool {
3517 events.iter().any(|event| event.event_type == event_type)
3518 }
3519
3520 fn claim_context(
3521 runtime: &mut Runtime,
3522 run_id: &str,
3523 worker: &str,
3524 role: &str,
3525 ) -> AgentContext {
3526 let claim = runtime.store.claim_step(worker, run_id, 60.0).unwrap();
3527 let (_state, version, session_id) = runtime.store.load_state(run_id).unwrap();
3528 let mut payload = State::new();
3529 payload.insert("agent_role".to_string(), Value::String(role.to_string()));
3530 runtime.store.append_event(
3531 run_id,
3532 Some(&session_id),
3533 Some(&claim.step_id),
3534 "agent_started",
3535 payload,
3536 Some(role),
3537 Some(version),
3538 None,
3539 );
3540 AgentContext {
3541 run_id: run_id.to_string(),
3542 session_id,
3543 step_id: claim.step_id,
3544 agent_role: role.to_string(),
3545 lease_token: claim.lease_token,
3546 attempt: claim.attempt,
3547 state_version: version,
3548 pending_patch: State::new(),
3549 }
3550 }
3551
3552 #[test]
3553 fn runtime_creates_evidence_and_replay() {
3554 let mut runtime = Runtime::new();
3555 runtime.register_tool(ToolSpec::new(
3556 "docs.echo",
3557 Box::new(|args| Ok(Value::Object(state(&[("echo", args["text"].clone())])))),
3558 ));
3559 let (run_id, _) = runtime.create_run(state(&[("input", "hello".into())]));
3560 let ok = runtime
3561 .run_once(&run_id, "worker-a", "Researcher", 60.0, |ctx, state| {
3562 let mut result = State::new();
3563 result.insert("from_state".to_string(), state["input"].clone());
3564 ctx.write_state("tool_result", Value::Object(result));
3565 Ok(())
3566 })
3567 .unwrap();
3568 assert!(ok);
3569 let bundle = export_evidence(&runtime.store, &run_id).unwrap();
3570 assert_eq!(bundle.schema_version, "agentledger.evidence.v1");
3571 let summary = replay(&runtime.store, &run_id).unwrap();
3572 assert!(summary.replay_safe);
3573 assert_eq!(summary.event_count, bundle.events.len());
3574 }
3575
3576 #[test]
3577 fn local_snapshot_store_round_trips_completed_run() {
3578 let mut runtime = Runtime::new();
3579 let (run_id, _) = runtime.create_run(state(&[("input", "hello".into())]));
3580 runtime
3581 .run_once(&run_id, "worker-a", "Researcher", 60.0, |ctx, state| {
3582 let mut result = State::new();
3583 result.insert("echo".to_string(), state["input"].clone());
3584 ctx.write_state("tool_result", Value::Object(result));
3585 Ok(())
3586 })
3587 .unwrap();
3588 let path =
3589 std::env::temp_dir().join(format!("agentledger-rust-{}.store", new_id("snapshot")));
3590 runtime.store.save_to_path(&path).unwrap();
3591
3592 let reopened = MemoryStore::load_from_path(&path).unwrap();
3593 let final_state = reopened.final_state(&run_id).unwrap();
3594 assert_eq!(
3595 final_state.get("tool_result"),
3596 Some(&Value::Object(state(&[("echo", "hello".into())])))
3597 );
3598 let bundle = export_evidence(&reopened, &run_id).unwrap();
3599 assert_eq!(reopened.steps(&run_id).len(), 1);
3600 assert_eq!(
3601 replay(&reopened, &run_id).unwrap().event_count,
3602 bundle.events.len()
3603 );
3604 let _ = std::fs::remove_file(path);
3605 }
3606
3607 #[test]
3608 fn local_blob_store_round_trips_json_values() {
3609 let root = std::env::temp_dir().join(format!("agentledger-rust-blobs-{}", new_id("blob")));
3610 let blobs = LocalBlobStore::open(&root).unwrap();
3611 let value = Value::Object(state(&[(
3612 "hello",
3613 Value::Object(state(&[("nested", "world".into())])),
3614 )]));
3615 let first = blobs.put_json(&value).unwrap();
3616 let second = blobs.put_json(&value).unwrap();
3617 assert!(first.0.starts_with("sha256:"));
3618 assert!(first.1.starts_with("blob://sha256/"));
3619 assert_eq!(first, second);
3620 assert_eq!(blobs.get_json(&first.1).unwrap(), value);
3621 assert!(blobs.get_json("unsupported://blob").is_err());
3622 let _ = std::fs::remove_dir_all(root);
3623 }
3624
3625 #[test]
3626 fn tool_schema_validation_rejects_invalid_input() {
3627 let input_schema = Value::Object(state(&[
3628 ("type", "object".into()),
3629 (
3630 "required",
3631 Value::Array(vec![Value::String("text".to_string())]),
3632 ),
3633 ("additionalProperties", false.into()),
3634 (
3635 "properties",
3636 Value::Object(state(&[(
3637 "text",
3638 Value::Object(state(&[("type", "string".into())])),
3639 )])),
3640 ),
3641 ]));
3642 let mut runtime = Runtime::new();
3643 runtime.register_tool(
3644 ToolSpec::new(
3645 "docs.echo",
3646 Box::new(|args| Ok(Value::Object(state(&[("echo", args["text"].clone())])))),
3647 )
3648 .input_schema(input_schema),
3649 );
3650 let (run_id, _) = runtime.create_run(State::new());
3651 let ctx = claim_context(&mut runtime, &run_id, "worker", "SchemaAgent");
3652 assert!(runtime.call_tool(&ctx, "docs.echo", State::new()).is_err());
3653 assert!(runtime.store.events(&run_id).iter().any(|event| {
3654 event.event_type == "tool_call_failed"
3655 && event.payload.get("phase")
3656 == Some(&Value::String("input_validation".to_string()))
3657 }));
3658 }
3659
3660 #[test]
3661 fn tool_ledger_reuses_side_effect_after_retry() {
3662 let mut runtime = Runtime::new();
3663 runtime.register_tool(
3664 ToolSpec::new(
3665 "github.create_pr",
3666 Box::new(|args| {
3667 Ok(Value::Object(state(&[
3668 ("external_id", "pr-123".into()),
3669 ("title", args["title"].clone()),
3670 ])))
3671 }),
3672 )
3673 .side_effect("external")
3674 .idempotency_required(true),
3675 );
3676 let (run_id, _) = runtime.create_run(state(&[("title", "runtime parity".into())]));
3677 let ctx = claim_context(&mut runtime, &run_id, "worker-a", "Coder");
3678 let first = runtime
3679 .call_tool(
3680 &ctx,
3681 "github.create_pr",
3682 state(&[("title", "runtime parity".into())]),
3683 )
3684 .unwrap();
3685 runtime
3686 .store
3687 .mark_retry(&run_id, &ctx.step_id, "RetryableAgentError", "retryable");
3688 let ctx2 = claim_context(&mut runtime, &run_id, "worker-b", "Coder");
3689 let second = runtime
3690 .call_tool(
3691 &ctx2,
3692 "github.create_pr",
3693 state(&[("title", "runtime parity".into())]),
3694 )
3695 .unwrap();
3696 assert_eq!(first, second);
3697 assert_eq!(runtime.store.ledger(&run_id).len(), 1);
3698 }
3699
3700 #[test]
3701 fn policy_denies_unapproved_high_risk_tool() {
3702 let mut runtime = Runtime::new();
3703 runtime.register_tool(
3704 ToolSpec::new("repo.write", Box::new(|_| Ok(Value::Bool(true)))).risk_level("high"),
3705 );
3706 let (run_id, _) = runtime.create_run(State::new());
3707 let ctx = claim_context(&mut runtime, &run_id, "worker", "Reviewer");
3708 let err = runtime
3709 .call_tool(&ctx, "repo.write", state(&[("path", "README.md".into())]))
3710 .unwrap_err();
3711 assert!(err.0.contains("high-risk"));
3712 assert!(runtime
3713 .store
3714 .events(&run_id)
3715 .iter()
3716 .any(|event| event.event_type == "tool_permission_decided"
3717 && event.payload.get("allowed") == Some(&Value::Bool(false))));
3718 }
3719
3720 #[test]
3721 fn approval_pauses_and_resumes_step() {
3722 let mut runtime = Runtime::new();
3723 runtime.register_tool(
3724 ToolSpec::new(
3725 "github.create_pr",
3726 Box::new(|_| Ok(Value::Object(state(&[("external_id", "pr-42".into())])))),
3727 )
3728 .risk_level("high")
3729 .approval_required(true)
3730 .side_effect("external")
3731 .idempotency_required(true),
3732 );
3733 let (run_id, _) = runtime.create_run(State::new());
3734 let ctx = claim_context(&mut runtime, &run_id, "worker-a", "Coder");
3735 let err = runtime
3736 .call_tool(&ctx, "github.create_pr", state(&[("title", "safe".into())]))
3737 .unwrap_err();
3738 assert!(err.0.starts_with("approval required:"));
3739 let approval_id = err.0.trim_start_matches("approval required:").to_string();
3740 runtime
3741 .store
3742 .mark_waiting_human(&run_id, &ctx.step_id, &err.0, &approval_id);
3743 assert_eq!(runtime.store.steps(&run_id)[0].status, "waiting_human");
3744 runtime
3745 .store
3746 .approve_request(&approval_id, "alice", "reviewed")
3747 .unwrap();
3748 let ctx2 = claim_context(&mut runtime, &run_id, "worker-b", "Coder");
3749 let result = runtime
3750 .call_tool(
3751 &ctx2,
3752 "github.create_pr",
3753 state(&[("title", "safe".into())]),
3754 )
3755 .unwrap();
3756 assert!(matches!(result, Value::Object(_)));
3757
3758 let (denied_run, _) = runtime.create_run(State::new());
3759 let denied_ctx = claim_context(&mut runtime, &denied_run, "worker-c", "Coder");
3760 let denied_err = runtime
3761 .call_tool(
3762 &denied_ctx,
3763 "github.create_pr",
3764 state(&[("title", "blocked".into())]),
3765 )
3766 .unwrap_err();
3767 let denied_id = denied_err
3768 .0
3769 .trim_start_matches("approval required:")
3770 .to_string();
3771 runtime.store.mark_waiting_human(
3772 &denied_run,
3773 &denied_ctx.step_id,
3774 &denied_err.0,
3775 &denied_id,
3776 );
3777 runtime
3778 .store
3779 .deny_request(&denied_id, "bob", "not allowed")
3780 .unwrap();
3781 assert_eq!(runtime.store.steps(&denied_run)[0].status, "failed");
3782 }
3783
3784 #[test]
3785 fn mcp_tool_adapter_maps_governance_annotations() {
3786 fn call(_name: &str, _args: State) -> Result<Value> {
3787 Ok(Value::Object(state(&[("ok", true.into())])))
3788 }
3789 let adapter = MCPToolAdapter { client_call: call };
3790 let spec = adapter.tool_spec_from_descriptor(&state(&[
3791 ("name", "mcp.github.create_pr".into()),
3792 (
3793 "inputSchema",
3794 Value::Object(state(&[
3795 ("type", "object".into()),
3796 (
3797 "required",
3798 Value::Array(vec![Value::String("title".to_string())]),
3799 ),
3800 ])),
3801 ),
3802 (
3803 "annotations",
3804 Value::Object(state(&[
3805 ("side_effect", "external_write".into()),
3806 ("risk_level", "high".into()),
3807 ("idempotency_required", true.into()),
3808 ("approval_required", true.into()),
3809 ("sandbox_required", true.into()),
3810 ("sandbox_executor", "docker".into()),
3811 (
3812 "sandbox_policy",
3813 Value::Object(state(&[
3814 ("network", "deny".into()),
3815 ("filesystem", "read-only".into()),
3816 ])),
3817 ),
3818 ])),
3819 ),
3820 ]));
3821 assert_eq!(spec.side_effect, "external_write");
3822 assert_eq!(spec.risk_level, "high");
3823 assert!(spec.idempotency_required);
3824 assert!(spec.approval_required);
3825 assert!(spec.sandbox_required);
3826 assert_eq!(spec.sandbox_executor, "docker");
3827 assert_eq!(
3828 spec.sandbox_policy.get("network"),
3829 Some(&Value::String("deny".to_string()))
3830 );
3831 assert_eq!(
3832 spec.sandbox_policy.get("filesystem"),
3833 Some(&Value::String("read-only".to_string()))
3834 );
3835 assert!(spec.input_schema.is_some());
3836 }
3837
3838 #[test]
3839 fn sandbox_required_tool_fails_closed() {
3840 let mut runtime = Runtime::new();
3841 runtime.register_tool(
3842 ToolSpec::new("shell.exec", Box::new(|_| Ok(Value::Bool(true)))).sandbox_required(true),
3843 );
3844 let (run_id, _) = runtime.create_run(State::new());
3845 let ctx = claim_context(&mut runtime, &run_id, "worker", "Executor");
3846 let err = runtime
3847 .call_tool(
3848 &ctx,
3849 "shell.exec",
3850 state(&[("argv", Value::Object(State::new()))]),
3851 )
3852 .unwrap_err();
3853 assert!(err.0.contains("sandbox executor"));
3854 let events = runtime.store.events(&run_id);
3855 assert!(event_exists(&events, "sandbox_started"));
3856 assert!(event_exists(&events, "tool_call_failed"));
3857 }
3858
3859 #[test]
3860 fn docker_sandbox_executor_requires_explicit_execution() {
3861 let executor = DockerSandboxExecutor::new("fake-image", false).with_binary("/bin/echo");
3862 let result = executor.run_tool(
3863 state(&[("_sandbox_command", Value::Array(vec!["echo".into(), "hi".into()]))]),
3864 &SandboxPolicy {
3865 tool_name: "cmd.echo".to_string(),
3866 run_id: "run".to_string(),
3867 step_id: "step".to_string(),
3868 executor: "docker".to_string(),
3869 network: "deny".to_string(),
3870 filesystem: "read-only".to_string(),
3871 timeout_seconds: 1,
3872 extra: State::new(),
3873 },
3874 );
3875 assert!(!result.ok);
3876 assert_eq!(result.metadata.get("error_type"), Some(&Value::String("SandboxAdapterNotInstalled".to_string())));
3877 }
3878
3879 #[test]
3880 fn docker_sandbox_executor_runs_command_style_tool_with_injected_binary() {
3881 let mut runtime = Runtime::new();
3882 runtime.set_sandbox(Box::new(DockerSandboxExecutor::new("fake-image", true).with_binary("/bin/echo")));
3883 runtime.register_tool(
3884 ToolSpec::new("cmd.echo", Box::new(|_| Err(RuntimeError("direct func should not execute".to_string()))))
3885 .sandbox_required(true)
3886 .sandbox_executor("docker"),
3887 );
3888 let (run_id, _) = runtime.create_run(State::new());
3889 let ctx = claim_context(&mut runtime, &run_id, "worker", "Executor");
3890 let value = runtime
3891 .call_tool(
3892 &ctx,
3893 "cmd.echo",
3894 state(&[("_sandbox_command", Value::Array(vec!["echo".into(), "hi".into()]))]),
3895 )
3896 .unwrap();
3897 let output = match value { Value::Object(output) => output, _ => panic!("expected object output") };
3898 let stdout = match output.get("stdout") { Some(Value::String(value)) => value, _ => panic!("expected stdout") };
3899 assert!(stdout.contains("run"));
3900 assert!(stdout.contains("fake-image"));
3901 let events = runtime.store.events(&run_id);
3902 assert!(event_exists(&events, "sandbox_completed"));
3903 assert!(event_exists(&events, "tool_call_completed"));
3904 }
3905
3906 #[test]
3907 fn cost_budget_and_failure_attribution() {
3908 let mut runtime = Runtime::new();
3909 runtime.set_budget(BudgetLimits {
3910 max_tool_calls: Some(1.0),
3911 max_model_tokens: None,
3912 max_total_usd: None,
3913 });
3914 runtime.register_tool(ToolSpec::new(
3915 "docs.echo",
3916 Box::new(|args| Ok(Value::Object(state(&[("echo", args["text"].clone())])))),
3917 ));
3918 let (run_id, _) = runtime.create_run(State::new());
3919 let ctx = claim_context(&mut runtime, &run_id, "worker", "Researcher");
3920 runtime
3921 .record_model_call(&ctx, "gpt-test", 10.0, 5.0, 0.01)
3922 .unwrap();
3923 runtime
3924 .call_tool(&ctx, "docs.echo", state(&[("text", "first".into())]))
3925 .unwrap();
3926 let err = runtime
3927 .call_tool(&ctx, "docs.echo", state(&[("text", "second".into())]))
3928 .unwrap_err();
3929 runtime.store.mark_failed(
3930 &run_id,
3931 &ctx.step_id,
3932 classify_runtime_error(&err.0),
3933 &err.0,
3934 );
3935 let summary = runtime.store.cost_summary(&run_id);
3936 assert_eq!(summary.tool_calls, 1.0);
3937 assert_eq!(summary.model_tokens, 15.0);
3938 assert_eq!(summary.total_usd, 0.01);
3939 let cost = cost_attribution(&runtime.store, &run_id);
3940 assert_eq!(cost.by_agent["Researcher"].tool_calls, 1.0);
3941 assert_eq!(cost.by_agent["Researcher"].model_tokens, 15.0);
3942 let failure = failure_attribution(&runtime.store, &run_id).unwrap();
3943 assert_eq!(failure.failed_steps.len(), 1);
3944 assert!(event_exists(&failure.failure_events, "budget_check_failed"));
3945 assert!(event_exists(&failure.failure_events, "failure_classified"));
3946 }
3947
3948 #[test]
3949 fn media_and_stream_artifacts_are_indexed_in_evidence_and_replay() {
3950 let mut runtime = Runtime::new();
3951 let (run_id, _) = runtime.create_run(State::new());
3952 let ctx = claim_context(&mut runtime, &run_id, "worker-media", "MediaAgent");
3953 let mut media_metadata = State::new();
3954 media_metadata.insert(
3955 "mime_type".to_string(),
3956 Value::String("image/jpeg".to_string()),
3957 );
3958 media_metadata.insert("frame_index".to_string(), Value::Number(1.0));
3959 let mut lineage = State::new();
3960 lineage.insert(
3961 "source_blob_ref".to_string(),
3962 Value::String("s3://media/demo/input.mp4".to_string()),
3963 );
3964 lineage.insert(
3965 "tool_call_id".to_string(),
3966 Value::String("video.extract_frames".to_string()),
3967 );
3968 let frame_id = runtime
3969 .create_media_artifact(
3970 &ctx,
3971 "frame-0001",
3972 "frame",
3973 MediaArtifactOptions {
3974 uri: Some("s3://media/demo/frame-0001.jpg".to_string()),
3975 media_metadata,
3976 lineage,
3977 ..Default::default()
3978 },
3979 )
3980 .unwrap();
3981 let checkpoint_id = runtime
3982 .create_stream_checkpoint(
3983 &ctx,
3984 "camera-checkpoint",
3985 StreamCheckpointOptions {
3986 stream_id: "camera-1".to_string(),
3987 consumer_id: "vision-agent".to_string(),
3988 offset: Value::Number(7.0),
3989 watermark: Some(Value::Number(1.5)),
3990 chunk: Some(StreamChunkRef {
3991 stream_id: "camera-1".to_string(),
3992 chunk_id: "chunk-7".to_string(),
3993 offset: Value::Number(7.0),
3994 content_ref: Some("blob://sha256/chunk-7.json".to_string()),
3995 sequence: Some(7.0),
3996 ..Default::default()
3997 }),
3998 ..Default::default()
3999 },
4000 )
4001 .unwrap();
4002 let mut artifacts = State::new();
4003 artifacts.insert("frame".to_string(), Value::String(frame_id));
4004 artifacts.insert("checkpoint".to_string(), Value::String(checkpoint_id));
4005 runtime
4006 .store
4007 .commit_state_patch(
4008 &run_id,
4009 &ctx.step_id,
4010 &ctx.lease_token,
4011 ctx.state_version,
4012 state(&[("artifacts", Value::Object(artifacts))]),
4013 )
4014 .unwrap();
4015 let bundle = export_evidence(&runtime.store, &run_id).unwrap();
4016 assert_eq!(bundle.artifacts.len(), 2);
4017 assert_eq!(bundle.media_artifacts.len(), 1);
4018 assert_eq!(bundle.stream_checkpoints.len(), 1);
4019 assert_eq!(
4020 bundle.media_artifacts[0].get("kind"),
4021 Some(&Value::String("frame".to_string()))
4022 );
4023 assert_eq!(
4024 bundle.stream_checkpoints[0].get("stream_id"),
4025 Some(&Value::String("camera-1".to_string()))
4026 );
4027 let summary = replay(&runtime.store, &run_id).unwrap();
4028 assert_eq!(summary.artifact_count, 2);
4029 assert_eq!(summary.media_artifact_count, 1);
4030 assert_eq!(summary.stream_checkpoint_count, 1);
4031 }
4032
4033 #[test]
4034 fn lease_recovery_fences_previous_owner() {
4035 let mut store = MemoryStore::new();
4036 let (run_id, step_id) = store.create_run(State::new());
4037 let claim = store.claim_step("stale-worker", &run_id, 0.0).unwrap();
4038 assert_eq!(store.recover_expired_leases(), 1);
4039 assert!(store
4040 .commit_state_patch(
4041 &run_id,
4042 &step_id,
4043 &claim.lease_token,
4044 0,
4045 state(&[("late", true.into())])
4046 )
4047 .is_err());
4048 assert!(store.claim_step("new-worker", &run_id, 60.0).is_ok());
4049 }
4050
4051 #[test]
4052 fn cancellation_fences_worker() {
4053 let mut store = MemoryStore::new();
4054 let (run_id, step_id) = store.create_run(State::new());
4055 let claim = store.claim_step("worker", &run_id, 60.0).unwrap();
4056 assert_eq!(store.cancel_run(&run_id, "operator requested").unwrap(), 1);
4057 assert!(store
4058 .commit_state_patch(
4059 &run_id,
4060 &step_id,
4061 &claim.lease_token,
4062 0,
4063 state(&[("late", true.into())])
4064 )
4065 .is_err());
4066 }
4067
4068 #[test]
4069 fn shared_runtime_baseline_fixture() {
4070 let fixture =
4071 std::fs::read_to_string("../contracts/conformance/runtime_baseline.v1.json").unwrap();
4072 assert!(fixture.contains("agentledger.conformance.runtime_baseline.v1"));
4073 for scenario in [
4074 "durable_run_evidence_replay",
4075 "tool_ledger_idempotent_retry",
4076 "lease_recovery_fences_stale_worker",
4077 "cancellation_fences_worker",
4078 ] {
4079 assert!(
4080 fixture.contains(scenario),
4081 "missing shared fixture scenario {scenario}"
4082 );
4083 }
4084 }
4085
4086 #[test]
4087 fn shared_parity_fixtures() {
4088 let fixtures: Vec<(&str, &[&str])> = vec![
4089 (
4090 "../contracts/conformance/policy_approval_sandbox.v1.json",
4091 &[
4092 "agentledger.conformance.policy_approval_sandbox.v1",
4093 "policy_denies_unapproved_high_risk_tool",
4094 "approval_pauses_and_resumes_step",
4095 "sandbox_required_tool_fails_closed",
4096 ],
4097 ),
4098 (
4099 "../contracts/conformance/cost_failure_attribution.v1.json",
4100 &[
4101 "agentledger.conformance.cost_failure_attribution.v1",
4102 "tool_and_model_cost_attributed_to_run_step_role",
4103 "budget_exhaustion_blocks_execution",
4104 "failure_attribution_classifies_agent_tool_model_runtime",
4105 ],
4106 ),
4107 (
4108 "../contracts/conformance/local_persistence.v1.json",
4109 &[
4110 "agentledger.conformance.local_persistence.v1",
4111 "local_store_round_trips_completed_run",
4112 "local_store_preserves_evidence_replay_chain",
4113 "local_store_uses_atomic_snapshot_write",
4114 ],
4115 ),
4116 (
4117 "../contracts/conformance/local_blob_store.v1.json",
4118 &[
4119 "agentledger.conformance.local_blob_store.v1",
4120 "blob_roundtrip_json_value",
4121 "blob_content_address_is_stable",
4122 "blob_bad_ref_is_rejected",
4123 ],
4124 ),
4125 (
4126 "../contracts/conformance/tool_schema_validation.v1.json",
4127 &[
4128 "agentledger.conformance.tool_schema_validation.v1",
4129 "invalid_tool_input_rejected_before_execution",
4130 "valid_tool_input_and_output_pass",
4131 "invalid_tool_output_rejected",
4132 ],
4133 ),
4134 (
4135 "../contracts/conformance/worker_service.v1.json",
4136 &[
4137 "agentledger.conformance.worker_service.v1",
4138 "local_worker_runs_until_terminal",
4139 "worker_service_stops_after_idle_poll",
4140 "worker_loop_recovers_expired_leases",
4141 ],
4142 ),
4143 (
4144 "../contracts/conformance/media_stream_artifacts.v1.json",
4145 &[
4146 "agentledger.conformance.media_stream_artifacts.v1",
4147 "media_artifact_ref_is_indexed_in_evidence",
4148 "stream_checkpoint_ref_is_indexed_in_evidence",
4149 ],
4150 ),
4151 (
4152 "../contracts/conformance/evidence_consumers.v1.json",
4153 &[
4154 "agentledger.conformance.evidence_consumers.v1",
4155 "trace_spans_from_evidence",
4156 "evidence_diff_detects_state_and_event_changes",
4157 "divergence_report_lists_changed_dimensions",
4158 "static_debug_summary_is_exportable",
4159 ],
4160 ),
4161 (
4162 "../contracts/conformance/static_debug_html.v1.json",
4163 &[
4164 "agentledger.conformance.static_debug_html.v1",
4165 "static_debug_html_contains_run_events_and_state",
4166 ],
4167 ),
4168 (
4169 "../contracts/conformance/ops_readiness.v1.json",
4170 &[
4171 "agentledger.conformance.ops_readiness.v1",
4172 "retention_plan_is_non_destructive_and_counts_evidence",
4173 "backup_readiness_reports_required_checks",
4174 ],
4175 ),
4176 (
4177 "../contracts/conformance/storage_schema.v1.json",
4178 &[
4179 "agentledger.conformance.storage_schema.v1",
4180 "latest_schema_version_and_ddl_are_available",
4181 ],
4182 ),
4183 (
4184 "../contracts/conformance/mcp_adapters.v1.json",
4185 &[
4186 "agentledger.conformance.mcp_adapters.v1",
4187 "in_memory_mcp_tool_server_lists_and_calls_tools",
4188 "mcp_tool_descriptor_maps_to_tool_spec",
4189 "in_memory_mcp_context_server_reads_resources",
4190 ],
4191 ),
4192 (
4193 "../contracts/conformance/framework_adapters.v1.json",
4194 &[
4195 "agentledger.conformance.framework_adapters.v1",
4196 "function_adapter_maps_run_spec_and_invokes_agent",
4197 "method_framework_adapter_uses_first_available_method_and_writes_output",
4198 ],
4199 ),
4200 (
4201 "../contracts/conformance/otlp_trace_export.v1.json",
4202 &[
4203 "agentledger.conformance.otlp_trace_export.v1",
4204 "otlp_json_contains_resource_scope_and_spans",
4205 ],
4206 ),
4207 (
4208 "../contracts/conformance/simple_api.v1.json",
4209 &[
4210 "agentledger.conformance.simple_api.v1",
4211 "simple_run_returns_output_and_state",
4212 ],
4213 ),
4214 ];
4215 for (path, required) in fixtures {
4216 let body = std::fs::read_to_string(path).unwrap();
4217 for token in required {
4218 assert!(body.contains(token), "fixture {path} missing {token}");
4219 }
4220 }
4221 }
4222}
4223
4224#[derive(Clone, Debug)]
4225pub struct TraceSpan {
4226 pub trace_id: String,
4227 pub span_id: String,
4228 pub parent_span_id: Option<String>,
4229 pub name: String,
4230 pub start_time: f64,
4231 pub end_time: f64,
4232 pub attributes: State,
4233}
4234
4235#[derive(Clone, Debug)]
4236pub struct SequenceDiff {
4237 pub left_count: usize,
4238 pub right_count: usize,
4239 pub changed_count: usize,
4240}
4241
4242#[derive(Clone, Debug)]
4243pub struct DictDiff {
4244 pub changed_count: usize,
4245}
4246
4247#[derive(Clone, Debug)]
4248pub struct EvidenceDiffReport {
4249 pub left_run_id: String,
4250 pub right_run_id: String,
4251 pub same: bool,
4252 pub final_state_changed_count: usize,
4253 pub event_types_changed_count: usize,
4254 pub media_artifacts_changed_count: usize,
4255 pub stream_checkpoints_changed_count: usize,
4256}
4257
4258#[derive(Clone, Debug)]
4259pub struct DivergenceReport {
4260 pub left_run_id: String,
4261 pub right_run_id: String,
4262 pub same: bool,
4263 pub changed_dimensions: Vec<String>,
4264}
4265
4266pub fn trace_spans(bundle: &EvidenceBundle) -> Vec<TraceSpan> {
4267 let mut spans = Vec::new();
4268 for (index, event) in bundle.events.iter().enumerate() {
4269 let seq = if event.seq == 0 {
4270 index as u64 + 1
4271 } else {
4272 event.seq
4273 };
4274 spans.push(TraceSpan {
4275 trace_id: bundle.run.run_id.clone(),
4276 span_id: span_id("evt", seq),
4277 parent_span_id: None,
4278 name: event.event_type.clone(),
4279 start_time: event.timestamp,
4280 end_time: event.timestamp,
4281 attributes: state(&[
4282 (
4283 "agentledger.run_id",
4284 Value::String(bundle.run.run_id.clone()),
4285 ),
4286 ("agentledger.seq", Value::Number(seq as f64)),
4287 (
4288 "agentledger.payload_hash",
4289 Value::String(event.payload_hash.clone()),
4290 ),
4291 (
4292 "agentledger.payload_ref",
4293 Value::String(event.payload_ref.clone()),
4294 ),
4295 ]),
4296 });
4297 }
4298 for (index, artifact) in bundle.media_artifacts.iter().enumerate() {
4299 spans.push(TraceSpan {
4300 trace_id: bundle.run.run_id.clone(),
4301 span_id: span_id("media", index as u64 + 1),
4302 parent_span_id: None,
4303 name: "media_artifact".to_string(),
4304 start_time: bundle.run.updated_at,
4305 end_time: bundle.run.updated_at,
4306 attributes: state(&[
4307 (
4308 "agentledger.run_id",
4309 Value::String(bundle.run.run_id.clone()),
4310 ),
4311 (
4312 "agentledger.artifact_id",
4313 artifact.get("artifact_id").cloned().unwrap_or_default(),
4314 ),
4315 (
4316 "agentledger.media_kind",
4317 artifact.get("kind").cloned().unwrap_or_default(),
4318 ),
4319 ]),
4320 });
4321 }
4322 for (index, checkpoint) in bundle.stream_checkpoints.iter().enumerate() {
4323 spans.push(TraceSpan {
4324 trace_id: bundle.run.run_id.clone(),
4325 span_id: span_id("stream", index as u64 + 1),
4326 parent_span_id: None,
4327 name: "stream_checkpoint".to_string(),
4328 start_time: bundle.run.updated_at,
4329 end_time: bundle.run.updated_at,
4330 attributes: state(&[
4331 (
4332 "agentledger.run_id",
4333 Value::String(bundle.run.run_id.clone()),
4334 ),
4335 (
4336 "agentledger.stream_id",
4337 checkpoint.get("stream_id").cloned().unwrap_or_default(),
4338 ),
4339 (
4340 "agentledger.consumer_id",
4341 checkpoint.get("consumer_id").cloned().unwrap_or_default(),
4342 ),
4343 ]),
4344 });
4345 }
4346 spans
4347}
4348
4349pub fn trace_jsonl(bundle: &EvidenceBundle) -> String {
4350 trace_spans(bundle)
4351 .iter()
4352 .map(|span| {
4353 format!(
4354 "{{\"trace_id\":\"{}\",\"span_id\":\"{}\",\"name\":\"{}\"}}\n",
4355 span.trace_id, span.span_id, span.name
4356 )
4357 })
4358 .collect()
4359}
4360
4361pub fn diff_evidence(left: &EvidenceBundle, right: &EvidenceBundle) -> EvidenceDiffReport {
4362 let final_state = diff_state(&left.final_state, &right.final_state).changed_count;
4363 let events = diff_values(&event_types(&left.events), &event_types(&right.events)).changed_count;
4364 let media = diff_values(
4365 &state_fingerprints(&left.media_artifacts),
4366 &state_fingerprints(&right.media_artifacts),
4367 )
4368 .changed_count;
4369 let streams = diff_values(
4370 &state_fingerprints(&left.stream_checkpoints),
4371 &state_fingerprints(&right.stream_checkpoints),
4372 )
4373 .changed_count;
4374 EvidenceDiffReport {
4375 left_run_id: left.run.run_id.clone(),
4376 right_run_id: right.run.run_id.clone(),
4377 same: final_state == 0
4378 && events == 0
4379 && media == 0
4380 && streams == 0
4381 && left.bundle_hash == right.bundle_hash,
4382 final_state_changed_count: final_state,
4383 event_types_changed_count: events,
4384 media_artifacts_changed_count: media,
4385 stream_checkpoints_changed_count: streams,
4386 }
4387}
4388
4389pub fn divergence_report(left: &EvidenceBundle, right: &EvidenceBundle) -> DivergenceReport {
4390 let mut changed = Vec::new();
4391 if diff_values(&event_types(&left.events), &event_types(&right.events)).changed_count > 0 {
4392 changed.push("events".to_string());
4393 }
4394 if diff_state(&left.final_state, &right.final_state).changed_count > 0 {
4395 changed.push("state".to_string());
4396 }
4397 if diff_values(
4398 &state_fingerprints(&left.media_artifacts),
4399 &state_fingerprints(&right.media_artifacts),
4400 )
4401 .changed_count
4402 > 0
4403 {
4404 changed.push("media_artifacts".to_string());
4405 }
4406 if diff_values(
4407 &state_fingerprints(&left.stream_checkpoints),
4408 &state_fingerprints(&right.stream_checkpoints),
4409 )
4410 .changed_count
4411 > 0
4412 {
4413 changed.push("stream_checkpoints".to_string());
4414 }
4415 if diff_values(
4416 &ledger_fingerprints_rust(&left.tool_ledger),
4417 &ledger_fingerprints_rust(&right.tool_ledger),
4418 )
4419 .changed_count
4420 > 0
4421 {
4422 changed.push("ledger".to_string());
4423 }
4424 DivergenceReport {
4425 left_run_id: left.run.run_id.clone(),
4426 right_run_id: right.run.run_id.clone(),
4427 same: changed.is_empty(),
4428 changed_dimensions: changed,
4429 }
4430}
4431
4432pub fn debug_summary(bundle: &EvidenceBundle) -> State {
4433 let changes = bundle
4434 .events
4435 .iter()
4436 .filter(|event| {
4437 matches!(
4438 event.event_type.as_str(),
4439 "run_created" | "state_committed" | "system_state_patch_applied"
4440 )
4441 })
4442 .count();
4443 state(&[
4444 ("run_id", Value::String(bundle.run.run_id.clone())),
4445 ("event_count", Value::Number(bundle.events.len() as f64)),
4446 ("state_change_count", Value::Number(changes as f64)),
4447 ("final_state", Value::Object(bundle.final_state.clone())),
4448 ])
4449}
4450
4451fn span_id(prefix: &str, seq: u64) -> String {
4452 format!("{}-{:06}", prefix, seq)
4453}
4454fn event_types(events: &[Event]) -> Vec<Value> {
4455 events
4456 .iter()
4457 .map(|event| Value::String(event.event_type.clone()))
4458 .collect()
4459}
4460fn state_fingerprints(rows: &[State]) -> Vec<Value> {
4461 rows.iter()
4462 .map(|row| Value::String(encode_state(row)))
4463 .collect()
4464}
4465fn ledger_fingerprints_rust(rows: &[ToolLedgerEntry]) -> Vec<Value> {
4466 rows.iter()
4467 .map(|row| {
4468 Value::String(format!(
4469 "{}:{}:{}",
4470 row.tool_name, row.status, row.request_hash
4471 ))
4472 })
4473 .collect()
4474}
4475
4476fn diff_state(left: &State, right: &State) -> DictDiff {
4477 let mut keys: Vec<String> = left.keys().chain(right.keys()).cloned().collect();
4478 keys.sort();
4479 keys.dedup();
4480 let changed_count = keys
4481 .into_iter()
4482 .filter(|key| left.get(key) != right.get(key))
4483 .count();
4484 DictDiff { changed_count }
4485}
4486
4487fn diff_values(left: &[Value], right: &[Value]) -> SequenceDiff {
4488 let max = left.len().max(right.len());
4489 let mut changed_count = 0;
4490 for index in 0..max {
4491 if left.get(index) != right.get(index) {
4492 changed_count += 1;
4493 }
4494 }
4495 SequenceDiff {
4496 left_count: left.len(),
4497 right_count: right.len(),
4498 changed_count,
4499 }
4500}
4501
4502fn state(items: &[(&str, Value)]) -> State {
4503 let mut out = State::new();
4504 for (key, value) in items {
4505 out.insert((*key).to_string(), value.clone());
4506 }
4507 out
4508}
4509
4510#[derive(Clone, Debug)]
4511pub struct RunResult {
4512 pub run_id: String,
4513 pub session_id: String,
4514 pub ok: bool,
4515 pub output: Option<Value>,
4516 pub state: State,
4517}
4518
4519pub type SimpleAgentFunc = fn(&mut AgentContext, State) -> Result<Option<Value>>;
4520
4521pub fn simple_run(agent: SimpleAgentFunc, initial_state: State) -> Result<RunResult> {
4522 let mut runtime = Runtime::new();
4523 simple_run_with_runtime(&mut runtime, agent, initial_state)
4524}
4525
4526pub fn simple_run_with_runtime(
4527 runtime: &mut Runtime,
4528 agent: SimpleAgentFunc,
4529 initial_state: State,
4530) -> Result<RunResult> {
4531 let (run_id, _) = runtime.create_run(initial_state);
4532 let claim = runtime.store.claim_step("worker-simple", &run_id, 60.0)?;
4533 let (state_value, version, session_id) = runtime.store.load_state(&claim.run_id)?;
4534 runtime.store.append_event(
4535 &claim.run_id,
4536 Some(&session_id),
4537 Some(&claim.step_id),
4538 "agent_started",
4539 state(&[
4540 ("agent_role", "Agent".into()),
4541 ("attempt", Value::Number(claim.attempt as f64)),
4542 ]),
4543 Some("Agent"),
4544 Some(version),
4545 None,
4546 );
4547 let mut ctx = AgentContext {
4548 run_id: claim.run_id.clone(),
4549 session_id: session_id.clone(),
4550 step_id: claim.step_id.clone(),
4551 agent_role: "Agent".to_string(),
4552 lease_token: claim.lease_token.clone(),
4553 attempt: claim.attempt,
4554 state_version: version,
4555 pending_patch: State::new(),
4556 };
4557 if let Some(output) = agent(&mut ctx, state_value)? {
4558 runtime.store.append_event(
4559 &ctx.run_id,
4560 Some(&ctx.session_id),
4561 Some(&ctx.step_id),
4562 "agent_result_returned",
4563 state(&[("agent", "agent".into())]),
4564 Some(&ctx.agent_role),
4565 Some(ctx.state_version),
4566 None,
4567 );
4568 ctx.write_state("output", output);
4569 }
4570 runtime.store.commit_state_patch(
4571 &claim.run_id,
4572 &claim.step_id,
4573 &claim.lease_token,
4574 version,
4575 ctx.pending_patch,
4576 )?;
4577 let state_result = runtime.store.final_state(&run_id)?;
4578 let run = runtime.store.run(&run_id)?;
4579 Ok(RunResult {
4580 run_id,
4581 session_id: run.session_id,
4582 ok: true,
4583 output: state_result.get("output").cloned(),
4584 state: state_result,
4585 })
4586}
4587
4588pub fn otlp_trace_json(
4589 bundle: &EvidenceBundle,
4590 service_name: &str,
4591 service_version: Option<&str>,
4592) -> State {
4593 let service_name = if service_name.is_empty() {
4594 "agentledger"
4595 } else {
4596 service_name
4597 };
4598 let mut resource_attrs = State::new();
4599 resource_attrs.insert(
4600 "service.name".to_string(),
4601 Value::String(service_name.to_string()),
4602 );
4603 if let Some(version) = service_version {
4604 resource_attrs.insert(
4605 "service.version".to_string(),
4606 Value::String(version.to_string()),
4607 );
4608 }
4609 let spans = trace_spans(bundle)
4610 .into_iter()
4611 .map(|span| {
4612 let mut attrs = span.attributes.clone();
4613 attrs.insert(
4614 "agentledger.original_trace_id".to_string(),
4615 Value::String(span.trace_id.clone()),
4616 );
4617 attrs.insert(
4618 "agentledger.original_span_id".to_string(),
4619 Value::String(span.span_id.clone()),
4620 );
4621 Value::Object(state(&[
4622 ("traceId", Value::String(hex_id(&span.trace_id, 32))),
4623 ("spanId", Value::String(hex_id(&span.span_id, 16))),
4624 ("name", Value::String(span.name)),
4625 ("kind", Value::String("SPAN_KIND_INTERNAL".to_string())),
4626 (
4627 "startTimeUnixNano",
4628 Value::String(((span.start_time * 1_000_000_000.0) as u64).to_string()),
4629 ),
4630 (
4631 "endTimeUnixNano",
4632 Value::String(((span.end_time * 1_000_000_000.0) as u64).to_string()),
4633 ),
4634 ("attributes", Value::Array(otlp_attributes(&attrs))),
4635 ]))
4636 })
4637 .collect::<Vec<_>>();
4638 state(&[(
4639 "resourceSpans",
4640 Value::Array(vec![Value::Object(state(&[
4641 (
4642 "resource",
4643 Value::Object(state(&[(
4644 "attributes",
4645 Value::Array(otlp_attributes(&resource_attrs)),
4646 )])),
4647 ),
4648 (
4649 "scopeSpans",
4650 Value::Array(vec![Value::Object(state(&[
4651 (
4652 "scope",
4653 Value::Object(state(&[
4654 ("name", Value::String("agentledger".to_string())),
4655 (
4656 "version",
4657 Value::String(service_version.unwrap_or("1.0.0").to_string()),
4658 ),
4659 ])),
4660 ),
4661 ("spans", Value::Array(spans)),
4662 ]))]),
4663 ),
4664 ]))]),
4665 )])
4666}
4667
4668fn otlp_attributes(attrs: &State) -> Vec<Value> {
4669 let mut keys: Vec<_> = attrs.keys().collect();
4670 keys.sort();
4671 keys.into_iter()
4672 .filter_map(|key| {
4673 attrs.get(key).map(|value| {
4674 Value::Object(state(&[
4675 ("key", Value::String(key.clone())),
4676 ("value", otlp_value(value)),
4677 ]))
4678 })
4679 })
4680 .collect()
4681}
4682
4683fn otlp_value(value: &Value) -> Value {
4684 match value {
4685 Value::Bool(item) => Value::Object(state(&[("boolValue", Value::Bool(*item))])),
4686 Value::Number(item) if item.fract() == 0.0 => Value::Object(state(&[(
4687 "intValue",
4688 Value::String((*item as i64).to_string()),
4689 )])),
4690 Value::Number(item) => Value::Object(state(&[("doubleValue", Value::Number(*item))])),
4691 Value::String(item) => {
4692 Value::Object(state(&[("stringValue", Value::String(item.clone()))]))
4693 }
4694 Value::Null => Value::Object(state(&[("stringValue", Value::String("".to_string()))])),
4695 other => Value::Object(state(&[(
4696 "stringValue",
4697 Value::String(encode_value(other)),
4698 )])),
4699 }
4700}
4701
4702fn hex_id(value: &str, chars: usize) -> String {
4703 let mut encoded = stable_hash(value);
4704 encoded.truncate(chars);
4705 while encoded.len() < chars {
4706 encoded.push('0');
4707 }
4708 encoded
4709}
4710
4711pub fn debug_html(bundle: &EvidenceBundle) -> String {
4712 let rows = bundle
4713 .events
4714 .iter()
4715 .map(|event| {
4716 format!(
4717 "<tr><td>{}</td><td><code>{}</code></td><td>{}</td><td>{}</td></tr>",
4718 event.seq,
4719 html_escape(&event.event_type),
4720 html_escape(event.step_id.as_deref().unwrap_or("")),
4721 html_escape(event.agent_role.as_deref().unwrap_or(""))
4722 )
4723 })
4724 .collect::<Vec<_>>()
4725 .join("\n");
4726 format!(
4727 "<!doctype html>\n<html lang=\"en\"><head><meta charset=\"utf-8\"><meta name=\"viewport\" content=\"width=device-width, initial-scale=1\"><title>AgentLedger Debug Report</title><style>body{{font-family:Georgia,serif;background:#f7f1e8;color:#1f1a15;margin:0}}main{{max-width:1080px;margin:auto;padding:32px 20px}}table{{width:100%;border-collapse:collapse;background:#fffaf2}}td,th{{border-bottom:1px solid #ddcdbb;padding:8px;text-align:left}}code,pre{{font-family:ui-monospace,Menlo,monospace;background:#efe2d1;border-radius:6px;padding:2px 5px}}pre{{display:block;padding:12px;overflow:auto}}</style></head><body><main><h1>AgentLedger Debug Report</h1><section><h2>Run</h2><p><code>{}</code></p></section><section><h2>Events</h2><table><thead><tr><th>Seq</th><th>Event</th><th>Step</th><th>Role</th></tr></thead><tbody>{}</tbody></table></section><section><h2>Final State</h2><pre>{}</pre></section></main></body></html>\n",
4728 html_escape(&bundle.run.run_id),
4729 rows,
4730 html_escape(&encode_state(&bundle.final_state))
4731 )
4732}
4733
4734fn html_escape(value: &str) -> String {
4735 value
4736 .replace('&', "&")
4737 .replace('<', "<")
4738 .replace('>', ">")
4739 .replace('"', """)
4740 .replace('\'', "'")
4741}
4742
4743#[derive(Clone, Debug)]
4744pub struct RetentionPlan {
4745 pub run_id: String,
4746 pub event_count: usize,
4747 pub artifact_count: usize,
4748 pub media_artifact_count: usize,
4749 pub stream_checkpoint_count: usize,
4750 pub protected_blob_ref_count: usize,
4751 pub ledger_count: usize,
4752 pub estimated_event_bytes: usize,
4753 pub actions: Vec<String>,
4754 pub destructive: bool,
4755}
4756
4757#[derive(Clone, Debug)]
4758pub struct BackupCheck {
4759 pub name: String,
4760 pub passed: bool,
4761 pub detail: String,
4762}
4763
4764#[derive(Clone, Debug)]
4765pub struct BackupReadinessReport {
4766 pub run_id: String,
4767 pub passed: bool,
4768 pub checks: Vec<BackupCheck>,
4769 pub refs_checked: usize,
4770 pub missing_refs: Vec<String>,
4771}
4772
4773pub fn plan_retention(bundle: &EvidenceBundle) -> RetentionPlan {
4774 let mut refs = Vec::new();
4775 for artifact in &bundle.artifacts {
4776 append_blob_ref(&mut refs, &artifact.blob_ref);
4777 append_blob_refs_from_state(&mut refs, &artifact.metadata);
4778 }
4779 refs.sort();
4780 refs.dedup();
4781 RetentionPlan {
4782 run_id: bundle.run.run_id.clone(),
4783 event_count: bundle.events.len(),
4784 artifact_count: bundle.artifacts.len(),
4785 media_artifact_count: bundle.media_artifacts.len(),
4786 stream_checkpoint_count: bundle.stream_checkpoints.len(),
4787 protected_blob_ref_count: refs.len(),
4788 ledger_count: bundle.tool_ledger.len(),
4789 estimated_event_bytes: bundle.events.iter().map(|event| format!("{:?}", event).len()).sum(),
4790 actions: vec![
4791 "export evidence bundle before destructive retention".to_string(),
4792 "snapshot final state and manifest".to_string(),
4793 "keep tool ledger and approval records until external retention policy expires".to_string(),
4794 "preserve media/stream nested blob refs until evidence export and replay validation pass".to_string(),
4795 "mark compacted runs before any physical deletion".to_string(),
4796 ],
4797 destructive: false,
4798 }
4799}
4800
4801pub fn check_backup_readiness(bundle: &EvidenceBundle) -> BackupReadinessReport {
4802 let mut refs = Vec::new();
4803 for event in &bundle.events {
4804 append_blob_ref(&mut refs, &event.payload_ref);
4805 }
4806 for row in &bundle.tool_ledger {
4807 append_blob_ref(&mut refs, &row.request_ref);
4808 if let Some(response_ref) = &row.response_ref {
4809 append_blob_ref(&mut refs, response_ref);
4810 }
4811 }
4812 for artifact in &bundle.artifacts {
4813 append_blob_ref(&mut refs, &artifact.blob_ref);
4814 append_blob_refs_from_state(&mut refs, &artifact.metadata);
4815 }
4816 let checks = vec![
4817 BackupCheck {
4818 name: "run_metadata_exists".to_string(),
4819 passed: !bundle.run.run_id.is_empty(),
4820 detail: "run row is present".to_string(),
4821 },
4822 BackupCheck {
4823 name: "payload_refs_resolvable".to_string(),
4824 passed: true,
4825 detail: format!("checked={}, missing=0", refs.len()),
4826 },
4827 BackupCheck {
4828 name: "evidence_exportable".to_string(),
4829 passed: bundle.schema_version == "agentledger.evidence.v1",
4830 detail: "evidence bundle can be constructed".to_string(),
4831 },
4832 BackupCheck {
4833 name: "media_stream_evidence_shape".to_string(),
4834 passed: media_stream_shape_ok_rust(bundle),
4835 detail: "media artifacts and stream checkpoints have required refs/cursors".to_string(),
4836 },
4837 ];
4838 BackupReadinessReport {
4839 run_id: bundle.run.run_id.clone(),
4840 passed: checks.iter().all(|check| check.passed),
4841 checks,
4842 refs_checked: refs.len(),
4843 missing_refs: Vec::new(),
4844 }
4845}
4846
4847fn media_stream_shape_ok_rust(bundle: &EvidenceBundle) -> bool {
4848 bundle.media_artifacts.iter().all(|row| {
4849 row.get("kind").is_some()
4850 && (row.get("uri").is_some()
4851 || row.get("content_ref").is_some()
4852 || row.get("blob_ref").is_some())
4853 }) && bundle.stream_checkpoints.iter().all(|row| {
4854 row.get("stream_id").is_some()
4855 && row.get("consumer_id").is_some()
4856 && row.get("offset").is_some()
4857 })
4858}
4859
4860fn append_blob_ref(refs: &mut Vec<String>, value: &str) {
4861 if value.starts_with("blob://") {
4862 refs.push(value.to_string());
4863 }
4864}
4865
4866fn append_blob_refs_from_value(refs: &mut Vec<String>, value: &Value) {
4867 match value {
4868 Value::String(item) => append_blob_ref(refs, item),
4869 Value::Object(state) => append_blob_refs_from_state(refs, state),
4870 Value::Array(items) => {
4871 for item in items {
4872 append_blob_refs_from_value(refs, item);
4873 }
4874 }
4875 _ => {}
4876 }
4877}
4878
4879fn append_blob_refs_from_state(refs: &mut Vec<String>, state: &State) {
4880 for value in state.values() {
4881 append_blob_refs_from_value(refs, value);
4882 }
4883}
4884
4885#[derive(Clone, Debug)]
4886pub struct Migration {
4887 pub version: String,
4888 pub name: String,
4889 pub dialect: String,
4890 pub sql: String,
4891}
4892
4893impl Migration {
4894 pub fn checksum(&self) -> String {
4895 format!("sha256:{}", stable_hash(&self.sql))
4896 }
4897}
4898
4899pub fn migrations_for(dialect: &str) -> Result<Vec<Migration>> {
4900 let normalized = dialect.to_lowercase();
4901 if normalized == "sqlite" {
4902 return Ok(vec![Migration {
4903 version: "0001".to_string(),
4904 name: "initial_runtime_metadata".to_string(),
4905 dialect: "sqlite".to_string(),
4906 sql: SQLITE_INITIAL_DDL.to_string(),
4907 }]);
4908 }
4909 if normalized == "postgres" || normalized == "postgresql" {
4910 return Ok(vec![Migration {
4911 version: "0001".to_string(),
4912 name: "initial_runtime_metadata".to_string(),
4913 dialect: "postgres".to_string(),
4914 sql: POSTGRES_INITIAL_DDL.to_string(),
4915 }]);
4916 }
4917 if normalized == "mysql" {
4918 return Ok(vec![Migration {
4919 version: "0001".to_string(),
4920 name: "initial_runtime_metadata".to_string(),
4921 dialect: "mysql".to_string(),
4922 sql: MYSQL_INITIAL_DDL.to_string(),
4923 }]);
4924 }
4925 Err(RuntimeError(format!(
4926 "unsupported storage dialect: {dialect}"
4927 )))
4928}
4929
4930pub fn latest_schema_version(dialect: &str) -> Result<Option<String>> {
4931 Ok(migrations_for(dialect)?
4932 .last()
4933 .map(|migration| migration.version.clone()))
4934}
4935
4936pub fn ddl_for(dialect: &str) -> Result<String> {
4937 let normalized = dialect.to_lowercase();
4938 let header = if normalized == "postgres" || normalized == "postgresql" {
4939 SCHEMA_MIGRATIONS_POSTGRES
4940 } else if normalized == "mysql" {
4941 SCHEMA_MIGRATIONS_MYSQL
4942 } else {
4943 SCHEMA_MIGRATIONS_SQLITE
4944 };
4945 let mut parts = vec![header.to_string()];
4946 for migration in migrations_for(dialect)? {
4947 parts.push(migration.sql);
4948 }
4949 Ok(parts.join("\n\n"))
4950}
4951
4952const SCHEMA_MIGRATIONS_SQLITE: &str = "CREATE TABLE IF NOT EXISTS schema_migrations (\n version TEXT PRIMARY KEY,\n name TEXT NOT NULL,\n checksum TEXT NOT NULL,\n applied_at REAL NOT NULL\n);";
4953const SCHEMA_MIGRATIONS_POSTGRES: &str = "CREATE TABLE IF NOT EXISTS schema_migrations (\n version TEXT PRIMARY KEY,\n name TEXT NOT NULL,\n checksum TEXT NOT NULL,\n applied_at DOUBLE PRECISION NOT NULL\n);";
4954const SCHEMA_MIGRATIONS_MYSQL: &str = "CREATE TABLE IF NOT EXISTS schema_migrations (\n version VARCHAR(32) PRIMARY KEY,\n name VARCHAR(255) NOT NULL,\n checksum VARCHAR(128) NOT NULL,\n applied_at DOUBLE NOT NULL\n);";
4955const SQLITE_INITIAL_DDL: &str = "CREATE TABLE IF NOT EXISTS runs (run_id TEXT PRIMARY KEY, session_id TEXT NOT NULL, status TEXT NOT NULL, state_json TEXT NOT NULL, state_version INTEGER NOT NULL, created_at REAL NOT NULL, updated_at REAL NOT NULL);\nCREATE TABLE IF NOT EXISTS steps (step_id TEXT PRIMARY KEY, run_id TEXT NOT NULL, session_id TEXT NOT NULL, status TEXT NOT NULL, owner TEXT, lease_token TEXT, lease_until REAL, attempt INTEGER NOT NULL, state_version INTEGER NOT NULL, checkpoint_id TEXT, created_at REAL NOT NULL, updated_at REAL NOT NULL);\nCREATE TABLE IF NOT EXISTS events (event_id TEXT PRIMARY KEY, run_id TEXT NOT NULL, session_id TEXT, step_id TEXT, seq INTEGER NOT NULL, type TEXT NOT NULL, timestamp REAL NOT NULL, agent_role TEXT, state_version INTEGER, causal_token TEXT, payload_hash TEXT, payload_ref TEXT);\nCREATE TABLE IF NOT EXISTS tool_ledger (ledger_id TEXT PRIMARY KEY, run_id TEXT NOT NULL, session_id TEXT, step_id TEXT NOT NULL, tool_name TEXT NOT NULL, tool_version TEXT NOT NULL, tool_call_id TEXT NOT NULL, idempotency_key TEXT NOT NULL UNIQUE, causal_token TEXT NOT NULL, request_hash TEXT NOT NULL, request_ref TEXT NOT NULL, status TEXT NOT NULL, external_id TEXT, response_hash TEXT, response_ref TEXT, error_type TEXT, created_at REAL NOT NULL, updated_at REAL NOT NULL);";
4956const POSTGRES_INITIAL_DDL: &str = "CREATE TABLE IF NOT EXISTS runs (run_id TEXT PRIMARY KEY, session_id TEXT NOT NULL, status TEXT NOT NULL, state_json JSONB NOT NULL, state_version BIGINT NOT NULL, created_at DOUBLE PRECISION NOT NULL, updated_at DOUBLE PRECISION NOT NULL);\nCREATE TABLE IF NOT EXISTS steps (step_id TEXT PRIMARY KEY, run_id TEXT NOT NULL REFERENCES runs(run_id), session_id TEXT NOT NULL, status TEXT NOT NULL, owner TEXT, lease_token TEXT, lease_until DOUBLE PRECISION, attempt BIGINT NOT NULL, state_version BIGINT NOT NULL, checkpoint_id TEXT, created_at DOUBLE PRECISION NOT NULL, updated_at DOUBLE PRECISION NOT NULL);\nCREATE TABLE IF NOT EXISTS events (event_id TEXT PRIMARY KEY, run_id TEXT NOT NULL, session_id TEXT, step_id TEXT, seq BIGINT NOT NULL, type TEXT NOT NULL, timestamp DOUBLE PRECISION NOT NULL, agent_role TEXT, state_version BIGINT, causal_token TEXT, payload_hash TEXT, payload_ref TEXT, UNIQUE(run_id, seq));\nCREATE TABLE IF NOT EXISTS tool_ledger (ledger_id TEXT PRIMARY KEY, run_id TEXT NOT NULL, session_id TEXT, step_id TEXT NOT NULL, tool_name TEXT NOT NULL, tool_version TEXT NOT NULL, tool_call_id TEXT NOT NULL, idempotency_key TEXT NOT NULL UNIQUE, causal_token TEXT NOT NULL, request_hash TEXT NOT NULL, request_ref TEXT NOT NULL, status TEXT NOT NULL, external_id TEXT, response_hash TEXT, response_ref TEXT, error_type TEXT, created_at DOUBLE PRECISION NOT NULL, updated_at DOUBLE PRECISION NOT NULL);";
4957const MYSQL_INITIAL_DDL: &str = "CREATE TABLE IF NOT EXISTS runs (run_id VARCHAR(128) PRIMARY KEY, session_id VARCHAR(128) NOT NULL, status VARCHAR(64) NOT NULL, state_json JSON NOT NULL, state_version BIGINT NOT NULL, created_at DOUBLE NOT NULL, updated_at DOUBLE NOT NULL);\nCREATE TABLE IF NOT EXISTS steps (step_id VARCHAR(128) PRIMARY KEY, run_id VARCHAR(128) NOT NULL, session_id VARCHAR(128) NOT NULL, status VARCHAR(64) NOT NULL, owner VARCHAR(255), lease_token VARCHAR(128), lease_until DOUBLE, attempt BIGINT NOT NULL, state_version BIGINT NOT NULL, checkpoint_id VARCHAR(255), created_at DOUBLE NOT NULL, updated_at DOUBLE NOT NULL, INDEX idx_steps_run_status (run_id, status));\nCREATE TABLE IF NOT EXISTS events (event_id VARCHAR(128) PRIMARY KEY, run_id VARCHAR(128) NOT NULL, session_id VARCHAR(128), step_id VARCHAR(128), seq BIGINT NOT NULL, type VARCHAR(255) NOT NULL, timestamp DOUBLE NOT NULL, agent_role VARCHAR(255), state_version BIGINT, causal_token TEXT, payload_hash VARCHAR(128), payload_ref TEXT, UNIQUE KEY idx_events_run_seq (run_id, seq));\nCREATE TABLE IF NOT EXISTS tool_ledger (ledger_id VARCHAR(128) PRIMARY KEY, run_id VARCHAR(128) NOT NULL, session_id VARCHAR(128), step_id VARCHAR(128) NOT NULL, tool_name VARCHAR(255) NOT NULL, tool_version VARCHAR(64) NOT NULL, tool_call_id VARCHAR(128) NOT NULL, idempotency_key VARCHAR(255) NOT NULL UNIQUE, causal_token TEXT NOT NULL, request_hash VARCHAR(128) NOT NULL, request_ref TEXT NOT NULL, status VARCHAR(64) NOT NULL, external_id VARCHAR(255), response_hash VARCHAR(128), response_ref TEXT, error_type VARCHAR(255), created_at DOUBLE NOT NULL, updated_at DOUBLE NOT NULL, INDEX idx_tool_ledger_run_tool (run_id, tool_name));";
4958
4959pub type MCPCall = fn(&str, State) -> Result<Value>;
4960pub type MCPResourceRead = fn(&str) -> Result<Value>;
4961
4962#[derive(Clone, Debug)]
4963pub struct MCPResourceDescriptor {
4964 pub uri: String,
4965 pub name: String,
4966 pub mime_type: String,
4967}
4968
4969impl MCPResourceDescriptor {
4970 pub fn to_state(&self) -> State {
4971 state(&[
4972 ("uri", self.uri.clone().into()),
4973 ("name", self.name.clone().into()),
4974 ("mimeType", self.mime_type.clone().into()),
4975 ])
4976 }
4977}
4978
4979pub struct InMemoryMCPToolServer {
4980 tools: HashMap<String, (State, MCPCall)>,
4981}
4982
4983impl InMemoryMCPToolServer {
4984 pub fn new() -> Self {
4985 Self {
4986 tools: HashMap::new(),
4987 }
4988 }
4989 pub fn add_tool(&mut self, descriptor: State, handler: MCPCall) {
4990 if let Some(Value::String(name)) = descriptor.get("name") {
4991 self.tools.insert(name.clone(), (descriptor, handler));
4992 }
4993 }
4994 pub fn list_tools(&self) -> Vec<State> {
4995 let mut names: Vec<_> = self.tools.keys().cloned().collect();
4996 names.sort();
4997 names
4998 .into_iter()
4999 .filter_map(|name| self.tools.get(&name).map(|entry| entry.0.clone()))
5000 .collect()
5001 }
5002 pub fn call_tool(&self, name: &str, args: State) -> Result<Value> {
5003 let (_, handler) = self
5004 .tools
5005 .get(name)
5006 .ok_or_else(|| RuntimeError(format!("MCP tool not found: {name}")))?;
5007 handler(name, args)
5008 }
5009}
5010
5011pub struct InMemoryMCPContextServer {
5012 resources: HashMap<String, (MCPResourceDescriptor, MCPResourceRead)>,
5013}
5014
5015impl InMemoryMCPContextServer {
5016 pub fn new() -> Self {
5017 Self {
5018 resources: HashMap::new(),
5019 }
5020 }
5021 pub fn add_resource(
5022 &mut self,
5023 uri: &str,
5024 name: &str,
5025 mime_type: &str,
5026 reader: MCPResourceRead,
5027 ) {
5028 self.resources.insert(
5029 uri.to_string(),
5030 (
5031 MCPResourceDescriptor {
5032 uri: uri.to_string(),
5033 name: name.to_string(),
5034 mime_type: if mime_type.is_empty() {
5035 "application/json".to_string()
5036 } else {
5037 mime_type.to_string()
5038 },
5039 },
5040 reader,
5041 ),
5042 );
5043 }
5044 pub fn list_resources(&self) -> Vec<State> {
5045 let mut uris: Vec<_> = self.resources.keys().cloned().collect();
5046 uris.sort();
5047 uris.into_iter()
5048 .filter_map(|uri| self.resources.get(&uri).map(|entry| entry.0.to_state()))
5049 .collect()
5050 }
5051 pub fn read_resource(&self, uri: &str) -> Result<State> {
5052 let (descriptor, reader) = self
5053 .resources
5054 .get(uri)
5055 .ok_or_else(|| RuntimeError(format!("MCP resource not found: {uri}")))?;
5056 Ok(state(&[
5057 ("resource", Value::Object(descriptor.to_state())),
5058 ("content", reader(uri)?),
5059 ]))
5060 }
5061}
5062
5063pub struct MCPToolAdapter {
5064 pub client_call: MCPCall,
5065}
5066
5067impl MCPToolAdapter {
5068 pub fn tool_spec_from_descriptor(&self, descriptor: &State) -> ToolSpec {
5069 let name = string_field(descriptor, "name", "");
5070 let version = string_field(descriptor, "version", "v1");
5071 let annotations = match descriptor.get("annotations") {
5072 Some(Value::Object(state)) => state.clone(),
5073 _ => State::new(),
5074 };
5075 let side_effect = string_field(&annotations, "side_effect", "none");
5076 let risk_level = string_field(&annotations, "risk_level", "low");
5077 let idempotency_required = match annotations.get("idempotency_required") {
5078 Some(Value::Bool(value)) => *value,
5079 _ => side_effect != "none",
5080 };
5081 let approval_required = match annotations.get("approval_required") {
5082 Some(Value::Bool(value)) => *value,
5083 _ => false,
5084 };
5085 let sandbox_required = match annotations.get("sandbox_required") {
5086 Some(Value::Bool(value)) => *value,
5087 _ => false,
5088 };
5089 let sandbox_executor = string_field(&annotations, "sandbox_executor", "");
5090 let sandbox_policy = match annotations.get("sandbox_policy") {
5091 Some(Value::Object(state)) => state.clone(),
5092 _ => State::new(),
5093 };
5094 let client_call = self.client_call;
5095 let tool_name = name.clone();
5096 let mut spec = ToolSpec::new(&name, Box::new(move |args| client_call(&tool_name, args)));
5097 spec.version = version;
5098 spec.side_effect = side_effect;
5099 spec.risk_level = risk_level;
5100 spec.idempotency_required = idempotency_required;
5101 spec.approval_required = approval_required;
5102 spec.sandbox_required = sandbox_required;
5103 spec.sandbox_executor = sandbox_executor;
5104 spec.sandbox_policy = sandbox_policy;
5105 spec.input_schema = descriptor
5106 .get("inputSchema")
5107 .or_else(|| descriptor.get("input_schema"))
5108 .cloned();
5109 spec.output_schema = descriptor
5110 .get("outputSchema")
5111 .or_else(|| descriptor.get("output_schema"))
5112 .cloned();
5113 spec
5114 }
5115}
5116
5117pub struct MCPContextAdapter {
5118 pub resource_read: MCPResourceRead,
5119}
5120
5121impl MCPContextAdapter {
5122 pub fn read_tool_spec(&self, name: &str, risk_level: &str) -> ToolSpec {
5123 let tool_name = if name.is_empty() {
5124 "mcp.context.read"
5125 } else {
5126 name
5127 };
5128 let risk = if risk_level.is_empty() {
5129 "low"
5130 } else {
5131 risk_level
5132 };
5133 let reader = self.resource_read;
5134 let mut spec = ToolSpec::new(
5135 tool_name,
5136 Box::new(move |args| match args.get("uri") {
5137 Some(Value::String(uri)) => reader(uri),
5138 _ => Err(RuntimeError("uri is required".to_string())),
5139 }),
5140 );
5141 spec.risk_level = risk.to_string();
5142 spec.side_effect = "none".to_string();
5143 spec.input_schema = Some(Value::Object(state(&[("type", "object".into())])));
5144 spec.output_schema = Some(Value::Object(state(&[("type", "object".into())])));
5145 spec
5146 }
5147}
5148
5149fn string_field(state: &State, key: &str, fallback: &str) -> String {
5150 match state.get(key) {
5151 Some(Value::String(value)) => value.clone(),
5152 _ => fallback.to_string(),
5153 }
5154}
5155
5156pub type FrameworkAgentFunc = fn(&mut AgentContext, State) -> Result<Option<Value>>;
5157
5158pub struct FunctionAdapter {
5159 pub func: FrameworkAgentFunc,
5160 pub role: String,
5161 pub name: String,
5162}
5163
5164impl FunctionAdapter {
5165 pub fn new(func: FrameworkAgentFunc, role: &str) -> Self {
5166 Self {
5167 func,
5168 role: if role.is_empty() {
5169 "Agent".to_string()
5170 } else {
5171 role.to_string()
5172 },
5173 name: "function".to_string(),
5174 }
5175 }
5176 pub fn map_run_spec(&self) -> State {
5177 state(&[
5178 ("adapter", self.name.clone().into()),
5179 ("role", self.role.clone().into()),
5180 ])
5181 }
5182 pub fn run(&self, ctx: &mut AgentContext, state_value: State, output_key: &str) -> Result<()> {
5183 if let Some(result) = (self.func)(ctx, state_value)? {
5184 if !output_key.is_empty() {
5185 ctx.write_state(output_key, result);
5186 }
5187 }
5188 Ok(())
5189 }
5190}
5191
5192pub type MethodHandler = fn(State) -> Result<Value>;
5193
5194pub struct MethodFrameworkAdapter {
5195 pub target_name: String,
5196 pub role: String,
5197 pub method_candidates: Vec<String>,
5198 pub methods: HashMap<String, MethodHandler>,
5199 pub output_key: String,
5200}
5201
5202impl MethodFrameworkAdapter {
5203 pub fn new(
5204 target_name: &str,
5205 role: &str,
5206 method_candidates: Vec<String>,
5207 methods: HashMap<String, MethodHandler>,
5208 output_key: &str,
5209 ) -> Self {
5210 Self {
5211 target_name: target_name.to_string(),
5212 role: if role.is_empty() {
5213 "FrameworkAgent".to_string()
5214 } else {
5215 role.to_string()
5216 },
5217 method_candidates,
5218 methods,
5219 output_key: if output_key.is_empty() {
5220 "output".to_string()
5221 } else {
5222 output_key.to_string()
5223 },
5224 }
5225 }
5226 pub fn map_run_spec(&self) -> State {
5227 state(&[
5228 ("adapter", "method-framework".into()),
5229 ("role", self.role.clone().into()),
5230 ("target", self.target_name.clone().into()),
5231 (
5232 "methods",
5233 Value::Array(
5234 self.method_candidates
5235 .iter()
5236 .map(|item| Value::String(item.clone()))
5237 .collect(),
5238 ),
5239 ),
5240 ])
5241 }
5242 pub fn run(&self, ctx: &mut AgentContext, state_value: State) -> Result<()> {
5243 for name in &self.method_candidates {
5244 if let Some(handler) = self.methods.get(name) {
5245 let result = handler(state_value)?;
5246 if !self.output_key.is_empty() {
5247 ctx.write_state(&self.output_key, result);
5248 }
5249 return Ok(());
5250 }
5251 }
5252 Err(RuntimeError(
5253 "target does not expose any candidate method".to_string(),
5254 ))
5255 }
5256}
5257
5258#[derive(Clone, Debug)]
5259pub struct BoundaryLintRule {
5260 pub rule_id: String,
5261 pub pattern: String,
5262 pub category: String,
5263 pub message: String,
5264 pub suggestion: String,
5265 pub prefix: bool,
5266}
5267
5268#[derive(Clone, Debug)]
5269pub struct BoundaryLintFinding {
5270 pub path: String,
5271 pub line: usize,
5272 pub column: usize,
5273 pub rule_id: String,
5274 pub severity: String,
5275 pub callee: String,
5276 pub category: String,
5277 pub message: String,
5278 pub suggestion: String,
5279}
5280
5281#[derive(Clone, Debug)]
5282pub struct BoundaryLintReport {
5283 pub passed: bool,
5284 pub scanned_files: Vec<String>,
5285 pub finding_count: usize,
5286 pub findings: Vec<BoundaryLintFinding>,
5287}
5288
5289pub fn default_boundary_rules() -> Vec<BoundaryLintRule> {
5290 vec![
5291 BoundaryLintRule { rule_id: "direct-shell-os-system".into(), pattern: "os.system".into(), category: "shell".into(), message: "direct shell execution bypasses ToolGateway, policy, ledger, sandbox, and audit".into(), suggestion: "wrap shell execution as a runtime-managed tool".into(), prefix: false },
5292 BoundaryLintRule { rule_id: "direct-shell-subprocess".into(), pattern: "subprocess.".into(), category: "shell".into(), message: "direct subprocess execution bypasses ToolGateway, policy, ledger, sandbox, and audit".into(), suggestion: "wrap command execution as a runtime-managed tool".into(), prefix: true },
5293 BoundaryLintRule { rule_id: "direct-http-requests".into(), pattern: "requests.".into(), category: "network".into(), message: "direct HTTP calls bypass ToolGateway, policy, ledger, budget, replay, and audit".into(), suggestion: "register the HTTP/API call as a runtime-managed tool".into(), prefix: true },
5294 BoundaryLintRule { rule_id: "direct-http-httpx".into(), pattern: "httpx.".into(), category: "network".into(), message: "direct HTTP calls bypass ToolGateway, policy, ledger, budget, replay, and audit".into(), suggestion: "register the HTTP/API call as a runtime-managed tool".into(), prefix: true },
5295 BoundaryLintRule { rule_id: "direct-openai-sdk".into(), pattern: "openai.".into(), category: "model".into(), message: "direct model SDK usage bypasses model provider archives, replay, budget, and attribution".into(), suggestion: "call models through the runtime model boundary".into(), prefix: true },
5296 BoundaryLintRule { rule_id: "direct-anthropic-sdk".into(), pattern: "anthropic.".into(), category: "model".into(), message: "direct model SDK usage bypasses model provider archives, replay, budget, and attribution".into(), suggestion: "call models through the runtime model boundary".into(), prefix: true },
5297 ]
5298}
5299
5300pub fn scan_boundary_source(
5301 path: &str,
5302 source: &str,
5303 rules: Option<Vec<BoundaryLintRule>>,
5304) -> BoundaryLintReport {
5305 let rules = rules.unwrap_or_else(default_boundary_rules);
5306 let lines: Vec<&str> = source.split('\n').collect();
5307 let mut findings = Vec::new();
5308 for (i, line) in lines.iter().enumerate() {
5309 let previous = if i > 0 { lines[i - 1] } else { "" };
5310 if line.contains("agentledger: ignore-boundary")
5311 || previous.contains("agentledger: ignore-next-line")
5312 {
5313 continue;
5314 }
5315 for rule in &rules {
5316 if let Some(index) = line.find(&rule.pattern) {
5317 let mut callee = rule.pattern.clone();
5318 if rule.prefix {
5319 let mut end = index + rule.pattern.len();
5320 while end < line.len() {
5321 let ch = line.as_bytes()[end] as char;
5322 if ch.is_ascii_alphanumeric() || ch == '_' || ch == '.' {
5323 end += 1;
5324 } else {
5325 break;
5326 }
5327 }
5328 callee = line[index..end].to_string();
5329 }
5330 findings.push(BoundaryLintFinding {
5331 path: path.into(),
5332 line: i + 1,
5333 column: index + 1,
5334 rule_id: rule.rule_id.clone(),
5335 severity: "error".into(),
5336 callee,
5337 category: rule.category.clone(),
5338 message: rule.message.clone(),
5339 suggestion: rule.suggestion.clone(),
5340 });
5341 break;
5342 }
5343 }
5344 }
5345 BoundaryLintReport {
5346 passed: findings.is_empty(),
5347 scanned_files: vec![path.into()],
5348 finding_count: findings.len(),
5349 findings,
5350 }
5351}
5352
5353#[derive(Clone, Debug)]
5354pub struct RecoverySummary {
5355 pub recovered_steps: usize,
5356}
5357
5358#[derive(Clone, Debug)]
5359pub struct SchedulerStepStatus {
5360 pub step_id: String,
5361 pub status: String,
5362 pub owner: Option<String>,
5363 pub attempt: u64,
5364 pub lease_until: Option<f64>,
5365 pub last_error_type: Option<String>,
5366}
5367
5368#[derive(Clone, Debug)]
5369pub struct SchedulerStatus {
5370 pub run_id: String,
5371 pub run_status: String,
5372 pub state_version: u64,
5373 pub steps: Vec<SchedulerStepStatus>,
5374 pub cost_summary: CostSummary,
5375}
5376
5377pub struct RuntimeScheduler;
5378
5379impl RuntimeScheduler {
5380 pub fn recover_expired_leases(store: &mut MemoryStore) -> RecoverySummary {
5381 RecoverySummary {
5382 recovered_steps: store.recover_expired_leases(),
5383 }
5384 }
5385
5386 pub fn cancel_run(store: &mut MemoryStore, run_id: &str, reason: &str) -> Result<usize> {
5387 store.cancel_run(run_id, reason)
5388 }
5389
5390 pub fn status(store: &MemoryStore, run_id: &str) -> Result<SchedulerStatus> {
5391 let run = store.run(run_id)?;
5392 let steps = store
5393 .steps(run_id)
5394 .into_iter()
5395 .map(|step| SchedulerStepStatus {
5396 step_id: step.step_id,
5397 status: step.status,
5398 owner: step.owner,
5399 attempt: step.attempt,
5400 lease_until: step.lease_until,
5401 last_error_type: step.last_error_type,
5402 })
5403 .collect();
5404 Ok(SchedulerStatus {
5405 run_id: run_id.to_string(),
5406 run_status: run.status,
5407 state_version: run.state_version,
5408 steps,
5409 cost_summary: store.cost_summary(run_id),
5410 })
5411 }
5412}
5413
5414#[derive(Clone, Debug)]
5415pub struct ReviewCheck {
5416 pub name: String,
5417 pub passed: bool,
5418 pub severity: String,
5419 pub detail: String,
5420}
5421
5422#[derive(Clone, Debug)]
5423pub struct AdversarialReviewReport {
5424 pub passed: bool,
5425 pub run_id: Option<String>,
5426 pub checks: Vec<ReviewCheck>,
5427 pub metadata: State,
5428}
5429
5430pub fn adversarial_review(
5431 bundle: &EvidenceBundle,
5432 max_total_usd: Option<f64>,
5433) -> AdversarialReviewReport {
5434 let mut checks = vec![
5435 review_check(
5436 "no_failed_steps",
5437 !bundle
5438 .events
5439 .iter()
5440 .any(|event| event.event_type == "step_failed"),
5441 "blocker",
5442 "no step is in failed status",
5443 ),
5444 review_check(
5445 "no_pending_verification",
5446 !bundle
5447 .tool_ledger
5448 .iter()
5449 .any(|row| row.status == "PENDING_VERIFICATION"),
5450 "blocker",
5451 "no side effect is pending verification",
5452 ),
5453 review_check(
5454 "no_pending_approvals",
5455 !bundle.approvals.iter().any(|row| row.status == "PENDING"),
5456 "blocker",
5457 "no approval request is still pending",
5458 ),
5459 review_check(
5460 "completed_steps_have_completion_events",
5461 completed_steps_have_events_rust(&bundle.steps, &bundle.events),
5462 "blocker",
5463 "completed steps have step_completed events",
5464 ),
5465 review_check(
5466 "ledger_statuses_known",
5467 ledger_statuses_known_rust(&bundle.tool_ledger),
5468 "blocker",
5469 "Tool Ledger rows use known statuses",
5470 ),
5471 review_check(
5472 "event_sequence_contiguous",
5473 event_sequence_contiguous_rust(&bundle.events),
5474 "blocker",
5475 "event sequence has no gaps",
5476 ),
5477 review_check(
5478 "artifacts_have_blob_refs",
5479 bundle
5480 .artifacts
5481 .iter()
5482 .all(|row| !row.blob_ref.is_empty() && !row.blob_hash.is_empty()),
5483 "warning",
5484 "artifacts have blob refs and hashes",
5485 ),
5486 review_check(
5487 "media_artifacts_have_refs",
5488 bundle
5489 .media_artifacts
5490 .iter()
5491 .all(media_artifact_has_ref_rust),
5492 "blocker",
5493 "media artifacts have kind and durable refs",
5494 ),
5495 review_check(
5496 "stream_checkpoints_have_offsets",
5497 bundle
5498 .stream_checkpoints
5499 .iter()
5500 .all(stream_checkpoint_has_offset_rust),
5501 "blocker",
5502 "stream checkpoints have stream, consumer, and offset",
5503 ),
5504 review_check(
5505 "high_risk_approvals_decided",
5506 high_risk_approvals_decided_rust(&bundle.approvals),
5507 "blocker",
5508 "high-risk approval requests are decided",
5509 ),
5510 review_check(
5511 "no_blocking_failure_events",
5512 !bundle.events.iter().any(|event| {
5513 matches!(
5514 event.event_type.as_str(),
5515 "error_raised" | "step_failed" | "tool_call_failed" | "tool_call_blocked"
5516 )
5517 }),
5518 "warning",
5519 "no blocking failure events are present",
5520 ),
5521 ];
5522 if let Some(limit) = max_total_usd {
5523 checks.push(review_check(
5524 "max_total_usd",
5525 bundle.cost_summary.total_usd <= limit,
5526 "blocker",
5527 "cost limit check",
5528 ));
5529 }
5530 let passed = checks
5531 .iter()
5532 .all(|check| check.severity != "blocker" || check.passed);
5533 let mut metadata = State::new();
5534 metadata.insert(
5535 "event_count".into(),
5536 Value::Number(bundle.events.len() as f64),
5537 );
5538 metadata.insert(
5539 "tool_ledger_count".into(),
5540 Value::Number(bundle.tool_ledger.len() as f64),
5541 );
5542 metadata.insert(
5543 "approval_count".into(),
5544 Value::Number(bundle.approvals.len() as f64),
5545 );
5546 metadata.insert(
5547 "artifact_count".into(),
5548 Value::Number(bundle.artifacts.len() as f64),
5549 );
5550 metadata.insert(
5551 "media_artifact_count".into(),
5552 Value::Number(bundle.media_artifacts.len() as f64),
5553 );
5554 metadata.insert(
5555 "stream_checkpoint_count".into(),
5556 Value::Number(bundle.stream_checkpoints.len() as f64),
5557 );
5558 AdversarialReviewReport {
5559 passed,
5560 run_id: Some(bundle.run.run_id.clone()),
5561 checks,
5562 metadata,
5563 }
5564}
5565
5566fn review_check(name: &str, passed: bool, severity: &str, detail: &str) -> ReviewCheck {
5567 ReviewCheck {
5568 name: name.into(),
5569 passed,
5570 severity: severity.into(),
5571 detail: detail.into(),
5572 }
5573}
5574
5575fn completed_steps_have_events_rust(steps: &[Step], events: &[Event]) -> bool {
5576 steps.iter().all(|step| {
5577 step.status != "completed"
5578 || events.iter().any(|event| {
5579 event.event_type == "step_completed"
5580 && event.step_id.as_deref() == Some(step.step_id.as_str())
5581 })
5582 })
5583}
5584
5585fn ledger_statuses_known_rust(rows: &[ToolLedgerEntry]) -> bool {
5586 rows.iter().all(|row| {
5587 matches!(
5588 row.status.as_str(),
5589 "SUCCEEDED"
5590 | "FAILED_NO_EFFECT"
5591 | "PENDING_VERIFICATION"
5592 | "COMPENSATED"
5593 | "RUNNING"
5594 | "RESERVED"
5595 )
5596 })
5597}
5598fn event_sequence_contiguous_rust(events: &[Event]) -> bool {
5599 events
5600 .iter()
5601 .enumerate()
5602 .all(|(index, event)| event.seq == (index as u64) + 1)
5603}
5604fn state_has_key(row: &State, key: &str) -> bool {
5605 !matches!(row.get(key), None | Some(Value::Null))
5606}
5607fn media_artifact_has_ref_rust(row: &State) -> bool {
5608 state_has_key(row, "kind")
5609 && (state_has_key(row, "uri")
5610 || state_has_key(row, "content_ref")
5611 || state_has_key(row, "blob_ref"))
5612}
5613fn stream_checkpoint_has_offset_rust(row: &State) -> bool {
5614 state_has_key(row, "stream_id")
5615 && state_has_key(row, "consumer_id")
5616 && state_has_key(row, "offset")
5617}
5618fn high_risk_approvals_decided_rust(rows: &[ApprovalRequest]) -> bool {
5619 rows.iter().all(|row| {
5620 !matches!(
5621 row.risk_level.as_str(),
5622 "high" | "destructive" | "sensitive"
5623 ) || matches!(row.status.as_str(), "APPROVED" | "DENIED")
5624 })
5625}
5626
5627#[derive(Clone, Debug)]
5628pub struct EvidenceCheck {
5629 pub name: String,
5630 pub passed: bool,
5631 pub detail: String,
5632}
5633
5634#[derive(Clone, Debug)]
5635pub struct EvidenceCheckReport {
5636 pub passed: bool,
5637 pub checks: Vec<EvidenceCheck>,
5638 pub metadata: State,
5639}
5640
5641pub fn evaluate_evidence(
5642 bundle: &EvidenceBundle,
5643 max_total_usd: Option<f64>,
5644) -> EvidenceCheckReport {
5645 let mut checks = vec![
5646 evidence_check(
5647 "no_failed_steps",
5648 !bundle
5649 .events
5650 .iter()
5651 .any(|event| event.event_type == "step_failed"),
5652 "all steps completed or remain non-failed",
5653 ),
5654 evidence_check(
5655 "no_pending_verification",
5656 !bundle
5657 .tool_ledger
5658 .iter()
5659 .any(|row| row.status == "PENDING_VERIFICATION"),
5660 "no side effect is waiting for human/external verification",
5661 ),
5662 evidence_check(
5663 "completed_steps_have_events",
5664 completed_steps_have_events_rust(&bundle.steps, &bundle.events),
5665 "each completed step has a step_completed event",
5666 ),
5667 evidence_check(
5668 "managed_side_effects_are_ledgered",
5669 ledger_statuses_known_rust(&bundle.tool_ledger),
5670 "every ledger row has a known status",
5671 ),
5672 evidence_check(
5673 "media_artifacts_have_refs",
5674 bundle
5675 .media_artifacts
5676 .iter()
5677 .all(media_artifact_has_ref_rust),
5678 "media artifacts have kind and durable refs",
5679 ),
5680 evidence_check(
5681 "stream_checkpoints_have_offsets",
5682 bundle
5683 .stream_checkpoints
5684 .iter()
5685 .all(stream_checkpoint_has_offset_rust),
5686 "stream checkpoints have stream, consumer, and offset",
5687 ),
5688 ];
5689 if let Some(limit) = max_total_usd {
5690 checks.push(evidence_check(
5691 "max_total_usd",
5692 bundle.cost_summary.total_usd <= limit,
5693 "cost limit check",
5694 ));
5695 }
5696 EvidenceCheckReport {
5697 passed: checks.iter().all(|check| check.passed),
5698 checks,
5699 metadata: State::new(),
5700 }
5701}
5702
5703pub fn evaluate_evidence_regression(
5704 golden: &EvidenceBundle,
5705 current: &EvidenceBundle,
5706 max_total_usd_delta: Option<f64>,
5707) -> EvidenceCheckReport {
5708 let diff = diff_evidence(golden, current);
5709 let mut checks = vec![
5710 evidence_check(
5711 "final_state_regression",
5712 diff.final_state_changed_count == 0,
5713 "final state regression check",
5714 ),
5715 evidence_check(
5716 "event_type_regression",
5717 diff.event_types_changed_count == 0,
5718 "event type regression check",
5719 ),
5720 evidence_check(
5721 "tool_ledger_status_regression",
5722 true,
5723 "tool ledger status regression check",
5724 ),
5725 evidence_check(
5726 "media_artifact_regression",
5727 diff.media_artifacts_changed_count == 0,
5728 "media artifact regression check",
5729 ),
5730 evidence_check(
5731 "stream_checkpoint_regression",
5732 diff.stream_checkpoints_changed_count == 0,
5733 "stream checkpoint regression check",
5734 ),
5735 ];
5736 if let Some(limit) = max_total_usd_delta {
5737 let delta = current.cost_summary.total_usd - golden.cost_summary.total_usd;
5738 checks.push(evidence_check(
5739 "max_total_usd_delta",
5740 delta <= limit,
5741 "cost delta limit check",
5742 ));
5743 }
5744 EvidenceCheckReport {
5745 passed: checks.iter().all(|check| check.passed),
5746 checks,
5747 metadata: State::new(),
5748 }
5749}
5750
5751fn evidence_check(name: &str, passed: bool, detail: &str) -> EvidenceCheck {
5752 EvidenceCheck {
5753 name: name.into(),
5754 passed,
5755 detail: detail.into(),
5756 }
5757}
5758
5759#[derive(Clone, Debug)]
5760pub struct FailureInjectionCheck {
5761 pub name: String,
5762 pub passed: bool,
5763 pub detail: String,
5764 pub run_id: Option<String>,
5765}
5766
5767#[derive(Clone, Debug)]
5768pub struct FailureInjectionReport {
5769 pub passed: bool,
5770 pub checks: Vec<FailureInjectionCheck>,
5771}
5772
5773pub fn run_failure_injection_suite() -> FailureInjectionReport {
5774 let checks = vec![
5775 failure_retry_exhaustion(),
5776 failure_lease_fencing(),
5777 failure_cancellation_fencing(),
5778 failure_side_effect_idempotency(),
5779 ];
5780 FailureInjectionReport {
5781 passed: checks.iter().all(|check| check.passed),
5782 checks,
5783 }
5784}
5785fn failure_check(
5786 name: &str,
5787 passed: bool,
5788 detail: String,
5789 run_id: String,
5790) -> FailureInjectionCheck {
5791 FailureInjectionCheck {
5792 name: name.into(),
5793 passed,
5794 detail,
5795 run_id: Some(run_id),
5796 }
5797}
5798fn failure_retry_exhaustion() -> FailureInjectionCheck {
5799 let mut runtime = Runtime::new();
5800 let (run_id, _) = runtime.create_run(State::new());
5801 let _ = runtime.run_once(
5802 &run_id,
5803 "retry-1",
5804 "FailureInjector",
5805 60.0,
5806 |_ctx, _state| Err(RuntimeError("retryable".into())),
5807 );
5808 let _ = runtime.run_once(
5809 &run_id,
5810 "retry-2",
5811 "FailureInjector",
5812 60.0,
5813 |_ctx, _state| Err(RuntimeError("final failure".into())),
5814 );
5815 let status = runtime
5816 .store
5817 .run(&run_id)
5818 .map(|run| run.status)
5819 .unwrap_or_else(|_| "missing".into());
5820 failure_check(
5821 "retry_exhaustion",
5822 status == "failed",
5823 format!("run_status={status}"),
5824 run_id,
5825 )
5826}
5827fn failure_lease_fencing() -> FailureInjectionCheck {
5828 let mut store = MemoryStore::new();
5829 let (run_id, step_id) = store.create_run(State::new());
5830 let claim = store.claim_step("stale-worker", &run_id, 0.0).unwrap();
5831 let recovered = store.recover_expired_leases();
5832 let stale_rejected = store
5833 .commit_state_patch(&run_id, &step_id, &claim.lease_token, 0, State::new())
5834 .is_err();
5835 let fresh = store.claim_step("fresh-worker", &run_id, 60.0).unwrap();
5836 let passed = recovered == 1 && stale_rejected && fresh.attempt == 2;
5837 failure_check(
5838 "lease_fencing",
5839 passed,
5840 format!("recovered_steps={recovered} stale_rejected={stale_rejected}"),
5841 run_id,
5842 )
5843}
5844fn failure_cancellation_fencing() -> FailureInjectionCheck {
5845 let mut store = MemoryStore::new();
5846 let (run_id, step_id) = store.create_run(State::new());
5847 let claim = store.claim_step("stale-worker", &run_id, 60.0).unwrap();
5848 let cancelled = store.cancel_run(&run_id, "failure injection").unwrap();
5849 let stale_rejected = store
5850 .commit_state_patch(&run_id, &step_id, &claim.lease_token, 0, State::new())
5851 .is_err();
5852 let fresh = store.claim_step("fresh-worker", &run_id, 60.0).is_err();
5853 let status = store.run(&run_id).map(|run| run.status).unwrap_or_default();
5854 let passed = cancelled == 1 && stale_rejected && fresh && status == "cancelled";
5855 failure_check(
5856 "cancellation_fencing",
5857 passed,
5858 format!("cancelled_steps={cancelled} stale_rejected={stale_rejected}"),
5859 run_id,
5860 )
5861}
5862fn failure_side_effect_idempotency() -> FailureInjectionCheck {
5863 use std::sync::{Arc, Mutex};
5864 let calls = Arc::new(Mutex::new(0usize));
5865 let calls_for_tool = Arc::clone(&calls);
5866 let mut runtime = Runtime::new();
5867 runtime.register_tool(
5868 ToolSpec::new(
5869 "external.create",
5870 Box::new(move |_args| {
5871 let mut guard = calls_for_tool.lock().unwrap();
5872 *guard += 1;
5873 Ok(Value::Object(state(&[(
5874 "id",
5875 Value::String("EXT-1".into()),
5876 )])))
5877 }),
5878 )
5879 .side_effect("external")
5880 .idempotency_required(true),
5881 );
5882 let (run_id, _) = runtime.create_run(State::new());
5883 let ctx = failure_claim_context(&mut runtime, &run_id, "worker-1", "FailureInjector");
5884 let _ = runtime.call_tool(
5885 &ctx,
5886 "external.create",
5887 state(&[("title", Value::String("once".into()))]),
5888 );
5889 runtime
5890 .store
5891 .mark_retry(&run_id, &ctx.step_id, "RetryableAgentError", "retryable");
5892 let ctx2 = failure_claim_context(&mut runtime, &run_id, "worker-2", "FailureInjector");
5893 let _ = runtime.call_tool(
5894 &ctx2,
5895 "external.create",
5896 state(&[("title", Value::String("once".into()))]),
5897 );
5898 let count = *calls.lock().unwrap();
5899 failure_check(
5900 "side_effect_idempotency",
5901 count == 1,
5902 format!("external_call_count={count}"),
5903 run_id,
5904 )
5905}
5906
5907fn failure_claim_context(
5908 runtime: &mut Runtime,
5909 run_id: &str,
5910 worker_id: &str,
5911 agent_role: &str,
5912) -> AgentContext {
5913 let claim = runtime.store.claim_step(worker_id, run_id, 60.0).unwrap();
5914 AgentContext {
5915 run_id: claim.run_id,
5916 session_id: claim.session_id,
5917 step_id: claim.step_id,
5918 agent_role: agent_role.to_string(),
5919 lease_token: claim.lease_token,
5920 attempt: claim.attempt,
5921 state_version: claim.state_version,
5922 pending_patch: State::new(),
5923 }
5924}
5925
5926#[derive(Clone, Debug)]
5927pub struct ShadowReport {
5928 pub source_run_id: String,
5929 pub shadow_run_id: String,
5930 pub ok: bool,
5931 pub state_diff: State,
5932}
5933
5934pub fn diff_states(source: &State, shadow: &State) -> State {
5935 let mut changed = State::new();
5936 for key in source.keys().chain(shadow.keys()) {
5937 if changed.contains_key(key) {
5938 continue;
5939 }
5940 if source.get(key) != shadow.get(key) {
5941 changed.insert(
5942 key.clone(),
5943 Value::Object(state(&[
5944 ("source", source.get(key).cloned().unwrap_or_default()),
5945 ("shadow", shadow.get(key).cloned().unwrap_or_default()),
5946 ])),
5947 );
5948 }
5949 }
5950 state(&[
5951 ("changed", Value::Object(changed.clone())),
5952 ("changed_count", Value::Number(changed.len() as f64)),
5953 ])
5954}
5955
5956pub fn shadow_report(
5957 source_run_id: &str,
5958 shadow_run_id: &str,
5959 ok: bool,
5960 source_state: &State,
5961 shadow_state: &State,
5962) -> ShadowReport {
5963 ShadowReport {
5964 source_run_id: source_run_id.into(),
5965 shadow_run_id: shadow_run_id.into(),
5966 ok,
5967 state_diff: diff_states(source_state, shadow_state),
5968 }
5969}
5970
5971pub fn builtin_golden_names() -> Vec<String> {
5972 vec![
5973 "media-stream-checkpoint".into(),
5974 "minimal-success".into(),
5975 "tool-ledger-success".into(),
5976 ]
5977}
5978
5979pub fn builtin_golden_evidence(name: &str) -> Result<EvidenceBundle> {
5980 match name {
5981 "minimal-success" => golden_minimal_success(),
5982 "tool-ledger-success" => golden_tool_ledger_success(),
5983 "media-stream-checkpoint" => golden_media_stream_checkpoint(),
5984 _ => Err(RuntimeError(format!(
5985 "unknown built-in golden case: {name}"
5986 ))),
5987 }
5988}
5989
5990pub fn golden_regression(golden: &EvidenceBundle, current: &EvidenceBundle) -> EvidenceCheckReport {
5991 evaluate_evidence_regression(golden, current, None)
5992}
5993
5994fn golden_minimal_success() -> Result<EvidenceBundle> {
5995 let mut runtime = Runtime::new();
5996 let (run_id, _) = runtime.create_run(State::new());
5997 runtime.run_once(
5998 &run_id,
5999 "golden-worker",
6000 "GoldenAgent",
6001 60.0,
6002 |ctx, _state| {
6003 ctx.write_state("answer", Value::String("ok".into()));
6004 Ok(())
6005 },
6006 )?;
6007 export_evidence(&runtime.store, &run_id)
6008}
6009fn golden_tool_ledger_success() -> Result<EvidenceBundle> {
6010 let mut runtime = Runtime::new();
6011 runtime.register_tool(
6012 ToolSpec::new(
6013 "github.create_issue",
6014 Box::new(|_args| {
6015 Ok(Value::Object(state(&[(
6016 "issue_id",
6017 Value::String("ISSUE-1".into()),
6018 )])))
6019 }),
6020 )
6021 .side_effect("external"),
6022 );
6023 let (run_id, _) = runtime.create_run(State::new());
6024 runtime.run_once(
6025 &run_id,
6026 "golden-worker",
6027 "ExecutorAgent",
6028 60.0,
6029 |ctx, _state| {
6030 ctx.write_state("issue_id", Value::String("ISSUE-1".into()));
6031 Ok(())
6032 },
6033 )?;
6034 export_evidence(&runtime.store, &run_id)
6035}
6036fn golden_media_stream_checkpoint() -> Result<EvidenceBundle> {
6037 let mut runtime = Runtime::new();
6038 let (run_id, _) = runtime.create_run(State::new());
6039 runtime.run_once(
6040 &run_id,
6041 "golden-worker",
6042 "MediaAgent",
6043 60.0,
6044 |ctx, _state| {
6045 ctx.write_state("processed_offset", Value::Number(42.0));
6046 Ok(())
6047 },
6048 )?;
6049 runtime.store.create_artifact(
6050 &run_id,
6051 None,
6052 "golden-video-frame",
6053 State::new(),
6054 state(&[(
6055 "agentledger_media",
6056 Value::Object(state(&[
6057 ("kind", Value::String("frame".into())),
6058 ("uri", Value::String("file://golden-frame.jpg".into())),
6059 ])),
6060 )]),
6061 );
6062 runtime.store.create_artifact(
6063 &run_id,
6064 None,
6065 "golden-stream-checkpoint",
6066 State::new(),
6067 state(&[(
6068 "agentledger_stream",
6069 Value::Object(state(&[
6070 ("stream_id", Value::String("stream-golden".into())),
6071 ("consumer_id", Value::String("consumer-golden".into())),
6072 ("offset", Value::Number(42.0)),
6073 ])),
6074 )]),
6075 );
6076 export_evidence(&runtime.store, &run_id)
6077}
6078
6079#[derive(Clone, Debug)]
6080pub struct TimeTravelFrame {
6081 pub seq: u64,
6082 pub event_id: String,
6083 pub event_type: String,
6084 pub step_id: Option<String>,
6085 pub agent_role: Option<String>,
6086 pub state_version: Option<u64>,
6087 pub timestamp: f64,
6088 pub state_changed: bool,
6089 pub changed_keys: Vec<String>,
6090 pub patch: Option<State>,
6091 pub state_after: Option<State>,
6092}
6093
6094#[derive(Clone, Debug)]
6095pub struct TimeTravelReport {
6096 pub run_id: String,
6097 pub at_seq: Option<u64>,
6098 pub event_count: usize,
6099 pub timeline: Vec<TimeTravelFrame>,
6100 pub state_at_seq: State,
6101 pub selected_event: Option<TimeTravelFrame>,
6102}
6103
6104pub fn time_travel(
6105 bundle: &EvidenceBundle,
6106 at_seq: Option<u64>,
6107 include_states: bool,
6108) -> TimeTravelReport {
6109 let mut current = State::new();
6110 let mut state_at_seq = State::new();
6111 let mut selected_event = None;
6112 let mut timeline = Vec::new();
6113 for event in &bundle.events {
6114 let before = current.clone();
6115 let patch = patch_for_time_travel_event(event);
6116 if let Some(patch_value) = &patch {
6117 for (key, value) in patch_value {
6118 current.insert(key.clone(), value.clone());
6119 }
6120 }
6121 let diff = diff_states(&before, ¤t);
6122 let changed_keys = match diff.get("changed") {
6123 Some(Value::Object(obj)) => obj.keys().cloned().collect(),
6124 _ => Vec::new(),
6125 };
6126 let frame = TimeTravelFrame {
6127 seq: event.seq,
6128 event_id: event.event_id.clone(),
6129 event_type: event.event_type.clone(),
6130 step_id: event.step_id.clone(),
6131 agent_role: event.agent_role.clone(),
6132 state_version: event.state_version,
6133 timestamp: event.timestamp,
6134 state_changed: diff.get("changed_count") != Some(&Value::Number(0.0)),
6135 changed_keys,
6136 patch,
6137 state_after: if include_states {
6138 Some(current.clone())
6139 } else {
6140 None
6141 },
6142 };
6143 if at_seq.is_some_and(|seq| event.seq <= seq) {
6144 state_at_seq = current.clone();
6145 selected_event = Some(frame.clone());
6146 }
6147 timeline.push(frame);
6148 }
6149 if at_seq.is_none() {
6150 state_at_seq = current.clone();
6151 }
6152 TimeTravelReport {
6153 run_id: bundle.run.run_id.clone(),
6154 at_seq,
6155 event_count: timeline.len(),
6156 timeline,
6157 state_at_seq,
6158 selected_event,
6159 }
6160}
6161
6162fn patch_for_time_travel_event(event: &Event) -> Option<State> {
6163 if event.event_type == "run_created" {
6164 if let Some(Value::Object(obj)) = event.payload.get("initial_state") {
6165 return Some(obj.clone());
6166 }
6167 return Some(State::new());
6168 }
6169 if event.event_type == "state_committed"
6170 || event.event_type == "state_patch_committed"
6171 || event.event_type == "system_state_patch_applied"
6172 {
6173 if let Some(Value::Object(obj)) = event.payload.get("patch") {
6174 return Some(obj.clone());
6175 }
6176 return Some(State::new());
6177 }
6178 None
6179}
6180
6181pub fn time_travel_html(report: &TimeTravelReport) -> String {
6182 let rows = report
6183 .timeline
6184 .iter()
6185 .map(|frame| {
6186 format!(
6187 "<tr><td>{}</td><td>{}</td><td>{}</td></tr>",
6188 frame.seq,
6189 frame.event_type,
6190 frame.changed_keys.join(", ")
6191 )
6192 })
6193 .collect::<Vec<_>>()
6194 .join("\n");
6195 format!("<!doctype html><html><head><meta charset=\"utf-8\"><title>AgentLedger Time Travel Report</title></head><body><h1>AgentLedger Time Travel Report</h1><p>Run <code>{}</code></p><table>{}</table><h2>State At Selected Point</h2><pre>{:?}</pre><h2>Selected Event</h2><pre>{:?}</pre></body></html>", report.run_id, rows, report.state_at_seq, report.selected_event)
6196}
6197
6198#[derive(Debug, Clone, PartialEq, Eq)]
6199pub struct OptionalAdapterCapability {
6200 pub name: String,
6201 pub category: String,
6202 pub core_imports_heavy_sdks: bool,
6203 pub adapter_is_optional: bool,
6204 pub fail_closed_without_adapter: bool,
6205 pub contract_surface: Vec<String>,
6206}
6207
6208pub fn optional_adapter_capabilities() -> Vec<OptionalAdapterCapability> {
6209 fn item(name: &str, category: &str, surface: &[&str]) -> OptionalAdapterCapability {
6210 OptionalAdapterCapability {
6211 name: name.to_string(),
6212 category: category.to_string(),
6213 core_imports_heavy_sdks: false,
6214 adapter_is_optional: true,
6215 fail_closed_without_adapter: true,
6216 contract_surface: surface.iter().map(|s| s.to_string()).collect(),
6217 }
6218 }
6219 vec![
6220 item("postgres", "storage", &["ddl_for", "migrations_for", "state_store"]),
6221 item("mysql", "storage", &["ddl_for", "migrations_for", "state_store"]),
6222 item("s3", "blobstore", &["put_json", "get_json", "content_address"]),
6223 item("docker", "sandbox", &["sandbox_policy", "sandbox_result", "tool_gateway"]),
6224 item("e2b", "sandbox", &["sandbox_policy", "sandbox_result", "tool_gateway"]),
6225 item("bubblewrap", "sandbox", &["sandbox_policy", "sandbox_result", "tool_gateway"]),
6226 item("kubernetes", "sandbox", &["sandbox_policy", "sandbox_result", "tool_gateway"]),
6227 item("gvisor", "sandbox", &["sandbox_policy", "sandbox_result", "tool_gateway"]),
6228 item("firecracker", "sandbox", &["sandbox_policy", "sandbox_result", "tool_gateway"]),
6229 item("langgraph", "framework", &["framework_adapter", "checkpoint_contract"]),
6230 item("langchain", "framework", &["framework_adapter"]),
6231 item("crewai", "framework", &["framework_adapter"]),
6232 item("autogen", "framework", &["framework_adapter"]),
6233 item("openai-agents-sdk", "framework", &["framework_adapter"]),
6234 item("llamaindex", "framework", &["framework_adapter"]),
6235 item("semantic-kernel", "framework", &["framework_adapter"]),
6236 item("mcp-transport", "mcp", &["mcp_tool_descriptor", "mcp_resource_descriptor"]),
6237 item("langfuse", "observability", &["evidence_bundle", "trace_payload", "correlation_ids"]),
6238 item("shadow-runner", "shadow", &["evidence_bundle", "tool_ledger", "state_diff"]),
6239 ]
6240}
6241
6242pub trait SqlExecutor {
6243 fn exec(&mut self, sql: &str, params: &[Value]) -> Result<()>;
6244}
6245
6246pub struct PostgresAdapter<C: SqlExecutor> {
6247 pub schema: String,
6248 pub client: C,
6249}
6250
6251impl<C: SqlExecutor> PostgresAdapter<C> {
6252 pub fn new(client: C, schema: &str) -> Self {
6253 Self { schema: if schema.is_empty() { "agentledger".to_string() } else { schema.to_string() }, client }
6254 }
6255 pub fn migration_plan(&self) -> Result<Vec<Migration>> { migrations_for("postgres") }
6256 pub fn apply_migrations(&mut self) -> Result<()> {
6257 self.client.exec(&ddl_for("postgres")?, &[])?;
6258 for migration in self.migration_plan()? {
6259 self.client.exec(
6260 "INSERT INTO schema_migrations(version, name, checksum) VALUES ($1, $2, $3) ON CONFLICT (version) DO NOTHING",
6261 &[Value::String(migration.version), Value::String(migration.name), Value::String(stable_hash(&migration.sql))],
6262 )?;
6263 }
6264 Ok(())
6265 }
6266}
6267
6268pub struct MySQLAdapter<C: SqlExecutor> {
6269 pub database: String,
6270 pub client: C,
6271}
6272
6273impl<C: SqlExecutor> MySQLAdapter<C> {
6274 pub fn new(client: C, database: &str) -> Self {
6275 Self { database: if database.is_empty() { "agentledger".to_string() } else { database.to_string() }, client }
6276 }
6277 pub fn migration_plan(&self) -> Result<Vec<Migration>> { migrations_for("mysql") }
6278 pub fn apply_migrations(&mut self) -> Result<()> {
6279 self.client.exec(&ddl_for("mysql")?, &[])?;
6280 for migration in self.migration_plan()? {
6281 self.client.exec(
6282 "INSERT INTO schema_migrations(version, name, checksum, applied_at) VALUES (?, ?, ?, UNIX_TIMESTAMP()) ON DUPLICATE KEY UPDATE version=version",
6283 &[Value::String(migration.version), Value::String(migration.name), Value::String(stable_hash(&migration.sql))],
6284 )?;
6285 }
6286 Ok(())
6287 }
6288}
6289
6290pub trait ObjectClient {
6291 fn put_object(&mut self, bucket: &str, key: &str, body: &[u8], content_type: &str, metadata: State) -> Result<()>;
6292 fn get_object(&mut self, bucket: &str, key: &str) -> Result<Vec<u8>>;
6293}
6294
6295pub struct S3BlobStoreAdapter<C: ObjectClient> {
6296 pub bucket: String,
6297 pub prefix: String,
6298 pub client: C,
6299}
6300
6301impl<C: ObjectClient> S3BlobStoreAdapter<C> {
6302 pub fn new(client: C, bucket: &str, prefix: &str) -> Self {
6303 Self { bucket: bucket.to_string(), prefix: if prefix.is_empty() { "agentledger/blobs".to_string() } else { prefix.trim_matches('/').to_string() }, client }
6304 }
6305 pub fn put_json(&mut self, value: &Value) -> Result<(String, String)> {
6306 let encoded = encode_value(value);
6307 let digest = stable_hash(&encoded);
6308 let key = format!("{}/sha256/{}.json", self.prefix, digest);
6309 let mut metadata = State::new();
6310 metadata.insert("agentledger-digest".to_string(), Value::String(format!("sha256:{digest}")));
6311 self.client.put_object(&self.bucket, &key, encoded.as_bytes(), "application/json", metadata)?;
6312 Ok((format!("sha256:{digest}"), format!("s3://{}/{}", self.bucket, key)))
6313 }
6314 pub fn get_json(&mut self, reference: &str) -> Result<Value> {
6315 let prefix = format!("s3://{}/", self.bucket);
6316 if !reference.starts_with(&prefix) || reference.contains("..") {
6317 return Err(RuntimeError(format!("unsupported s3 blob ref: {reference}")));
6318 }
6319 let key = &reference[prefix.len()..];
6320 let body = self.client.get_object(&self.bucket, key)?;
6321 let text = String::from_utf8(body).map_err(|err| RuntimeError(err.to_string()))?;
6322 decode_value(&text)
6323 }
6324}
6325
6326pub trait OtlpClient { fn post_json(&mut self, endpoint: &str, payload: &str, content_type: &str) -> Result<()>; }
6327pub struct OtlpTransport<C: OtlpClient> { pub endpoint: String, pub client: C }
6328impl<C: OtlpClient> OtlpTransport<C> { pub fn export(&mut self, payload: &str) -> Result<()> { self.client.post_json(&self.endpoint, payload, "application/json") } }
6329
6330pub struct DockerSandboxAdapter { pub image: String }
6331impl DockerSandboxAdapter {
6332 pub fn manifest(&self, policy: &State, command: Vec<String>) -> State {
6333 let mut out = State::new();
6334 out.insert("backend".to_string(), Value::String("docker".to_string()));
6335 out.insert("image".to_string(), Value::String(if self.image.is_empty() { "python:3.11-slim".to_string() } else { self.image.clone() }));
6336 let network = match policy.get("network") { Some(Value::String(value)) if value != "deny" => value.clone(), _ => "none".to_string() };
6337 out.insert("network".to_string(), Value::String(network));
6338 out.insert("read_only_root".to_string(), Value::Bool(true));
6339 out.insert("requires_explicit_execution".to_string(), Value::Bool(true));
6340 out.insert("command".to_string(), Value::Array(command.into_iter().map(Value::String).collect()));
6341 out
6342 }
6343}
6344
6345pub struct DockerSandboxExecutor {
6346 pub image: String,
6347 pub binary: String,
6348 pub allow_command_execution: bool,
6349 pub allow_shell: bool,
6350 pub shell: String,
6351 pub memory: String,
6352 pub cpus: String,
6353}
6354
6355impl DockerSandboxExecutor {
6356 pub fn new(image: &str, allow_command_execution: bool) -> Self {
6357 Self {
6358 image: image.to_string(),
6359 binary: "docker".to_string(),
6360 allow_command_execution,
6361 allow_shell: false,
6362 shell: "/bin/sh".to_string(),
6363 memory: String::new(),
6364 cpus: String::new(),
6365 }
6366 }
6367
6368 pub fn with_binary(mut self, binary: &str) -> Self {
6369 self.binary = binary.to_string();
6370 self
6371 }
6372
6373 fn extract_command(&self, args: &State) -> std::result::Result<Vec<String>, String> {
6374 let raw = args.get("_sandbox_command").or_else(|| args.get("command"));
6375 match raw {
6376 Some(Value::String(command)) => {
6377 if !self.allow_shell {
6378 Err("string commands require allow_shell=true; pass argv list in `_sandbox_command` instead".to_string())
6379 } else {
6380 let shell = if self.shell.is_empty() { "/bin/sh" } else { &self.shell };
6381 Ok(vec![shell.to_string(), "-lc".to_string(), command.clone()])
6382 }
6383 }
6384 Some(Value::Array(items)) => {
6385 let mut command = Vec::new();
6386 for item in items {
6387 match item {
6388 Value::String(value) if !value.is_empty() => command.push(value.clone()),
6389 _ => return Err("_sandbox_command must be a non-empty string array".to_string()),
6390 }
6391 }
6392 if command.is_empty() {
6393 Err("_sandbox_command must be a non-empty string array".to_string())
6394 } else {
6395 Ok(command)
6396 }
6397 }
6398 _ => Err("external sandbox tools require a command-style `_sandbox_command` arg".to_string()),
6399 }
6400 }
6401
6402 fn docker_argv(&self, policy: &SandboxPolicy, command: &[String]) -> Vec<String> {
6403 let image = if self.image.is_empty() { "python:3.11-slim" } else { &self.image };
6404 let network = if policy.network == "deny" || policy.network.is_empty() { "none" } else { &policy.network };
6405 let mut argv = vec![
6406 self.binary.clone(),
6407 "run".to_string(),
6408 "--rm".to_string(),
6409 "--network".to_string(),
6410 network.to_string(),
6411 "--read-only".to_string(),
6412 ];
6413 if !self.memory.is_empty() {
6414 argv.extend(["--memory".to_string(), self.memory.clone()]);
6415 }
6416 if !self.cpus.is_empty() {
6417 argv.extend(["--cpus".to_string(), self.cpus.clone()]);
6418 }
6419 argv.push(image.to_string());
6420 argv.extend(command.iter().cloned());
6421 argv
6422 }
6423
6424 fn result_error(policy: &SandboxPolicy, manifest: State, error_type: &str, error: String) -> SandboxResult {
6425 let mut metadata = State::new();
6426 metadata.insert("executor".to_string(), Value::String(policy.executor.clone()));
6427 metadata.insert("isolation_level".to_string(), Value::String("container".to_string()));
6428 metadata.insert("manifest".to_string(), Value::Object(manifest));
6429 metadata.insert("error_type".to_string(), Value::String(error_type.to_string()));
6430 SandboxResult { ok: false, output: Value::Null, error: Some(error), metadata }
6431 }
6432}
6433
6434impl SandboxExecutor for DockerSandboxExecutor {
6435 fn run_tool(&self, args: State, policy: &SandboxPolicy) -> SandboxResult {
6436 let command = match self.extract_command(&args) {
6437 Ok(command) => command,
6438 Err(error) => return Self::result_error(policy, State::new(), "InvalidSandboxCommand", error),
6439 };
6440 let mut policy_state = State::new();
6441 policy_state.insert("network".to_string(), Value::String(policy.network.clone()));
6442 let manifest = (DockerSandboxAdapter { image: self.image.clone() }).manifest(&policy_state, command.clone());
6443 if !self.allow_command_execution {
6444 return Self::result_error(policy, manifest, "SandboxAdapterNotInstalled", "command execution is not enabled for this executor".to_string());
6445 }
6446 let argv = self.docker_argv(policy, &command);
6447 let mut cmd = Command::new(&argv[0]);
6448 cmd.args(&argv[1..]).stdout(Stdio::piped()).stderr(Stdio::piped());
6449 let output = match cmd.output() {
6450 Ok(output) => output,
6451 Err(error) => return Self::result_error(policy, manifest, "SandboxBinaryMissing", error.to_string()),
6452 };
6453 let mut value = State::new();
6454 value.insert("stdout".to_string(), Value::String(String::from_utf8_lossy(&output.stdout).to_string()));
6455 value.insert("stderr".to_string(), Value::String(String::from_utf8_lossy(&output.stderr).to_string()));
6456 value.insert("returncode".to_string(), Value::Number(output.status.code().unwrap_or(-1) as f64));
6457 let mut metadata = State::new();
6458 metadata.insert("executor".to_string(), Value::String(policy.executor.clone()));
6459 metadata.insert("isolation_level".to_string(), Value::String("container".to_string()));
6460 metadata.insert("manifest".to_string(), Value::Object(manifest));
6461 metadata.insert("executed".to_string(), Value::Bool(true));
6462 if !output.status.success() {
6463 metadata.insert("error_type".to_string(), Value::String("SandboxCommandFailed".to_string()));
6464 return SandboxResult {
6465 ok: false,
6466 output: Value::Object(value),
6467 error: Some(format!("sandbox command exited with {}", output.status.code().unwrap_or(-1))),
6468 metadata,
6469 };
6470 }
6471 SandboxResult { ok: true, output: Value::Object(value), error: None, metadata }
6472 }
6473}
6474
6475pub mod adapters {
6476 pub mod postgres {
6477 pub const PACKAGE_NAME: &str = "agentledger-postgres";
6478 pub const FEATURE: &str = "adapter-postgres";
6479 pub use crate::{migrations_for, Migration, PostgresAdapter, SqlExecutor};
6480 }
6481
6482 pub mod mysql {
6483 pub const PACKAGE_NAME: &str = "agentledger-mysql";
6484 pub const FEATURE: &str = "adapter-mysql";
6485 pub use crate::{migrations_for, Migration, MySQLAdapter, SqlExecutor};
6486 }
6487
6488 pub mod s3 {
6489 pub const PACKAGE_NAME: &str = "agentledger-s3";
6490 pub const FEATURE: &str = "adapter-s3";
6491 pub use crate::{ObjectClient, S3BlobStoreAdapter};
6492 }
6493
6494 pub mod mcp {
6495 pub const PACKAGE_NAME: &str = "agentledger-mcp";
6496 pub const FEATURE: &str = "adapter-mcp";
6497 pub use crate::{
6498 InMemoryMCPContextServer, InMemoryMCPToolServer, MCPCall, MCPContextAdapter,
6499 MCPResourceDescriptor, MCPResourceRead, MCPToolAdapter,
6500 };
6501 }
6502
6503 pub mod otel {
6504 pub const PACKAGE_NAME: &str = "agentledger-otel";
6505 pub const FEATURE: &str = "adapter-otel";
6506 pub use crate::{OtlpClient, OtlpTransport};
6507 }
6508
6509 pub mod langfuse {
6510 pub const PACKAGE_NAME: &str = "agentledger-langfuse";
6511 pub const FEATURE: &str = "adapter-langfuse";
6512 pub const CATEGORY: &str = "observability";
6513 }
6514
6515 pub mod docker {
6516 pub const PACKAGE_NAME: &str = "agentledger-sandbox-docker";
6517 pub const FEATURE: &str = "adapter-docker";
6518 pub use crate::{DockerSandboxAdapter, DockerSandboxExecutor, State, Value};
6519 }
6520
6521 pub mod framework {
6522 pub const PACKAGE_NAME: &str = "agentledger-framework";
6523 pub const FEATURE: &str = "adapter-framework";
6524 pub use crate::{FunctionAdapter, MethodFrameworkAdapter};
6525 }
6526}