1use std::collections::HashMap;
2use std::fmt;
3use std::fs;
4use std::path::{Path, PathBuf};
5use std::process::{Command, Stdio};
6use std::sync::atomic::{AtomicU64, Ordering};
7use std::time::{SystemTime, UNIX_EPOCH};
8
9static ID_COUNTER: AtomicU64 = AtomicU64::new(1);
10
11pub type State = HashMap<String, Value>;
12
13#[derive(Clone, Debug, PartialEq)]
14pub enum Value {
15 Null,
16 Bool(bool),
17 Number(f64),
18 String(String),
19 Object(State),
20 Array(Vec<Value>),
21}
22
23impl Default for Value {
24 fn default() -> Self {
25 Value::Null
26 }
27}
28
29impl From<&str> for Value {
30 fn from(value: &str) -> Self {
31 Value::String(value.to_string())
32 }
33}
34
35impl From<String> for Value {
36 fn from(value: String) -> Self {
37 Value::String(value)
38 }
39}
40
41impl From<bool> for Value {
42 fn from(value: bool) -> Self {
43 Value::Bool(value)
44 }
45}
46
47impl From<i64> for Value {
48 fn from(value: i64) -> Self {
49 Value::Number(value as f64)
50 }
51}
52
53impl From<f64> for Value {
54 fn from(value: f64) -> Self {
55 Value::Number(value)
56 }
57}
58
59#[derive(Clone, Debug)]
60pub struct Run {
61 pub run_id: String,
62 pub session_id: String,
63 pub status: String,
64 pub state: State,
65 pub state_version: u64,
66 pub created_at: f64,
67 pub updated_at: f64,
68}
69
70#[derive(Clone, Debug)]
71pub struct Step {
72 pub step_id: String,
73 pub run_id: String,
74 pub session_id: String,
75 pub status: String,
76 pub owner: Option<String>,
77 pub lease_token: Option<String>,
78 pub lease_until: Option<f64>,
79 pub attempt: u64,
80 pub state_version: u64,
81 pub checkpoint_id: Option<String>,
82 pub last_error_type: Option<String>,
83 pub last_error: Option<String>,
84 pub cancelled_at: Option<f64>,
85 pub created_at: f64,
86 pub updated_at: f64,
87}
88
89#[derive(Clone, Debug)]
90pub struct StepClaim {
91 pub run_id: String,
92 pub session_id: String,
93 pub step_id: String,
94 pub attempt: u64,
95 pub lease_token: String,
96 pub state_version: u64,
97 pub lease_until: f64,
98}
99
100#[derive(Clone, Debug)]
101pub struct Event {
102 pub event_id: String,
103 pub run_id: String,
104 pub session_id: Option<String>,
105 pub step_id: Option<String>,
106 pub seq: u64,
107 pub event_type: String,
108 pub timestamp: f64,
109 pub agent_role: Option<String>,
110 pub state_version: Option<u64>,
111 pub causal_token: Option<String>,
112 pub payload_hash: String,
113 pub payload_ref: String,
114 pub payload: State,
115}
116
117#[derive(Clone, Debug)]
118pub struct ToolLedgerEntry {
119 pub ledger_id: String,
120 pub run_id: String,
121 pub session_id: String,
122 pub step_id: String,
123 pub tool_name: String,
124 pub tool_version: String,
125 pub tool_call_id: String,
126 pub idempotency_key: String,
127 pub causal_token: String,
128 pub request_hash: String,
129 pub request_ref: String,
130 pub status: String,
131 pub external_id: Option<String>,
132 pub response_hash: Option<String>,
133 pub response_ref: Option<String>,
134 pub error_type: Option<String>,
135 pub response: Option<Value>,
136 pub created_at: f64,
137 pub updated_at: f64,
138}
139
140#[derive(Clone, Debug)]
141pub struct ApprovalRequest {
142 pub approval_id: String,
143 pub approval_key: String,
144 pub run_id: String,
145 pub session_id: String,
146 pub step_id: String,
147 pub tool_name: String,
148 pub risk_level: String,
149 pub status: String,
150 pub reason: String,
151 pub request_hash: String,
152 pub request_ref: String,
153 pub requested_by: String,
154 pub approved_by: Option<String>,
155 pub decision_reason: Option<String>,
156 pub created_at: f64,
157 pub updated_at: f64,
158}
159
160#[derive(Clone, Debug)]
161pub struct CostRecord {
162 pub cost_id: String,
163 pub run_id: String,
164 pub session_id: String,
165 pub step_id: String,
166 pub category: String,
167 pub name: String,
168 pub amount: f64,
169 pub unit: String,
170 pub metadata: State,
171 pub created_at: f64,
172}
173
174#[derive(Clone, Debug, Default, PartialEq)]
175pub struct CostSummary {
176 pub tool_calls: f64,
177 pub model_tokens: f64,
178 pub total_usd: f64,
179 pub by_category: HashMap<String, f64>,
180}
181
182pub const MEDIA_SCHEMA_VERSION: &str = "agentledger.media.v0";
183pub const STREAM_SCHEMA_VERSION: &str = "agentledger.stream.v0";
184
185#[derive(Clone, Debug)]
186pub struct LocalBlobStore {
187 root: PathBuf,
188}
189
190impl LocalBlobStore {
191 pub fn open(root: impl AsRef<Path>) -> Result<Self> {
192 fs::create_dir_all(root.as_ref()).map_err(|err| RuntimeError(err.to_string()))?;
193 Ok(Self {
194 root: root.as_ref().to_path_buf(),
195 })
196 }
197
198 pub fn put_json(&self, value: &Value) -> Result<(String, String)> {
199 let encoded = encode_value(value);
200 let digest = stable_hash(&encoded);
201 let dir = self.root.join("sha256");
202 fs::create_dir_all(&dir).map_err(|err| RuntimeError(err.to_string()))?;
203 let path = dir.join(format!("{digest}.json"));
204 if !path.exists() {
205 let tmp = path.with_extension("json.tmp");
206 fs::write(&tmp, encoded).map_err(|err| RuntimeError(err.to_string()))?;
207 fs::rename(&tmp, &path).map_err(|err| RuntimeError(err.to_string()))?;
208 }
209 Ok((
210 format!("sha256:{digest}"),
211 format!("blob://sha256/{digest}.json"),
212 ))
213 }
214
215 pub fn get_json(&self, reference: &str) -> Result<Value> {
216 let prefix = "blob://sha256/";
217 if !reference.starts_with(prefix) {
218 return Err(RuntimeError(format!("unsupported blob ref: {reference}")));
219 }
220 let name = &reference[prefix.len()..];
221 if !name.ends_with(".json")
222 || name.contains("..")
223 || name.contains('/')
224 || name.contains('\\')
225 {
226 return Err(RuntimeError(format!("unsupported blob ref: {reference}")));
227 }
228 let body = fs::read_to_string(self.root.join("sha256").join(name))
229 .map_err(|err| RuntimeError(err.to_string()))?;
230 decode_value(&body)
231 }
232}
233
234#[derive(Clone, Debug)]
235pub struct Artifact {
236 pub artifact_id: String,
237 pub run_id: String,
238 pub step_id: Option<String>,
239 pub name: String,
240 pub blob_hash: String,
241 pub blob_ref: String,
242 pub metadata: State,
243 pub created_at: f64,
244}
245
246#[derive(Clone, Debug, Default)]
247pub struct MediaArtifactOptions {
248 pub uri: Option<String>,
249 pub content_ref: Option<String>,
250 pub media_metadata: State,
251 pub lineage: State,
252 pub derived_outputs: State,
253 pub metadata: State,
254}
255
256#[derive(Clone, Debug, Default)]
257pub struct StreamChunkRef {
258 pub stream_id: String,
259 pub chunk_id: String,
260 pub offset: Value,
261 pub content_ref: Option<String>,
262 pub content_hash: Option<String>,
263 pub sequence: Option<f64>,
264 pub event_time: Option<f64>,
265 pub metadata: State,
266}
267
268#[derive(Clone, Debug, Default)]
269pub struct StreamCheckpointOptions {
270 pub stream_id: String,
271 pub consumer_id: String,
272 pub offset: Value,
273 pub watermark: Option<Value>,
274 pub chunk: Option<StreamChunkRef>,
275 pub partial_result_ref: Option<String>,
276 pub backpressure: State,
277 pub metadata: State,
278}
279
280#[derive(Clone, Debug, Default)]
281pub struct BudgetLimits {
282 pub max_tool_calls: Option<f64>,
283 pub max_model_tokens: Option<f64>,
284 pub max_total_usd: Option<f64>,
285}
286
287#[derive(Debug, Clone)]
288pub struct RuntimeError(pub String);
289
290impl fmt::Display for RuntimeError {
291 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
292 write!(f, "{}", self.0)
293 }
294}
295
296impl std::error::Error for RuntimeError {}
297
298pub type Result<T> = std::result::Result<T, RuntimeError>;
299pub type ToolFunc = Box<dyn Fn(State) -> Result<Value> + Send + Sync>;
300pub type AgentFunc = fn(&mut AgentContext, State) -> Result<()>;
301
302pub struct SandboxPolicy {
303 pub tool_name: String,
304 pub run_id: String,
305 pub step_id: String,
306 pub executor: String,
307 pub network: String,
308 pub filesystem: String,
309 pub timeout_seconds: u64,
310 pub extra: State,
311}
312
313pub struct SandboxResult {
314 pub ok: bool,
315 pub output: Value,
316 pub error: Option<String>,
317 pub metadata: State,
318}
319
320pub trait SandboxExecutor {
321 fn run_tool(&self, args: State, policy: &SandboxPolicy) -> SandboxResult;
322}
323
324pub struct DisabledSandboxExecutor;
325
326impl SandboxExecutor for DisabledSandboxExecutor {
327 fn run_tool(&self, _args: State, policy: &SandboxPolicy) -> SandboxResult {
328 let mut metadata = State::new();
329 metadata.insert("executor".to_string(), Value::String(policy.executor.clone()));
330 metadata.insert("isolation_level".to_string(), Value::String("none".to_string()));
331 metadata.insert("fail_closed".to_string(), Value::Bool(true));
332 SandboxResult {
333 ok: false,
334 output: Value::Null,
335 error: Some(format!("sandbox executor \"{}\" is disabled", policy.executor)),
336 metadata,
337 }
338 }
339}
340
341pub struct ToolSpec {
342 pub name: String,
343 pub version: String,
344 pub side_effect: String,
345 pub risk_level: String,
346 pub idempotency_required: bool,
347 pub approval_required: bool,
348 pub sandbox_required: bool,
349 pub sandbox_executor: String,
350 pub sandbox_policy: State,
351 pub input_schema: Option<Value>,
352 pub output_schema: Option<Value>,
353 pub func: ToolFunc,
354}
355
356impl ToolSpec {
357 pub fn new(name: &str, func: ToolFunc) -> Self {
358 Self {
359 name: name.to_string(),
360 version: "v1".to_string(),
361 side_effect: "none".to_string(),
362 risk_level: "low".to_string(),
363 idempotency_required: false,
364 approval_required: false,
365 sandbox_required: false,
366 sandbox_executor: String::new(),
367 sandbox_policy: State::new(),
368 input_schema: None,
369 output_schema: None,
370 func,
371 }
372 }
373
374 pub fn side_effect(mut self, side_effect: &str) -> Self {
375 self.side_effect = side_effect.to_string();
376 self
377 }
378
379 pub fn risk_level(mut self, risk_level: &str) -> Self {
380 self.risk_level = risk_level.to_string();
381 self
382 }
383
384 pub fn idempotency_required(mut self, required: bool) -> Self {
385 self.idempotency_required = required;
386 self
387 }
388
389 pub fn approval_required(mut self, required: bool) -> Self {
390 self.approval_required = required;
391 self
392 }
393
394 pub fn sandbox_required(mut self, required: bool) -> Self {
395 self.sandbox_required = required;
396 self
397 }
398
399 pub fn sandbox_executor(mut self, executor: &str) -> Self {
400 self.sandbox_executor = executor.to_string();
401 self
402 }
403
404 pub fn sandbox_policy(mut self, policy: State) -> Self {
405 self.sandbox_policy = policy;
406 self
407 }
408
409 pub fn input_schema(mut self, schema: Value) -> Self {
410 self.input_schema = Some(schema);
411 self
412 }
413
414 pub fn output_schema(mut self, schema: Value) -> Self {
415 self.output_schema = Some(schema);
416 self
417 }
418}
419
420#[derive(Default)]
421pub struct MemoryStore {
422 runs: HashMap<String, Run>,
423 steps: HashMap<String, Step>,
424 events: HashMap<String, Vec<Event>>,
425 tool_ledger: HashMap<String, ToolLedgerEntry>,
426 approval_requests: HashMap<String, ApprovalRequest>,
427 cost_records: HashMap<String, Vec<CostRecord>>,
428 artifacts: HashMap<String, Vec<Artifact>>,
429}
430
431impl MemoryStore {
432 pub fn new() -> Self {
433 Self::default()
434 }
435
436 pub fn save_to_path(&self, path: impl AsRef<Path>) -> Result<()> {
437 fs::write(path, encode_store(self)).map_err(|err| RuntimeError(err.to_string()))
438 }
439
440 pub fn load_from_path(path: impl AsRef<Path>) -> Result<Self> {
441 let body = fs::read_to_string(path).map_err(|err| RuntimeError(err.to_string()))?;
442 decode_store(&body)
443 }
444
445 pub fn create_run(&mut self, initial_state: State) -> (String, String) {
446 let run_id = new_id("run");
447 let session_id = new_id("sess");
448 let step_id = new_id("step");
449 let now = now_seconds();
450 self.runs.insert(
451 run_id.clone(),
452 Run {
453 run_id: run_id.clone(),
454 session_id: session_id.clone(),
455 status: "pending".to_string(),
456 state: initial_state.clone(),
457 state_version: 0,
458 created_at: now,
459 updated_at: now,
460 },
461 );
462 self.steps.insert(
463 step_id.clone(),
464 Step {
465 step_id: step_id.clone(),
466 run_id: run_id.clone(),
467 session_id: session_id.clone(),
468 status: "pending".to_string(),
469 owner: None,
470 lease_token: None,
471 lease_until: None,
472 attempt: 0,
473 state_version: 0,
474 checkpoint_id: None,
475 last_error_type: None,
476 last_error: None,
477 cancelled_at: None,
478 created_at: now,
479 updated_at: now,
480 },
481 );
482 let mut payload = State::new();
483 payload.insert("initial_state".to_string(), Value::Object(initial_state));
484 self.append_event(
485 &run_id,
486 Some(&session_id),
487 None,
488 "run_created",
489 payload,
490 None,
491 None,
492 None,
493 );
494 let mut step_payload = State::new();
495 step_payload.insert("step_id".to_string(), Value::String(step_id.clone()));
496 self.append_event(
497 &run_id,
498 Some(&session_id),
499 Some(&step_id),
500 "step_created",
501 step_payload,
502 None,
503 None,
504 None,
505 );
506 (run_id, step_id)
507 }
508
509 pub fn claim_step(
510 &mut self,
511 worker_id: &str,
512 run_id: &str,
513 lease_seconds: f64,
514 ) -> Result<StepClaim> {
515 let mut candidates: Vec<Step> = self
516 .steps
517 .values()
518 .filter(|step| {
519 (run_id.is_empty() || step.run_id == run_id)
520 && (step.status == "pending" || step.status == "retry_scheduled")
521 })
522 .cloned()
523 .collect();
524 candidates.sort_by(|a, b| {
525 a.created_at
526 .partial_cmp(&b.created_at)
527 .unwrap()
528 .then(a.step_id.cmp(&b.step_id))
529 });
530 let candidate = candidates
531 .first()
532 .ok_or_else(|| RuntimeError("agentledger: no runnable step".to_string()))?;
533 let now = now_seconds();
534 let step = self.steps.get_mut(&candidate.step_id).unwrap();
535 step.status = "running".to_string();
536 step.owner = Some(worker_id.to_string());
537 step.lease_token = Some(new_id("lease"));
538 step.lease_until = Some(now + lease_seconds);
539 step.attempt += 1;
540 step.updated_at = now;
541 let claim = StepClaim {
542 run_id: step.run_id.clone(),
543 session_id: step.session_id.clone(),
544 step_id: step.step_id.clone(),
545 attempt: step.attempt,
546 lease_token: step.lease_token.clone().unwrap(),
547 state_version: step.state_version,
548 lease_until: step.lease_until.unwrap(),
549 };
550 let run = self.runs.get_mut(&claim.run_id).unwrap();
551 run.status = "running".to_string();
552 run.updated_at = now;
553 let mut payload = State::new();
554 payload.insert(
555 "worker_id".to_string(),
556 Value::String(worker_id.to_string()),
557 );
558 payload.insert(
559 "lease_token".to_string(),
560 Value::String(claim.lease_token.clone()),
561 );
562 payload.insert("attempt".to_string(), Value::Number(claim.attempt as f64));
563 self.append_event(
564 &claim.run_id,
565 Some(&claim.session_id),
566 Some(&claim.step_id),
567 "step_claimed",
568 payload,
569 None,
570 None,
571 None,
572 );
573 Ok(claim)
574 }
575
576 pub fn load_state(&self, run_id: &str) -> Result<(State, u64, String)> {
577 let run = self
578 .runs
579 .get(run_id)
580 .ok_or_else(|| RuntimeError(format!("run not found: {run_id}")))?;
581 Ok((run.state.clone(), run.state_version, run.session_id.clone()))
582 }
583
584 pub fn recover_expired_leases(&mut self) -> usize {
585 let now = now_seconds();
586 let ids: Vec<String> = self.steps.keys().cloned().collect();
587 let mut recovered = 0;
588 for step_id in ids {
589 let should_recover = self
590 .steps
591 .get(&step_id)
592 .map(|step| {
593 step.status == "running" && step.lease_until.is_some_and(|until| until <= now)
594 })
595 .unwrap_or(false);
596 if !should_recover {
597 continue;
598 }
599 let (run_id, session_id, previous_owner, attempt) = {
600 let step = self.steps.get_mut(&step_id).unwrap();
601 let previous_owner = step.owner.clone().unwrap_or_default();
602 step.status = "retry_scheduled".to_string();
603 step.owner = None;
604 step.lease_token = None;
605 step.lease_until = None;
606 step.updated_at = now;
607 (
608 step.run_id.clone(),
609 step.session_id.clone(),
610 previous_owner,
611 step.attempt,
612 )
613 };
614 let run = self.runs.get_mut(&run_id).unwrap();
615 run.status = "retry_scheduled".to_string();
616 run.updated_at = now;
617 recovered += 1;
618 let mut payload = State::new();
619 payload.insert("previous_owner".to_string(), Value::String(previous_owner));
620 payload.insert("attempt".to_string(), Value::Number(attempt as f64));
621 self.append_event(
622 &run_id,
623 Some(&session_id),
624 Some(&step_id),
625 "lease_expired",
626 payload,
627 None,
628 None,
629 None,
630 );
631 let mut retry_payload = State::new();
632 retry_payload.insert("step_id".to_string(), Value::String(step_id.clone()));
633 retry_payload.insert(
634 "reason".to_string(),
635 Value::String("lease_expired".to_string()),
636 );
637 self.append_event(
638 &run_id,
639 Some(&session_id),
640 Some(&step_id),
641 "step_retry_scheduled",
642 retry_payload,
643 None,
644 None,
645 None,
646 );
647 }
648 recovered
649 }
650
651 pub fn cancel_run(&mut self, run_id: &str, reason: &str) -> Result<usize> {
652 let session_id = self
653 .runs
654 .get(run_id)
655 .ok_or_else(|| RuntimeError(format!("run not found: {run_id}")))?
656 .session_id
657 .clone();
658 let mut payload = State::new();
659 payload.insert("reason".to_string(), Value::String(reason.to_string()));
660 self.append_event(
661 run_id,
662 Some(&session_id),
663 None,
664 "run_cancel_requested",
665 payload,
666 None,
667 None,
668 None,
669 );
670 let ids: Vec<String> = self.steps.keys().cloned().collect();
671 let now = now_seconds();
672 let mut cancelled = 0;
673 for step_id in ids {
674 let should_cancel = self
675 .steps
676 .get(&step_id)
677 .map(|step| {
678 step.run_id == run_id
679 && !["completed", "failed", "cancelled"].contains(&step.status.as_str())
680 })
681 .unwrap_or(false);
682 if !should_cancel {
683 continue;
684 }
685 let step_session = {
686 let step = self.steps.get_mut(&step_id).unwrap();
687 step.status = "cancelled".to_string();
688 step.owner = None;
689 step.lease_token = None;
690 step.lease_until = None;
691 step.cancelled_at = Some(now);
692 step.updated_at = now;
693 step.session_id.clone()
694 };
695 cancelled += 1;
696 let mut step_payload = State::new();
697 step_payload.insert("reason".to_string(), Value::String(reason.to_string()));
698 self.append_event(
699 run_id,
700 Some(&step_session),
701 Some(&step_id),
702 "step_cancelled",
703 step_payload,
704 None,
705 None,
706 None,
707 );
708 }
709 let run = self.runs.get_mut(run_id).unwrap();
710 run.status = "cancelled".to_string();
711 run.updated_at = now;
712 let mut done_payload = State::new();
713 done_payload.insert("reason".to_string(), Value::String(reason.to_string()));
714 done_payload.insert(
715 "cancelled_steps".to_string(),
716 Value::Number(cancelled as f64),
717 );
718 self.append_event(
719 run_id,
720 Some(&session_id),
721 None,
722 "run_cancelled",
723 done_payload,
724 None,
725 None,
726 None,
727 );
728 Ok(cancelled)
729 }
730
731 pub fn commit_state_patch(
732 &mut self,
733 run_id: &str,
734 step_id: &str,
735 lease_token: &str,
736 base_version: u64,
737 patch: State,
738 ) -> Result<u64> {
739 self.validate_lease(step_id, lease_token)?;
740 let run = self
741 .runs
742 .get(run_id)
743 .ok_or_else(|| RuntimeError(format!("run not found: {run_id}")))?;
744 if run.state_version != base_version {
745 return Err(RuntimeError(format!(
746 "state version conflict: expected {base_version}, got {}",
747 run.state_version
748 )));
749 }
750 let now = now_seconds();
751 let new_version = base_version + 1;
752 let session_id = self.steps.get(step_id).unwrap().session_id.clone();
753 let run = self.runs.get_mut(run_id).unwrap();
754 run.state = merge_patch(&run.state, &patch);
755 run.state_version = new_version;
756 run.status = "completed".to_string();
757 run.updated_at = now;
758 let step = self.steps.get_mut(step_id).unwrap();
759 step.status = "completed".to_string();
760 step.state_version = new_version;
761 step.checkpoint_id = Some(format!("ckpt:{run_id}:{step_id}:{}", step.attempt));
762 step.updated_at = now;
763 let mut payload = State::new();
764 payload.insert("patch".to_string(), Value::Object(patch));
765 payload.insert(
766 "state_version".to_string(),
767 Value::Number(new_version as f64),
768 );
769 self.append_event(
770 run_id,
771 Some(&session_id),
772 Some(step_id),
773 "state_patch_committed",
774 payload,
775 None,
776 Some(new_version),
777 None,
778 );
779 let mut complete_payload = State::new();
780 complete_payload.insert("step_id".to_string(), Value::String(step_id.to_string()));
781 self.append_event(
782 run_id,
783 Some(&session_id),
784 Some(step_id),
785 "step_completed",
786 complete_payload,
787 None,
788 Some(new_version),
789 None,
790 );
791 Ok(new_version)
792 }
793
794 pub fn mark_retry(&mut self, run_id: &str, step_id: &str, error_type: &str, message: &str) {
795 let now = now_seconds();
796 let session_id = {
797 let step = self.steps.get_mut(step_id).unwrap();
798 step.status = "retry_scheduled".to_string();
799 step.owner = None;
800 step.lease_token = None;
801 step.lease_until = None;
802 step.last_error_type = Some(error_type.to_string());
803 step.last_error = Some(message.to_string());
804 step.updated_at = now;
805 step.session_id.clone()
806 };
807 let run = self.runs.get_mut(run_id).unwrap();
808 run.status = "retry_scheduled".to_string();
809 run.updated_at = now;
810 let mut classified = State::new();
811 classified.insert("error".to_string(), Value::String(message.to_string()));
812 classified.insert(
813 "error_type".to_string(),
814 Value::String(error_type.to_string()),
815 );
816 classified.insert("retryable".to_string(), Value::Bool(true));
817 classified.insert("source".to_string(), Value::String("agent".to_string()));
818 self.append_event(
819 run_id,
820 Some(&session_id),
821 Some(step_id),
822 "failure_classified",
823 classified,
824 None,
825 None,
826 None,
827 );
828 let mut payload = State::new();
829 payload.insert("error".to_string(), Value::String(message.to_string()));
830 payload.insert(
831 "error_type".to_string(),
832 Value::String(error_type.to_string()),
833 );
834 self.append_event(
835 run_id,
836 Some(&session_id),
837 Some(step_id),
838 "error_raised",
839 payload,
840 None,
841 None,
842 None,
843 );
844 let mut retry_payload = State::new();
845 retry_payload.insert("step_id".to_string(), Value::String(step_id.to_string()));
846 self.append_event(
847 run_id,
848 Some(&session_id),
849 Some(step_id),
850 "step_retry_scheduled",
851 retry_payload,
852 None,
853 None,
854 None,
855 );
856 }
857
858 pub fn mark_waiting_human(
859 &mut self,
860 run_id: &str,
861 step_id: &str,
862 reason: &str,
863 approval_id: &str,
864 ) {
865 let now = now_seconds();
866 let session_id = {
867 let step = self.steps.get_mut(step_id).unwrap();
868 step.status = "waiting_human".to_string();
869 step.owner = None;
870 step.lease_token = None;
871 step.lease_until = None;
872 step.last_error_type = Some("ApprovalRequired".to_string());
873 step.last_error = Some(reason.to_string());
874 step.updated_at = now;
875 step.session_id.clone()
876 };
877 let run = self.runs.get_mut(run_id).unwrap();
878 run.status = "waiting_human".to_string();
879 run.updated_at = now;
880 let mut payload = State::new();
881 payload.insert("reason".to_string(), Value::String(reason.to_string()));
882 payload.insert(
883 "approval_id".to_string(),
884 Value::String(approval_id.to_string()),
885 );
886 self.append_event(
887 run_id,
888 Some(&session_id),
889 Some(step_id),
890 "step_waiting_human",
891 payload,
892 None,
893 None,
894 None,
895 );
896 }
897
898 pub fn mark_failed(&mut self, run_id: &str, step_id: &str, error_type: &str, message: &str) {
899 let now = now_seconds();
900 let session_id = {
901 let step = self.steps.get_mut(step_id).unwrap();
902 step.status = "failed".to_string();
903 step.owner = None;
904 step.lease_token = None;
905 step.lease_until = None;
906 step.last_error_type = Some(error_type.to_string());
907 step.last_error = Some(message.to_string());
908 step.updated_at = now;
909 step.session_id.clone()
910 };
911 let run = self.runs.get_mut(run_id).unwrap();
912 run.status = "failed".to_string();
913 run.updated_at = now;
914 let mut classified = State::new();
915 classified.insert("error".to_string(), Value::String(message.to_string()));
916 classified.insert(
917 "error_type".to_string(),
918 Value::String(error_type.to_string()),
919 );
920 classified.insert("retryable".to_string(), Value::Bool(false));
921 classified.insert(
922 "source".to_string(),
923 Value::String(failure_source(error_type).to_string()),
924 );
925 self.append_event(
926 run_id,
927 Some(&session_id),
928 Some(step_id),
929 "failure_classified",
930 classified,
931 None,
932 None,
933 None,
934 );
935 let mut error = State::new();
936 error.insert("error".to_string(), Value::String(message.to_string()));
937 error.insert(
938 "error_type".to_string(),
939 Value::String(error_type.to_string()),
940 );
941 self.append_event(
942 run_id,
943 Some(&session_id),
944 Some(step_id),
945 "error_raised",
946 error,
947 None,
948 None,
949 None,
950 );
951 let mut payload = State::new();
952 payload.insert("step_id".to_string(), Value::String(step_id.to_string()));
953 payload.insert(
954 "error_type".to_string(),
955 Value::String(error_type.to_string()),
956 );
957 self.append_event(
958 run_id,
959 Some(&session_id),
960 Some(step_id),
961 "step_failed",
962 payload,
963 None,
964 None,
965 None,
966 );
967 }
968
969 pub fn append_event(
970 &mut self,
971 run_id: &str,
972 session_id: Option<&str>,
973 step_id: Option<&str>,
974 event_type: &str,
975 payload: State,
976 agent_role: Option<&str>,
977 state_version: Option<u64>,
978 causal_token: Option<&str>,
979 ) -> Event {
980 let events = self.events.entry(run_id.to_string()).or_default();
981 let payload_ref = format_state(&payload);
982 let event = Event {
983 event_id: new_id("evt"),
984 run_id: run_id.to_string(),
985 session_id: session_id.map(str::to_string),
986 step_id: step_id.map(str::to_string),
987 seq: events.len() as u64 + 1,
988 event_type: event_type.to_string(),
989 timestamp: now_seconds(),
990 agent_role: agent_role.map(str::to_string),
991 state_version,
992 causal_token: causal_token.map(str::to_string),
993 payload_hash: stable_hash(&payload_ref),
994 payload_ref,
995 payload,
996 };
997 events.push(event.clone());
998 event
999 }
1000
1001 pub fn reserve_ledger(&mut self, key: &str, entry: ToolLedgerEntry) -> Option<ToolLedgerEntry> {
1002 if let Some(existing) = self.tool_ledger.get(key) {
1003 return Some(existing.clone());
1004 }
1005 self.tool_ledger.insert(key.to_string(), entry);
1006 None
1007 }
1008
1009 pub fn update_ledger(
1010 &mut self,
1011 key: &str,
1012 status: &str,
1013 response: Option<Value>,
1014 error_type: Option<String>,
1015 ) {
1016 if let Some(entry) = self.tool_ledger.get_mut(key) {
1017 entry.status = status.to_string();
1018 entry.response = response.clone();
1019 entry.response_ref = response.as_ref().map(format_value);
1020 entry.response_hash = entry.response_ref.as_ref().map(|value| stable_hash(value));
1021 entry.error_type = error_type;
1022 entry.updated_at = now_seconds();
1023 }
1024 }
1025
1026 pub fn request_approval(
1027 &mut self,
1028 approval_key: &str,
1029 run_id: &str,
1030 session_id: &str,
1031 step_id: &str,
1032 tool_name: &str,
1033 risk_level: &str,
1034 reason: &str,
1035 request_hash: &str,
1036 request_ref: &str,
1037 requested_by: &str,
1038 ) -> ApprovalRequest {
1039 if let Some(existing) = self.approval_requests.get(approval_key) {
1040 return existing.clone();
1041 }
1042 let now = now_seconds();
1043 let approval = ApprovalRequest {
1044 approval_id: new_id("approval"),
1045 approval_key: approval_key.to_string(),
1046 run_id: run_id.to_string(),
1047 session_id: session_id.to_string(),
1048 step_id: step_id.to_string(),
1049 tool_name: tool_name.to_string(),
1050 risk_level: risk_level.to_string(),
1051 status: "PENDING".to_string(),
1052 reason: reason.to_string(),
1053 request_hash: request_hash.to_string(),
1054 request_ref: request_ref.to_string(),
1055 requested_by: requested_by.to_string(),
1056 approved_by: None,
1057 decision_reason: None,
1058 created_at: now,
1059 updated_at: now,
1060 };
1061 self.approval_requests
1062 .insert(approval_key.to_string(), approval.clone());
1063 approval
1064 }
1065
1066 pub fn approval_for_key(&self, approval_key: &str) -> Option<ApprovalRequest> {
1067 self.approval_requests.get(approval_key).cloned()
1068 }
1069
1070 pub fn approval_requests(&self, run_id: &str) -> Vec<ApprovalRequest> {
1071 self.approval_requests
1072 .values()
1073 .filter(|item| run_id.is_empty() || item.run_id == run_id)
1074 .cloned()
1075 .collect()
1076 }
1077
1078 pub fn approve_request(
1079 &mut self,
1080 approval_id: &str,
1081 approver: &str,
1082 reason: &str,
1083 ) -> Result<ApprovalRequest> {
1084 self.decide_approval(approval_id, "APPROVED", approver, reason)
1085 }
1086
1087 pub fn deny_request(
1088 &mut self,
1089 approval_id: &str,
1090 approver: &str,
1091 reason: &str,
1092 ) -> Result<ApprovalRequest> {
1093 self.decide_approval(approval_id, "DENIED", approver, reason)
1094 }
1095
1096 fn decide_approval(
1097 &mut self,
1098 approval_id: &str,
1099 status: &str,
1100 approver: &str,
1101 reason: &str,
1102 ) -> Result<ApprovalRequest> {
1103 let key = self
1104 .approval_requests
1105 .iter()
1106 .find(|(_, item)| item.approval_id == approval_id)
1107 .map(|(key, _)| key.clone())
1108 .ok_or_else(|| RuntimeError(format!("approval not found: {approval_id}")))?;
1109 let mut approval = self.approval_requests.get(&key).unwrap().clone();
1110 approval.status = status.to_string();
1111 approval.approved_by = Some(approver.to_string());
1112 approval.decision_reason = Some(reason.to_string());
1113 approval.updated_at = now_seconds();
1114 self.approval_requests.insert(key, approval.clone());
1115 let mut payload = State::new();
1116 payload.insert(
1117 "approval_id".to_string(),
1118 Value::String(approval_id.to_string()),
1119 );
1120 payload.insert(
1121 "tool".to_string(),
1122 Value::String(approval.tool_name.clone()),
1123 );
1124 payload.insert("status".to_string(), Value::String(status.to_string()));
1125 self.append_event(
1126 &approval.run_id,
1127 Some(&approval.session_id),
1128 Some(&approval.step_id),
1129 "tool_approval_decided",
1130 payload,
1131 None,
1132 None,
1133 None,
1134 );
1135 if self
1136 .steps
1137 .get(&approval.step_id)
1138 .map(|step| step.status.as_str())
1139 == Some("waiting_human")
1140 {
1141 if status == "APPROVED" {
1142 let step = self.steps.get_mut(&approval.step_id).unwrap();
1143 step.status = "pending".to_string();
1144 step.owner = None;
1145 step.lease_token = None;
1146 step.lease_until = None;
1147 step.updated_at = now_seconds();
1148 let run = self.runs.get_mut(&approval.run_id).unwrap();
1149 run.status = "pending".to_string();
1150 let mut retry = State::new();
1151 retry.insert(
1152 "step_id".to_string(),
1153 Value::String(approval.step_id.clone()),
1154 );
1155 retry.insert(
1156 "reason".to_string(),
1157 Value::String("approval_granted".to_string()),
1158 );
1159 self.append_event(
1160 &approval.run_id,
1161 Some(&approval.session_id),
1162 Some(&approval.step_id),
1163 "step_retry_scheduled",
1164 retry,
1165 None,
1166 None,
1167 None,
1168 );
1169 } else if status == "DENIED" {
1170 self.mark_failed(
1171 &approval.run_id,
1172 &approval.step_id,
1173 "ApprovalDenied",
1174 reason,
1175 );
1176 }
1177 }
1178 Ok(approval)
1179 }
1180
1181 pub fn record_cost(
1182 &mut self,
1183 run_id: &str,
1184 session_id: &str,
1185 step_id: &str,
1186 category: &str,
1187 name: &str,
1188 amount: f64,
1189 unit: &str,
1190 metadata: State,
1191 ) -> String {
1192 let cost_id = new_id("cost");
1193 let record = CostRecord {
1194 cost_id: cost_id.clone(),
1195 run_id: run_id.to_string(),
1196 session_id: session_id.to_string(),
1197 step_id: step_id.to_string(),
1198 category: category.to_string(),
1199 name: name.to_string(),
1200 amount,
1201 unit: unit.to_string(),
1202 metadata,
1203 created_at: now_seconds(),
1204 };
1205 self.cost_records
1206 .entry(run_id.to_string())
1207 .or_default()
1208 .push(record);
1209 let mut payload = State::new();
1210 payload.insert("cost_id".to_string(), Value::String(cost_id.clone()));
1211 payload.insert("category".to_string(), Value::String(category.to_string()));
1212 payload.insert("name".to_string(), Value::String(name.to_string()));
1213 payload.insert("amount".to_string(), Value::Number(amount));
1214 payload.insert("unit".to_string(), Value::String(unit.to_string()));
1215 self.append_event(
1216 run_id,
1217 Some(session_id),
1218 Some(step_id),
1219 "cost_recorded",
1220 payload,
1221 None,
1222 None,
1223 None,
1224 );
1225 cost_id
1226 }
1227
1228 pub fn cost_records(&self, run_id: &str) -> Vec<CostRecord> {
1229 self.cost_records.get(run_id).cloned().unwrap_or_default()
1230 }
1231
1232 pub fn cost_summary(&self, run_id: &str) -> CostSummary {
1233 let mut summary = CostSummary::default();
1234 for record in self.cost_records(run_id) {
1235 add_cost(&mut summary, &record);
1236 }
1237 summary
1238 }
1239
1240 pub fn create_artifact(
1241 &mut self,
1242 run_id: &str,
1243 step_id: Option<&str>,
1244 name: &str,
1245 content: State,
1246 metadata: State,
1247 ) -> Artifact {
1248 let blob_ref = format_state(&content);
1249 let artifact = Artifact {
1250 artifact_id: new_id("art"),
1251 run_id: run_id.to_string(),
1252 step_id: step_id.map(str::to_string),
1253 name: name.to_string(),
1254 blob_hash: format!("sha256:{}", stable_hash(&blob_ref)),
1255 blob_ref,
1256 metadata,
1257 created_at: now_seconds(),
1258 };
1259 self.artifacts
1260 .entry(run_id.to_string())
1261 .or_default()
1262 .push(artifact.clone());
1263 artifact
1264 }
1265
1266 pub fn artifacts(&self, run_id: &str) -> Vec<Artifact> {
1267 let mut items = self.artifacts.get(run_id).cloned().unwrap_or_default();
1268 items.sort_by(|a, b| {
1269 a.created_at
1270 .partial_cmp(&b.created_at)
1271 .unwrap()
1272 .then(a.artifact_id.cmp(&b.artifact_id))
1273 });
1274 items
1275 }
1276
1277 pub fn validate_lease(&self, step_id: &str, lease_token: &str) -> Result<()> {
1278 let step = self
1279 .steps
1280 .get(step_id)
1281 .ok_or_else(|| RuntimeError(format!("step not found: {step_id}")))?;
1282 if step.status != "running" || step.lease_token.as_deref() != Some(lease_token) {
1283 return Err(RuntimeError("invalid or stale lease token".to_string()));
1284 }
1285 if step.lease_until.is_some_and(|until| until <= now_seconds()) {
1286 return Err(RuntimeError("lease expired".to_string()));
1287 }
1288 Ok(())
1289 }
1290
1291 pub fn final_state(&self, run_id: &str) -> Result<State> {
1292 Ok(self.load_state(run_id)?.0)
1293 }
1294
1295 pub fn run(&self, run_id: &str) -> Result<Run> {
1296 self.runs
1297 .get(run_id)
1298 .cloned()
1299 .ok_or_else(|| RuntimeError(format!("run not found: {run_id}")))
1300 }
1301
1302 pub fn steps(&self, run_id: &str) -> Vec<Step> {
1303 self.steps
1304 .values()
1305 .filter(|step| step.run_id == run_id)
1306 .cloned()
1307 .collect()
1308 }
1309
1310 pub fn events(&self, run_id: &str) -> Vec<Event> {
1311 self.events.get(run_id).cloned().unwrap_or_default()
1312 }
1313
1314 pub fn ledger(&self, run_id: &str) -> Vec<ToolLedgerEntry> {
1315 self.tool_ledger
1316 .values()
1317 .filter(|entry| entry.run_id == run_id)
1318 .cloned()
1319 .collect()
1320 }
1321}
1322
1323pub struct ToolRegistry {
1324 tools: HashMap<String, ToolSpec>,
1325}
1326
1327impl ToolRegistry {
1328 pub fn new() -> Self {
1329 Self {
1330 tools: HashMap::new(),
1331 }
1332 }
1333
1334 pub fn register(&mut self, spec: ToolSpec) {
1335 self.tools.insert(spec.name.clone(), spec);
1336 }
1337
1338 pub fn get(&self, name: &str) -> Result<&ToolSpec> {
1339 self.tools
1340 .get(name)
1341 .ok_or_else(|| RuntimeError(format!("tool not registered: {name}")))
1342 }
1343}
1344
1345impl Default for ToolRegistry {
1346 fn default() -> Self {
1347 Self::new()
1348 }
1349}
1350
1351pub struct Runtime {
1352 pub store: MemoryStore,
1353 pub registry: ToolRegistry,
1354 pub budget: BudgetLimits,
1355 pub sandbox: Box<dyn SandboxExecutor>,
1356}
1357
1358impl Runtime {
1359 pub fn new() -> Self {
1360 Self {
1361 store: MemoryStore::new(),
1362 registry: ToolRegistry::new(),
1363 budget: BudgetLimits::default(),
1364 sandbox: Box::new(DisabledSandboxExecutor),
1365 }
1366 }
1367
1368 pub fn register_tool(&mut self, spec: ToolSpec) {
1369 self.registry.register(spec);
1370 }
1371
1372 pub fn set_budget(&mut self, budget: BudgetLimits) {
1373 self.budget = budget;
1374 }
1375
1376 pub fn set_sandbox(&mut self, sandbox: Box<dyn SandboxExecutor>) {
1377 self.sandbox = sandbox;
1378 }
1379
1380 pub fn create_run(&mut self, initial_state: State) -> (String, String) {
1381 self.store.create_run(initial_state)
1382 }
1383
1384 pub fn run_once(
1385 &mut self,
1386 run_id: &str,
1387 worker_id: &str,
1388 agent_role: &str,
1389 lease_seconds: f64,
1390 agent: AgentFunc,
1391 ) -> Result<bool> {
1392 let claim = match self.store.claim_step(worker_id, run_id, lease_seconds) {
1393 Ok(claim) => claim,
1394 Err(err) if err.0.contains("no runnable step") => return Ok(false),
1395 Err(err) => return Err(err),
1396 };
1397 let (state, version, session_id) = self.store.load_state(&claim.run_id)?;
1398 let mut payload = State::new();
1399 payload.insert(
1400 "agent_role".to_string(),
1401 Value::String(agent_role.to_string()),
1402 );
1403 payload.insert("attempt".to_string(), Value::Number(claim.attempt as f64));
1404 self.store.append_event(
1405 &claim.run_id,
1406 Some(&session_id),
1407 Some(&claim.step_id),
1408 "agent_started",
1409 payload,
1410 Some(agent_role),
1411 Some(version),
1412 None,
1413 );
1414 let mut ctx = AgentContext {
1415 run_id: claim.run_id.clone(),
1416 session_id,
1417 step_id: claim.step_id.clone(),
1418 agent_role: agent_role.to_string(),
1419 lease_token: claim.lease_token.clone(),
1420 attempt: claim.attempt,
1421 state_version: version,
1422 pending_patch: State::new(),
1423 };
1424 match agent(&mut ctx, state) {
1425 Ok(()) => {
1426 self.store.commit_state_patch(
1427 &claim.run_id,
1428 &claim.step_id,
1429 &claim.lease_token,
1430 version,
1431 ctx.pending_patch,
1432 )?;
1433 Ok(true)
1434 }
1435 Err(err) if err.0 == "retryable" => {
1436 self.store.mark_retry(
1437 &claim.run_id,
1438 &claim.step_id,
1439 "RetryableAgentError",
1440 "retryable",
1441 );
1442 Ok(false)
1443 }
1444 Err(err) if err.0.starts_with("approval required:") => {
1445 let approval_id = err.0.trim_start_matches("approval required:");
1446 self.store
1447 .mark_waiting_human(&claim.run_id, &claim.step_id, &err.0, approval_id);
1448 Ok(false)
1449 }
1450 Err(err) => {
1451 self.store.mark_failed(
1452 &claim.run_id,
1453 &claim.step_id,
1454 classify_runtime_error(&err.0),
1455 &err.0,
1456 );
1457 Err(err)
1458 }
1459 }
1460 }
1461
1462 pub fn call_tool(&mut self, ctx: &AgentContext, tool_name: &str, args: State) -> Result<Value> {
1463 let (
1464 version,
1465 side_effect,
1466 risk_level,
1467 idempotency_required,
1468 approval_required,
1469 sandbox_required,
1470 sandbox_executor,
1471 sandbox_policy,
1472 input_schema,
1473 output_schema,
1474 ) = {
1475 let spec = self.registry.get(tool_name)?;
1476 (
1477 spec.version.clone(),
1478 spec.side_effect.clone(),
1479 spec.risk_level.clone(),
1480 spec.idempotency_required,
1481 spec.approval_required,
1482 spec.sandbox_required,
1483 spec.sandbox_executor.clone(),
1484 spec.sandbox_policy.clone(),
1485 spec.input_schema.clone(),
1486 spec.output_schema.clone(),
1487 )
1488 };
1489 let mut request = State::new();
1490 request.insert("tool".to_string(), Value::String(tool_name.to_string()));
1491 request.insert("args".to_string(), Value::Object(args.clone()));
1492 let request_ref = format_state(&request);
1493 let request_hash = stable_hash(&request_ref);
1494 let causal_token = format!(
1495 "{}:{}:{}:{}",
1496 ctx.run_id, ctx.step_id, ctx.attempt, ctx.state_version
1497 );
1498 let idempotency_key = format!(
1499 "{}:{}:{}:{}",
1500 ctx.run_id, ctx.step_id, tool_name, request_hash
1501 );
1502 let managed = side_effect != "none" || idempotency_required;
1503 self.store.append_event(
1504 &ctx.run_id,
1505 Some(&ctx.session_id),
1506 Some(&ctx.step_id),
1507 "tool_call_requested",
1508 request,
1509 Some(&ctx.agent_role),
1510 Some(ctx.state_version),
1511 Some(&causal_token),
1512 );
1513 if let Some(schema) = input_schema.as_ref() {
1514 if let Err(err) = validate_tool_schema(schema, &Value::Object(args.clone()), "$arg") {
1515 let mut failed = State::new();
1516 failed.insert("tool".to_string(), Value::String(tool_name.to_string()));
1517 failed.insert("error".to_string(), Value::String(err.0.clone()));
1518 failed.insert(
1519 "phase".to_string(),
1520 Value::String("input_validation".to_string()),
1521 );
1522 self.store.append_event(
1523 &ctx.run_id,
1524 Some(&ctx.session_id),
1525 Some(&ctx.step_id),
1526 "tool_call_failed",
1527 failed,
1528 Some(&ctx.agent_role),
1529 Some(ctx.state_version),
1530 Some(&causal_token),
1531 );
1532 return Err(err);
1533 }
1534 }
1535
1536 let mut allowed = !is_high_risk(&risk_level);
1537 let mut reason = if allowed {
1538 "default allow for low/medium risk in local runtime".to_string()
1539 } else {
1540 "high-risk tool denied by default".to_string()
1541 };
1542 if let Some(approval) = self.store.approval_for_key(&idempotency_key) {
1543 if approval.status == "DENIED" {
1544 reason = format!("approval denied for tool {tool_name}");
1545 self.record_permission(ctx, tool_name, false, &reason, &causal_token);
1546 return Err(RuntimeError(reason));
1547 }
1548 if approval.status == "APPROVED" {
1549 allowed = true;
1550 reason = format!(
1551 "approved by {}",
1552 approval
1553 .approved_by
1554 .unwrap_or_else(|| "operator".to_string())
1555 );
1556 }
1557 } else if approval_required {
1558 let approval = self.store.request_approval(
1559 &idempotency_key,
1560 &ctx.run_id,
1561 &ctx.session_id,
1562 &ctx.step_id,
1563 tool_name,
1564 &risk_level,
1565 "tool requires approval",
1566 &request_hash,
1567 &request_ref,
1568 &ctx.agent_role,
1569 );
1570 self.record_permission(ctx, tool_name, false, "approval required", &causal_token);
1571 let mut payload = State::new();
1572 payload.insert("tool".to_string(), Value::String(tool_name.to_string()));
1573 payload.insert(
1574 "approval_id".to_string(),
1575 Value::String(approval.approval_id.clone()),
1576 );
1577 payload.insert(
1578 "approval_key".to_string(),
1579 Value::String(idempotency_key.clone()),
1580 );
1581 payload.insert("risk_level".to_string(), Value::String(risk_level));
1582 self.store.append_event(
1583 &ctx.run_id,
1584 Some(&ctx.session_id),
1585 Some(&ctx.step_id),
1586 "tool_approval_required",
1587 payload,
1588 Some(&ctx.agent_role),
1589 Some(ctx.state_version),
1590 Some(&causal_token),
1591 );
1592 return Err(RuntimeError(format!(
1593 "approval required:{}",
1594 approval.approval_id
1595 )));
1596 }
1597 self.record_permission(ctx, tool_name, allowed, &reason, &causal_token);
1598 if !allowed {
1599 return Err(RuntimeError(reason));
1600 }
1601 if let Some(max) = self.budget.max_tool_calls {
1602 let used = self.store.cost_summary(&ctx.run_id).tool_calls;
1603 if used >= max {
1604 let message = format!("tool call budget exceeded: {used}/{max}");
1605 let mut payload = State::new();
1606 payload.insert("category".to_string(), Value::String("tool".to_string()));
1607 payload.insert("tool".to_string(), Value::String(tool_name.to_string()));
1608 payload.insert("error".to_string(), Value::String(message.clone()));
1609 self.store.append_event(
1610 &ctx.run_id,
1611 Some(&ctx.session_id),
1612 Some(&ctx.step_id),
1613 "budget_check_failed",
1614 payload,
1615 Some(&ctx.agent_role),
1616 Some(ctx.state_version),
1617 Some(&causal_token),
1618 );
1619 return Err(RuntimeError(message));
1620 }
1621 }
1622 if managed {
1623 let entry = ToolLedgerEntry {
1624 ledger_id: new_id("ledger"),
1625 run_id: ctx.run_id.clone(),
1626 session_id: ctx.session_id.clone(),
1627 step_id: ctx.step_id.clone(),
1628 tool_name: tool_name.to_string(),
1629 tool_version: version,
1630 tool_call_id: new_id("toolcall"),
1631 idempotency_key: idempotency_key.clone(),
1632 causal_token: causal_token.clone(),
1633 request_hash: request_hash.clone(),
1634 request_ref: request_ref.clone(),
1635 status: "RESERVED".to_string(),
1636 external_id: None,
1637 response_hash: None,
1638 response_ref: None,
1639 error_type: None,
1640 response: None,
1641 created_at: now_seconds(),
1642 updated_at: now_seconds(),
1643 };
1644 if let Some(existing) = self.store.reserve_ledger(&idempotency_key, entry) {
1645 if existing.status == "SUCCEEDED" {
1646 let mut payload = State::new();
1647 payload.insert("tool".to_string(), Value::String(tool_name.to_string()));
1648 payload.insert("replayed_from_ledger".to_string(), Value::Bool(true));
1649 self.store.append_event(
1650 &ctx.run_id,
1651 Some(&ctx.session_id),
1652 Some(&ctx.step_id),
1653 "tool_call_completed",
1654 payload,
1655 Some(&ctx.agent_role),
1656 Some(ctx.state_version),
1657 Some(&causal_token),
1658 );
1659 self.store.record_cost(
1660 &ctx.run_id,
1661 &ctx.session_id,
1662 &ctx.step_id,
1663 "tool",
1664 tool_name,
1665 1.0,
1666 "call",
1667 State::new(),
1668 );
1669 return existing
1670 .response
1671 .ok_or_else(|| RuntimeError("missing ledger response".to_string()));
1672 }
1673 return Err(RuntimeError(
1674 "tool side effect already in progress".to_string(),
1675 ));
1676 }
1677 self.store
1678 .update_ledger(&idempotency_key, "RUNNING", None, None);
1679 }
1680 if sandbox_required {
1681 let executor = if sandbox_executor.is_empty() { "default".to_string() } else { sandbox_executor.clone() };
1682 let network = match sandbox_policy.get("network") {
1683 Some(Value::String(value)) => value.clone(),
1684 _ => "deny".to_string(),
1685 };
1686 let filesystem = match sandbox_policy.get("filesystem") {
1687 Some(Value::String(value)) => value.clone(),
1688 _ => "read-only".to_string(),
1689 };
1690 let timeout_seconds = match sandbox_policy.get("timeout_seconds") {
1691 Some(Value::Number(value)) if *value > 0.0 => *value as u64,
1692 _ => 30,
1693 };
1694 let policy = SandboxPolicy {
1695 tool_name: tool_name.to_string(),
1696 run_id: ctx.run_id.clone(),
1697 step_id: ctx.step_id.clone(),
1698 executor: executor.clone(),
1699 network,
1700 filesystem,
1701 timeout_seconds,
1702 extra: sandbox_policy.clone(),
1703 };
1704 let mut sandbox_payload = State::new();
1705 sandbox_payload.insert("tool_name".to_string(), Value::String(tool_name.to_string()));
1706 sandbox_payload.insert("executor".to_string(), Value::String(executor));
1707 sandbox_payload.insert("network".to_string(), Value::String(policy.network.clone()));
1708 sandbox_payload.insert("filesystem".to_string(), Value::String(policy.filesystem.clone()));
1709 sandbox_payload.insert("timeout_seconds".to_string(), Value::Number(policy.timeout_seconds as f64));
1710 self.store.append_event(
1711 &ctx.run_id,
1712 Some(&ctx.session_id),
1713 Some(&ctx.step_id),
1714 "sandbox_started",
1715 sandbox_payload,
1716 Some(&ctx.agent_role),
1717 Some(ctx.state_version),
1718 Some(&causal_token),
1719 );
1720 let result = self.sandbox.run_tool(args, &policy);
1721 let mut completed = State::new();
1722 completed.insert("ok".to_string(), Value::Bool(result.ok));
1723 completed.insert("metadata".to_string(), Value::Object(result.metadata.clone()));
1724 if let Some(error) = &result.error {
1725 completed.insert("error".to_string(), Value::String(error.clone()));
1726 }
1727 self.store.append_event(
1728 &ctx.run_id,
1729 Some(&ctx.session_id),
1730 Some(&ctx.step_id),
1731 "sandbox_completed",
1732 completed,
1733 Some(&ctx.agent_role),
1734 Some(ctx.state_version),
1735 Some(&causal_token),
1736 );
1737 if !result.ok {
1738 let message = result.error.unwrap_or_else(|| "sandboxed tool failed".to_string());
1739 let mut failed = State::new();
1740 failed.insert("tool".to_string(), Value::String(tool_name.to_string()));
1741 failed.insert("error".to_string(), Value::String(message.clone()));
1742 self.store.append_event(
1743 &ctx.run_id,
1744 Some(&ctx.session_id),
1745 Some(&ctx.step_id),
1746 "tool_call_failed",
1747 failed,
1748 Some(&ctx.agent_role),
1749 Some(ctx.state_version),
1750 Some(&causal_token),
1751 );
1752 return Err(RuntimeError(message));
1753 }
1754 let value = result.output;
1755 if let Some(schema) = output_schema.as_ref() {
1756 validate_tool_schema(schema, &value, "$result")?;
1757 }
1758 if managed {
1759 self.store.update_ledger(&idempotency_key, "SUCCEEDED", Some(value.clone()), None);
1760 }
1761 let mut payload = State::new();
1762 payload.insert("tool".to_string(), Value::String(tool_name.to_string()));
1763 payload.insert("idempotency_key".to_string(), Value::String(idempotency_key));
1764 self.store.append_event(
1765 &ctx.run_id,
1766 Some(&ctx.session_id),
1767 Some(&ctx.step_id),
1768 "tool_call_completed",
1769 payload,
1770 Some(&ctx.agent_role),
1771 Some(ctx.state_version),
1772 Some(&causal_token),
1773 );
1774 self.store.record_cost(
1775 &ctx.run_id,
1776 &ctx.session_id,
1777 &ctx.step_id,
1778 "tool",
1779 tool_name,
1780 1.0,
1781 "call",
1782 State::new(),
1783 );
1784 return Ok(value);
1785 }
1786 let result = {
1787 let spec = self.registry.get(tool_name)?;
1788 (spec.func)(args)
1789 };
1790 match result {
1791 Ok(value) => {
1792 if let Some(schema) = output_schema.as_ref() {
1793 if let Err(err) = validate_tool_schema(schema, &value, "$result") {
1794 if managed {
1795 self.store.update_ledger(
1796 &idempotency_key,
1797 "PENDING_VERIFICATION",
1798 None,
1799 Some("ToolOutputValidationError".to_string()),
1800 );
1801 }
1802 let mut failed = State::new();
1803 failed.insert("tool".to_string(), Value::String(tool_name.to_string()));
1804 failed.insert("error".to_string(), Value::String(err.0.clone()));
1805 failed.insert(
1806 "phase".to_string(),
1807 Value::String("output_validation".to_string()),
1808 );
1809 self.store.append_event(
1810 &ctx.run_id,
1811 Some(&ctx.session_id),
1812 Some(&ctx.step_id),
1813 "tool_call_failed",
1814 failed,
1815 Some(&ctx.agent_role),
1816 Some(ctx.state_version),
1817 Some(&causal_token),
1818 );
1819 return Err(err);
1820 }
1821 }
1822 if managed {
1823 self.store.update_ledger(
1824 &idempotency_key,
1825 "SUCCEEDED",
1826 Some(value.clone()),
1827 None,
1828 );
1829 }
1830 let mut payload = State::new();
1831 payload.insert("tool".to_string(), Value::String(tool_name.to_string()));
1832 payload.insert(
1833 "idempotency_key".to_string(),
1834 Value::String(idempotency_key),
1835 );
1836 self.store.append_event(
1837 &ctx.run_id,
1838 Some(&ctx.session_id),
1839 Some(&ctx.step_id),
1840 "tool_call_completed",
1841 payload,
1842 Some(&ctx.agent_role),
1843 Some(ctx.state_version),
1844 Some(&causal_token),
1845 );
1846 self.store.record_cost(
1847 &ctx.run_id,
1848 &ctx.session_id,
1849 &ctx.step_id,
1850 "tool",
1851 tool_name,
1852 1.0,
1853 "call",
1854 State::new(),
1855 );
1856 Ok(value)
1857 }
1858 Err(err) => {
1859 if managed {
1860 self.store.update_ledger(
1861 &idempotency_key,
1862 "PENDING_VERIFICATION",
1863 None,
1864 Some("RuntimeError".to_string()),
1865 );
1866 }
1867 let mut payload = State::new();
1868 payload.insert("tool".to_string(), Value::String(tool_name.to_string()));
1869 payload.insert("error".to_string(), Value::String(err.0.clone()));
1870 self.store.append_event(
1871 &ctx.run_id,
1872 Some(&ctx.session_id),
1873 Some(&ctx.step_id),
1874 "tool_call_failed",
1875 payload,
1876 Some(&ctx.agent_role),
1877 Some(ctx.state_version),
1878 Some(&causal_token),
1879 );
1880 Err(err)
1881 }
1882 }
1883 }
1884
1885 fn record_permission(
1886 &mut self,
1887 ctx: &AgentContext,
1888 tool_name: &str,
1889 allowed: bool,
1890 reason: &str,
1891 causal_token: &str,
1892 ) {
1893 let mut permission = State::new();
1894 permission.insert("tool".to_string(), Value::String(tool_name.to_string()));
1895 permission.insert("allowed".to_string(), Value::Bool(allowed));
1896 permission.insert("reason".to_string(), Value::String(reason.to_string()));
1897 self.store.append_event(
1898 &ctx.run_id,
1899 Some(&ctx.session_id),
1900 Some(&ctx.step_id),
1901 "tool_permission_decided",
1902 permission,
1903 Some(&ctx.agent_role),
1904 Some(ctx.state_version),
1905 Some(causal_token),
1906 );
1907 }
1908
1909 pub fn create_artifact(
1910 &mut self,
1911 ctx: &AgentContext,
1912 name: &str,
1913 content: State,
1914 metadata: State,
1915 ) -> Result<String> {
1916 let artifact =
1917 self.store
1918 .create_artifact(&ctx.run_id, Some(&ctx.step_id), name, content, metadata);
1919 let mut payload = State::new();
1920 payload.insert(
1921 "artifact_id".to_string(),
1922 Value::String(artifact.artifact_id.clone()),
1923 );
1924 payload.insert("name".to_string(), Value::String(name.to_string()));
1925 self.store.append_event(
1926 &ctx.run_id,
1927 Some(&ctx.session_id),
1928 Some(&ctx.step_id),
1929 "artifact_created",
1930 payload,
1931 Some(&ctx.agent_role),
1932 Some(ctx.state_version),
1933 None,
1934 );
1935 Ok(artifact.artifact_id)
1936 }
1937
1938 pub fn create_media_artifact(
1939 &mut self,
1940 ctx: &AgentContext,
1941 name: &str,
1942 kind: &str,
1943 options: MediaArtifactOptions,
1944 ) -> Result<String> {
1945 if !is_media_kind(kind) {
1946 return Err(RuntimeError(format!("unsupported media kind: {kind}")));
1947 }
1948 let mut media_metadata = options.media_metadata.clone();
1949 media_metadata.insert(
1950 "schema_version".to_string(),
1951 Value::String(MEDIA_SCHEMA_VERSION.to_string()),
1952 );
1953 media_metadata.insert("kind".to_string(), Value::String(kind.to_string()));
1954 let mut content = State::new();
1955 content.insert(
1956 "schema_version".to_string(),
1957 Value::String(MEDIA_SCHEMA_VERSION.to_string()),
1958 );
1959 content.insert("kind".to_string(), Value::String(kind.to_string()));
1960 if let Some(uri) = &options.uri {
1961 content.insert("uri".to_string(), Value::String(uri.clone()));
1962 }
1963 if let Some(content_ref) = &options.content_ref {
1964 content.insert(
1965 "content_ref".to_string(),
1966 Value::String(content_ref.clone()),
1967 );
1968 }
1969 content.insert(
1970 "metadata".to_string(),
1971 Value::Object(media_metadata.clone()),
1972 );
1973 if !options.lineage.is_empty() {
1974 content.insert(
1975 "lineage".to_string(),
1976 Value::Object(options.lineage.clone()),
1977 );
1978 }
1979 if !options.derived_outputs.is_empty() {
1980 content.insert(
1981 "derived_outputs".to_string(),
1982 Value::Object(options.derived_outputs.clone()),
1983 );
1984 }
1985 let mut media = State::new();
1986 media.insert(
1987 "schema_version".to_string(),
1988 Value::String(MEDIA_SCHEMA_VERSION.to_string()),
1989 );
1990 media.insert("kind".to_string(), Value::String(kind.to_string()));
1991 if let Some(uri) = options.uri {
1992 media.insert("uri".to_string(), Value::String(uri));
1993 }
1994 if let Some(content_ref) = options.content_ref {
1995 media.insert("content_ref".to_string(), Value::String(content_ref));
1996 }
1997 media.insert("metadata".to_string(), Value::Object(media_metadata));
1998 media.insert("lineage".to_string(), Value::Object(options.lineage));
1999 let mut artifact_metadata = options.metadata;
2000 artifact_metadata.insert("agentledger_media".to_string(), Value::Object(media));
2001 self.create_artifact(ctx, name, content, artifact_metadata)
2002 }
2003
2004 pub fn create_stream_checkpoint(
2005 &mut self,
2006 ctx: &AgentContext,
2007 name: &str,
2008 options: StreamCheckpointOptions,
2009 ) -> Result<String> {
2010 if options.stream_id.is_empty() || options.consumer_id.is_empty() {
2011 return Err(RuntimeError(
2012 "stream_id and consumer_id are required".to_string(),
2013 ));
2014 }
2015 let chunk = options.chunk.map(stream_chunk_to_state);
2016 let mut content = State::new();
2017 content.insert(
2018 "schema_version".to_string(),
2019 Value::String(STREAM_SCHEMA_VERSION.to_string()),
2020 );
2021 content.insert(
2022 "stream_id".to_string(),
2023 Value::String(options.stream_id.clone()),
2024 );
2025 content.insert(
2026 "consumer_id".to_string(),
2027 Value::String(options.consumer_id.clone()),
2028 );
2029 content.insert("offset".to_string(), options.offset.clone());
2030 if let Some(watermark) = &options.watermark {
2031 content.insert("watermark".to_string(), watermark.clone());
2032 }
2033 if let Some(chunk_state) = &chunk {
2034 content.insert("chunk".to_string(), Value::Object(chunk_state.clone()));
2035 }
2036 if let Some(partial_ref) = &options.partial_result_ref {
2037 content.insert(
2038 "partial_result_ref".to_string(),
2039 Value::String(partial_ref.clone()),
2040 );
2041 }
2042 if !options.backpressure.is_empty() {
2043 content.insert(
2044 "backpressure".to_string(),
2045 Value::Object(options.backpressure.clone()),
2046 );
2047 }
2048 if !options.metadata.is_empty() {
2049 content.insert("metadata".to_string(), Value::Object(options.metadata));
2050 }
2051 let mut stream = State::new();
2052 stream.insert(
2053 "schema_version".to_string(),
2054 Value::String(STREAM_SCHEMA_VERSION.to_string()),
2055 );
2056 stream.insert("stream_id".to_string(), Value::String(options.stream_id));
2057 stream.insert(
2058 "consumer_id".to_string(),
2059 Value::String(options.consumer_id),
2060 );
2061 stream.insert("offset".to_string(), options.offset);
2062 if let Some(watermark) = options.watermark {
2063 stream.insert("watermark".to_string(), watermark);
2064 }
2065 if let Some(chunk_state) = chunk {
2066 stream.insert("chunk".to_string(), Value::Object(chunk_state));
2067 }
2068 if let Some(partial_ref) = options.partial_result_ref {
2069 stream.insert("partial_result_ref".to_string(), Value::String(partial_ref));
2070 }
2071 if !options.backpressure.is_empty() {
2072 stream.insert(
2073 "backpressure".to_string(),
2074 Value::Object(options.backpressure),
2075 );
2076 }
2077 let mut artifact_metadata = State::new();
2078 artifact_metadata.insert("agentledger_stream".to_string(), Value::Object(stream));
2079 self.create_artifact(ctx, name, content, artifact_metadata)
2080 }
2081
2082 pub fn record_model_call(
2083 &mut self,
2084 ctx: &AgentContext,
2085 model: &str,
2086 input_tokens: f64,
2087 output_tokens: f64,
2088 total_usd: f64,
2089 ) -> Result<()> {
2090 let tokens = input_tokens + output_tokens;
2091 if let Some(max) = self.budget.max_model_tokens {
2092 let used = self.store.cost_summary(&ctx.run_id).model_tokens;
2093 if used + tokens > max {
2094 let message = format!("model token budget exceeded: {used}+{tokens}/{max}");
2095 let mut payload = State::new();
2096 payload.insert("category".to_string(), Value::String("model".to_string()));
2097 payload.insert("model".to_string(), Value::String(model.to_string()));
2098 payload.insert("error".to_string(), Value::String(message.clone()));
2099 self.store.append_event(
2100 &ctx.run_id,
2101 Some(&ctx.session_id),
2102 Some(&ctx.step_id),
2103 "budget_check_failed",
2104 payload,
2105 Some(&ctx.agent_role),
2106 Some(ctx.state_version),
2107 None,
2108 );
2109 return Err(RuntimeError(message));
2110 }
2111 }
2112 let mut payload = State::new();
2113 payload.insert("model".to_string(), Value::String(model.to_string()));
2114 payload.insert("input_tokens".to_string(), Value::Number(input_tokens));
2115 payload.insert("output_tokens".to_string(), Value::Number(output_tokens));
2116 payload.insert("total_tokens".to_string(), Value::Number(tokens));
2117 payload.insert("total_usd".to_string(), Value::Number(total_usd));
2118 self.store.append_event(
2119 &ctx.run_id,
2120 Some(&ctx.session_id),
2121 Some(&ctx.step_id),
2122 "model_call_completed",
2123 payload,
2124 Some(&ctx.agent_role),
2125 Some(ctx.state_version),
2126 None,
2127 );
2128 if tokens > 0.0 {
2129 self.store.record_cost(
2130 &ctx.run_id,
2131 &ctx.session_id,
2132 &ctx.step_id,
2133 "model",
2134 model,
2135 tokens,
2136 "token",
2137 State::new(),
2138 );
2139 }
2140 if total_usd > 0.0 {
2141 self.store.record_cost(
2142 &ctx.run_id,
2143 &ctx.session_id,
2144 &ctx.step_id,
2145 "model",
2146 model,
2147 total_usd,
2148 "usd",
2149 State::new(),
2150 );
2151 }
2152 Ok(())
2153 }
2154}
2155
2156impl Default for Runtime {
2157 fn default() -> Self {
2158 Self::new()
2159 }
2160}
2161
2162#[derive(Clone, Debug)]
2163pub struct AgentContext {
2164 pub run_id: String,
2165 pub session_id: String,
2166 pub step_id: String,
2167 pub agent_role: String,
2168 pub lease_token: String,
2169 pub attempt: u64,
2170 pub state_version: u64,
2171 pub pending_patch: State,
2172}
2173
2174impl AgentContext {
2175 pub fn write_state(&mut self, key: &str, value: Value) {
2176 self.pending_patch.insert(key.to_string(), value);
2177 }
2178}
2179
2180#[derive(Clone, Debug)]
2181pub struct EvidenceBundle {
2182 pub schema_version: String,
2183 pub bundle_hash: String,
2184 pub run: Run,
2185 pub steps: Vec<Step>,
2186 pub events: Vec<Event>,
2187 pub tool_ledger: Vec<ToolLedgerEntry>,
2188 pub approvals: Vec<ApprovalRequest>,
2189 pub artifacts: Vec<Artifact>,
2190 pub media_artifacts: Vec<State>,
2191 pub stream_checkpoints: Vec<State>,
2192 pub cost_records: Vec<CostRecord>,
2193 pub cost_summary: CostSummary,
2194 pub summary: State,
2195 pub final_state: State,
2196}
2197
2198#[derive(Clone, Debug, Default)]
2199pub struct WorkerRunSummary {
2200 pub worker_id: String,
2201 pub run_id: Option<String>,
2202 pub iterations: u64,
2203 pub attempts: u64,
2204 pub succeeded_attempts: u64,
2205 pub recovered_leases: u64,
2206 pub final_status: Option<String>,
2207 pub stopped_reason: String,
2208}
2209
2210pub struct LocalWorker {
2211 pub worker_id: String,
2212 pub agent_role: String,
2213 pub lease_seconds: f64,
2214 pub recover_expired: bool,
2215}
2216
2217impl LocalWorker {
2218 pub fn new(worker_id: &str, agent_role: &str) -> Self {
2219 Self {
2220 worker_id: worker_id.to_string(),
2221 agent_role: agent_role.to_string(),
2222 lease_seconds: 60.0,
2223 recover_expired: true,
2224 }
2225 }
2226
2227 pub fn run_until_idle(
2228 &self,
2229 runtime: &mut Runtime,
2230 run_id: &str,
2231 max_iterations: u64,
2232 agent: AgentFunc,
2233 ) -> Result<WorkerRunSummary> {
2234 let mut summary = WorkerRunSummary {
2235 worker_id: self.worker_id.clone(),
2236 run_id: Some(run_id.to_string()),
2237 stopped_reason: "max_iterations".to_string(),
2238 ..WorkerRunSummary::default()
2239 };
2240 for index in 1..=max_iterations {
2241 summary.iterations = index;
2242 if self.recover_expired {
2243 summary.recovered_leases += runtime.store.recover_expired_leases() as u64;
2244 }
2245 let status = runtime.store.run(run_id)?.status;
2246 if is_terminal_status(&status) {
2247 summary.final_status = Some(status);
2248 summary.stopped_reason = "terminal_status".to_string();
2249 break;
2250 }
2251 let ok = runtime.run_once(
2252 run_id,
2253 &self.worker_id,
2254 &self.agent_role,
2255 self.lease_seconds,
2256 agent,
2257 )?;
2258 if !ok {
2259 summary.stopped_reason = "idle".to_string();
2260 break;
2261 }
2262 summary.attempts += 1;
2263 summary.succeeded_attempts += 1;
2264 }
2265 let status = runtime.store.run(run_id)?.status;
2266 summary.final_status = Some(status.clone());
2267 if is_terminal_status(&status) {
2268 summary.stopped_reason = "terminal_status".to_string();
2269 }
2270 Ok(summary)
2271 }
2272}
2273
2274#[derive(Clone, Debug, Default)]
2275pub struct WorkerServiceSummary {
2276 pub worker_id: String,
2277 pub run_id: Option<String>,
2278 pub loops: u64,
2279 pub attempts: u64,
2280 pub succeeded_attempts: u64,
2281 pub recovered_leases: u64,
2282 pub idle_polls: u64,
2283 pub stopped_reason: String,
2284 pub final_status: Option<String>,
2285 pub stop_requested: bool,
2286}
2287
2288pub struct WorkerService {
2289 pub worker: LocalWorker,
2290 pub stop_requested: bool,
2291 pub stop_reason: String,
2292}
2293
2294impl WorkerService {
2295 pub fn new(worker: LocalWorker) -> Self {
2296 Self {
2297 worker,
2298 stop_requested: false,
2299 stop_reason: "stop_requested".to_string(),
2300 }
2301 }
2302
2303 pub fn request_stop(&mut self, reason: &str) {
2304 self.stop_requested = true;
2305 self.stop_reason = reason.to_string();
2306 }
2307
2308 pub fn serve(
2309 &mut self,
2310 runtime: &mut Runtime,
2311 run_id: Option<&str>,
2312 max_loops: u64,
2313 max_idle_polls: Option<u64>,
2314 agent: AgentFunc,
2315 ) -> Result<WorkerServiceSummary> {
2316 let mut summary = WorkerServiceSummary {
2317 worker_id: self.worker.worker_id.clone(),
2318 run_id: run_id.map(str::to_string),
2319 stopped_reason: "max_loops".to_string(),
2320 ..WorkerServiceSummary::default()
2321 };
2322 while summary.loops < max_loops {
2323 if self.stop_requested {
2324 summary.stopped_reason = self.stop_reason.clone();
2325 summary.stop_requested = true;
2326 break;
2327 }
2328 summary.loops += 1;
2329 let Some(run_id) = run_id else {
2330 summary.idle_polls += 1;
2331 if max_idle_polls.is_some_and(|limit| summary.idle_polls >= limit) {
2332 summary.stopped_reason = "idle".to_string();
2333 break;
2334 }
2335 continue;
2336 };
2337 let run_summary = self.worker.run_until_idle(runtime, run_id, 1, agent)?;
2338 summary.attempts += run_summary.attempts;
2339 summary.succeeded_attempts += run_summary.succeeded_attempts;
2340 summary.recovered_leases += run_summary.recovered_leases;
2341 summary.final_status = run_summary.final_status.clone();
2342 if summary
2343 .final_status
2344 .as_deref()
2345 .is_some_and(is_terminal_status)
2346 {
2347 summary.stopped_reason = "terminal_status".to_string();
2348 break;
2349 }
2350 if run_summary.attempts == 0 {
2351 summary.idle_polls += 1;
2352 if max_idle_polls.is_some_and(|limit| summary.idle_polls >= limit) {
2353 summary.stopped_reason = "idle".to_string();
2354 break;
2355 }
2356 } else {
2357 summary.idle_polls = 0;
2358 }
2359 }
2360 Ok(summary)
2361 }
2362}
2363
2364fn is_terminal_status(status: &str) -> bool {
2365 matches!(status, "completed" | "failed" | "cancelled")
2366}
2367
2368pub fn export_evidence(store: &MemoryStore, run_id: &str) -> Result<EvidenceBundle> {
2369 let run = store.run(run_id)?;
2370 let steps = store.steps(run_id);
2371 let events = store.events(run_id);
2372 let tool_ledger = store.ledger(run_id);
2373 let approvals = store.approval_requests(run_id);
2374 let artifacts = store.artifacts(run_id);
2375 let media_artifacts = media_artifacts_from(&artifacts);
2376 let stream_checkpoints = stream_checkpoints_from(&artifacts);
2377 let cost_records = store.cost_records(run_id);
2378 let cost_summary = store.cost_summary(run_id);
2379 let summary = evidence_summary(
2380 &steps,
2381 &events,
2382 &tool_ledger,
2383 &approvals,
2384 &artifacts,
2385 &media_artifacts,
2386 &stream_checkpoints,
2387 &cost_records,
2388 &cost_summary,
2389 );
2390 let final_state = store.final_state(run_id)?;
2391 let basis = format!(
2392 "{}:{}:{}:{}:{}",
2393 run_id,
2394 steps.len(),
2395 events.len(),
2396 tool_ledger.len(),
2397 cost_records.len()
2398 );
2399 Ok(EvidenceBundle {
2400 schema_version: "agentledger.evidence.v1".to_string(),
2401 bundle_hash: stable_hash(&basis),
2402 run,
2403 steps,
2404 events,
2405 tool_ledger,
2406 approvals,
2407 artifacts,
2408 media_artifacts,
2409 stream_checkpoints,
2410 cost_records,
2411 cost_summary,
2412 summary,
2413 final_state,
2414 })
2415}
2416
2417fn evidence_summary(
2418 steps: &[Step],
2419 events: &[Event],
2420 ledger: &[ToolLedgerEntry],
2421 approvals: &[ApprovalRequest],
2422 artifacts: &[Artifact],
2423 media_artifacts: &[State],
2424 stream_checkpoints: &[State],
2425 cost_records: &[CostRecord],
2426 cost_summary: &CostSummary,
2427) -> State {
2428 let mut summary = State::new();
2429 summary.insert("event_count".into(), Value::Number(events.len() as f64));
2430 summary.insert("step_count".into(), Value::Number(steps.len() as f64));
2431 summary.insert(
2432 "tool_ledger_count".into(),
2433 Value::Number(ledger.len() as f64),
2434 );
2435 summary.insert(
2436 "approval_count".into(),
2437 Value::Number(approvals.len() as f64),
2438 );
2439 summary.insert(
2440 "artifact_count".into(),
2441 Value::Number(artifacts.len() as f64),
2442 );
2443 summary.insert(
2444 "media_artifact_count".into(),
2445 Value::Number(media_artifacts.len() as f64),
2446 );
2447 summary.insert(
2448 "stream_checkpoint_count".into(),
2449 Value::Number(stream_checkpoints.len() as f64),
2450 );
2451 summary.insert(
2452 "cost_record_count".into(),
2453 Value::Number(cost_records.len() as f64),
2454 );
2455 summary.insert(
2456 "has_failed_steps".into(),
2457 Value::Bool(steps.iter().any(|step| step.status == "failed")),
2458 );
2459 summary.insert(
2460 "has_pending_verification".into(),
2461 Value::Bool(
2462 ledger
2463 .iter()
2464 .any(|row| row.status == "PENDING_VERIFICATION"),
2465 ),
2466 );
2467 summary.insert(
2468 "has_pending_approvals".into(),
2469 Value::Bool(approvals.iter().any(|row| row.status == "PENDING")),
2470 );
2471 let mut cost = State::new();
2472 cost.insert("tool_calls".into(), Value::Number(cost_summary.tool_calls));
2473 cost.insert(
2474 "model_tokens".into(),
2475 Value::Number(cost_summary.model_tokens),
2476 );
2477 cost.insert("total_usd".into(), Value::Number(cost_summary.total_usd));
2478 summary.insert("cost_summary".into(), Value::Object(cost));
2479 summary
2480}
2481
2482#[derive(Clone, Debug)]
2483pub struct ReplaySummary {
2484 pub run_id: String,
2485 pub event_count: usize,
2486 pub tool_call_count: usize,
2487 pub final_state: State,
2488 pub event_hash: String,
2489 pub replay_safe: bool,
2490 pub artifact_count: usize,
2491 pub media_artifact_count: usize,
2492 pub stream_checkpoint_count: usize,
2493}
2494
2495pub fn replay(store: &MemoryStore, run_id: &str) -> Result<ReplaySummary> {
2496 let events = store.events(run_id);
2497 let tool_call_count = events
2498 .iter()
2499 .filter(|event| event.event_type.starts_with("tool_call_"))
2500 .count();
2501 let digest = events
2502 .iter()
2503 .map(|event| format!("{}:{}:{}", event.seq, event.event_type, event.payload_hash))
2504 .collect::<Vec<_>>()
2505 .join("|");
2506 let artifacts = store.artifacts(run_id);
2507 Ok(ReplaySummary {
2508 run_id: run_id.to_string(),
2509 event_count: events.len(),
2510 tool_call_count,
2511 final_state: store.final_state(run_id)?,
2512 event_hash: stable_hash(&digest),
2513 replay_safe: true,
2514 artifact_count: artifacts.len(),
2515 media_artifact_count: media_artifacts_from(&artifacts).len(),
2516 stream_checkpoint_count: stream_checkpoints_from(&artifacts).len(),
2517 })
2518}
2519
2520#[derive(Clone, Debug)]
2521pub struct CostAttributionReport {
2522 pub run_id: String,
2523 pub total: CostSummary,
2524 pub by_agent: HashMap<String, CostSummary>,
2525 pub by_step: HashMap<String, CostSummary>,
2526 pub by_name: HashMap<String, CostSummary>,
2527}
2528
2529pub fn cost_attribution(store: &MemoryStore, run_id: &str) -> CostAttributionReport {
2530 let mut step_roles = HashMap::new();
2531 for event in store.events(run_id) {
2532 if let (Some(step_id), Some(agent_role)) = (event.step_id, event.agent_role) {
2533 step_roles.insert(step_id, agent_role);
2534 }
2535 }
2536 let mut report = CostAttributionReport {
2537 run_id: run_id.to_string(),
2538 total: CostSummary::default(),
2539 by_agent: HashMap::new(),
2540 by_step: HashMap::new(),
2541 by_name: HashMap::new(),
2542 };
2543 for record in store.cost_records(run_id) {
2544 add_cost(&mut report.total, &record);
2545 let agent = step_roles
2546 .get(&record.step_id)
2547 .cloned()
2548 .unwrap_or_else(|| "<unknown>".to_string());
2549 add_cost(report.by_agent.entry(agent).or_default(), &record);
2550 add_cost(
2551 report.by_step.entry(record.step_id.clone()).or_default(),
2552 &record,
2553 );
2554 add_cost(
2555 report.by_name.entry(record.name.clone()).or_default(),
2556 &record,
2557 );
2558 }
2559 report
2560}
2561
2562#[derive(Clone, Debug)]
2563pub struct FailureAttributionReport {
2564 pub run_id: String,
2565 pub run_status: String,
2566 pub failed_steps: Vec<Step>,
2567 pub pending_verification: Vec<ToolLedgerEntry>,
2568 pub pending_approvals: Vec<ApprovalRequest>,
2569 pub failure_events: Vec<Event>,
2570 pub summary: State,
2571}
2572
2573pub fn failure_attribution(store: &MemoryStore, run_id: &str) -> Result<FailureAttributionReport> {
2574 let run = store.run(run_id)?;
2575 let failed_steps: Vec<Step> = store
2576 .steps(run_id)
2577 .into_iter()
2578 .filter(|step| step.status == "failed")
2579 .collect();
2580 let pending_verification: Vec<ToolLedgerEntry> = store
2581 .ledger(run_id)
2582 .into_iter()
2583 .filter(|entry| entry.status == "PENDING_VERIFICATION")
2584 .collect();
2585 let pending_approvals: Vec<ApprovalRequest> = store
2586 .approval_requests(run_id)
2587 .into_iter()
2588 .filter(|entry| entry.status == "PENDING")
2589 .collect();
2590 let failure_events: Vec<Event> = store
2591 .events(run_id)
2592 .into_iter()
2593 .filter(|event| is_failure_event(&event.event_type))
2594 .collect();
2595 let mut summary = State::new();
2596 summary.insert(
2597 "failed_step_count".to_string(),
2598 Value::Number(failed_steps.len() as f64),
2599 );
2600 summary.insert(
2601 "pending_verification_count".to_string(),
2602 Value::Number(pending_verification.len() as f64),
2603 );
2604 summary.insert(
2605 "pending_approval_count".to_string(),
2606 Value::Number(pending_approvals.len() as f64),
2607 );
2608 summary.insert(
2609 "failure_event_count".to_string(),
2610 Value::Number(failure_events.len() as f64),
2611 );
2612 Ok(FailureAttributionReport {
2613 run_id: run_id.to_string(),
2614 run_status: run.status,
2615 failed_steps,
2616 pending_verification,
2617 pending_approvals,
2618 failure_events,
2619 summary,
2620 })
2621}
2622
2623fn encode_store(store: &MemoryStore) -> String {
2624 let mut lines = vec!["AGENTLEDGER_RUST_STORE_V1".to_string()];
2625 let mut runs: Vec<_> = store.runs.values().collect();
2626 runs.sort_by(|a, b| a.run_id.cmp(&b.run_id));
2627 for run in runs {
2628 lines.push(join_fields(&[
2629 "R".to_string(),
2630 hex_encode(&run.run_id),
2631 hex_encode(&run.session_id),
2632 hex_encode(&run.status),
2633 run.state_version.to_string(),
2634 run.created_at.to_string(),
2635 run.updated_at.to_string(),
2636 encode_state(&run.state),
2637 ]));
2638 }
2639
2640 let mut steps: Vec<_> = store.steps.values().collect();
2641 steps.sort_by(|a, b| a.step_id.cmp(&b.step_id));
2642 for step in steps {
2643 lines.push(join_fields(&[
2644 "S".to_string(),
2645 hex_encode(&step.step_id),
2646 hex_encode(&step.run_id),
2647 hex_encode(&step.session_id),
2648 hex_encode(&step.status),
2649 encode_option_string(&step.owner),
2650 encode_option_string(&step.lease_token),
2651 encode_option_f64(step.lease_until),
2652 step.attempt.to_string(),
2653 step.state_version.to_string(),
2654 encode_option_string(&step.checkpoint_id),
2655 encode_option_string(&step.last_error_type),
2656 encode_option_string(&step.last_error),
2657 encode_option_f64(step.cancelled_at),
2658 step.created_at.to_string(),
2659 step.updated_at.to_string(),
2660 ]));
2661 }
2662
2663 let mut run_ids: Vec<_> = store.events.keys().collect();
2664 run_ids.sort();
2665 for run_id in run_ids {
2666 if let Some(events) = store.events.get(run_id) {
2667 for event in events {
2668 lines.push(join_fields(&[
2669 "E".to_string(),
2670 hex_encode(&event.event_id),
2671 hex_encode(&event.run_id),
2672 encode_option_string(&event.session_id),
2673 encode_option_string(&event.step_id),
2674 event.seq.to_string(),
2675 hex_encode(&event.event_type),
2676 event.timestamp.to_string(),
2677 encode_option_string(&event.agent_role),
2678 encode_option_u64(event.state_version),
2679 encode_option_string(&event.causal_token),
2680 hex_encode(&event.payload_hash),
2681 hex_encode(&event.payload_ref),
2682 encode_state(&event.payload),
2683 ]));
2684 }
2685 }
2686 }
2687
2688 let mut ledgers: Vec<_> = store.tool_ledger.values().collect();
2689 ledgers.sort_by(|a, b| a.idempotency_key.cmp(&b.idempotency_key));
2690 for entry in ledgers {
2691 lines.push(join_fields(&[
2692 "L".to_string(),
2693 hex_encode(&entry.ledger_id),
2694 hex_encode(&entry.run_id),
2695 hex_encode(&entry.session_id),
2696 hex_encode(&entry.step_id),
2697 hex_encode(&entry.tool_name),
2698 hex_encode(&entry.tool_version),
2699 hex_encode(&entry.tool_call_id),
2700 hex_encode(&entry.idempotency_key),
2701 hex_encode(&entry.causal_token),
2702 hex_encode(&entry.request_hash),
2703 hex_encode(&entry.request_ref),
2704 hex_encode(&entry.status),
2705 encode_option_string(&entry.external_id),
2706 encode_option_string(&entry.response_hash),
2707 encode_option_string(&entry.response_ref),
2708 encode_option_string(&entry.error_type),
2709 encode_option_value(&entry.response),
2710 entry.created_at.to_string(),
2711 entry.updated_at.to_string(),
2712 ]));
2713 }
2714
2715 let mut approvals: Vec<_> = store.approval_requests.values().collect();
2716 approvals.sort_by(|a, b| a.approval_key.cmp(&b.approval_key));
2717 for approval in approvals {
2718 lines.push(join_fields(&[
2719 "A".to_string(),
2720 hex_encode(&approval.approval_id),
2721 hex_encode(&approval.approval_key),
2722 hex_encode(&approval.run_id),
2723 hex_encode(&approval.session_id),
2724 hex_encode(&approval.step_id),
2725 hex_encode(&approval.tool_name),
2726 hex_encode(&approval.risk_level),
2727 hex_encode(&approval.status),
2728 hex_encode(&approval.reason),
2729 hex_encode(&approval.request_hash),
2730 hex_encode(&approval.request_ref),
2731 hex_encode(&approval.requested_by),
2732 encode_option_string(&approval.approved_by),
2733 encode_option_string(&approval.decision_reason),
2734 approval.created_at.to_string(),
2735 approval.updated_at.to_string(),
2736 ]));
2737 }
2738
2739 let mut cost_run_ids: Vec<_> = store.cost_records.keys().collect();
2740 cost_run_ids.sort();
2741 for run_id in cost_run_ids {
2742 if let Some(records) = store.cost_records.get(run_id) {
2743 for record in records {
2744 lines.push(join_fields(&[
2745 "C".to_string(),
2746 hex_encode(&record.cost_id),
2747 hex_encode(&record.run_id),
2748 hex_encode(&record.session_id),
2749 hex_encode(&record.step_id),
2750 hex_encode(&record.category),
2751 hex_encode(&record.name),
2752 record.amount.to_string(),
2753 hex_encode(&record.unit),
2754 encode_state(&record.metadata),
2755 record.created_at.to_string(),
2756 ]));
2757 }
2758 }
2759 }
2760
2761 let mut artifact_run_ids: Vec<_> = store.artifacts.keys().collect();
2762 artifact_run_ids.sort();
2763 for run_id in artifact_run_ids {
2764 if let Some(artifacts) = store.artifacts.get(run_id) {
2765 for artifact in artifacts {
2766 lines.push(join_fields(&[
2767 "F".to_string(),
2768 hex_encode(&artifact.artifact_id),
2769 hex_encode(&artifact.run_id),
2770 encode_option_string(&artifact.step_id),
2771 hex_encode(&artifact.name),
2772 hex_encode(&artifact.blob_hash),
2773 hex_encode(&artifact.blob_ref),
2774 encode_state(&artifact.metadata),
2775 artifact.created_at.to_string(),
2776 ]));
2777 }
2778 }
2779 }
2780
2781 lines.push(String::new());
2782 lines.join("\n")
2783}
2784
2785fn decode_store(body: &str) -> Result<MemoryStore> {
2786 let mut lines = body.lines();
2787 match lines.next() {
2788 Some("AGENTLEDGER_RUST_STORE_V1") => {}
2789 _ => {
2790 return Err(RuntimeError(
2791 "invalid Rust store snapshot header".to_string(),
2792 ))
2793 }
2794 }
2795 let mut store = MemoryStore::new();
2796 for line in lines.filter(|line| !line.trim().is_empty()) {
2797 let fields: Vec<&str> = line.split('\t').collect();
2798 let tag = fields.first().copied().unwrap_or_default();
2799 match tag {
2800 "R" => {
2801 require_len(tag, &fields, 8)?;
2802 let run = Run {
2803 run_id: hex_decode(fields[1])?,
2804 session_id: hex_decode(fields[2])?,
2805 status: hex_decode(fields[3])?,
2806 state_version: parse_u64(fields[4])?,
2807 created_at: parse_f64(fields[5])?,
2808 updated_at: parse_f64(fields[6])?,
2809 state: decode_state(fields[7])?,
2810 };
2811 store.runs.insert(run.run_id.clone(), run);
2812 }
2813 "S" => {
2814 require_len(tag, &fields, 16)?;
2815 let step = Step {
2816 step_id: hex_decode(fields[1])?,
2817 run_id: hex_decode(fields[2])?,
2818 session_id: hex_decode(fields[3])?,
2819 status: hex_decode(fields[4])?,
2820 owner: decode_option_string(fields[5])?,
2821 lease_token: decode_option_string(fields[6])?,
2822 lease_until: decode_option_f64(fields[7])?,
2823 attempt: parse_u64(fields[8])?,
2824 state_version: parse_u64(fields[9])?,
2825 checkpoint_id: decode_option_string(fields[10])?,
2826 last_error_type: decode_option_string(fields[11])?,
2827 last_error: decode_option_string(fields[12])?,
2828 cancelled_at: decode_option_f64(fields[13])?,
2829 created_at: parse_f64(fields[14])?,
2830 updated_at: parse_f64(fields[15])?,
2831 };
2832 store.steps.insert(step.step_id.clone(), step);
2833 }
2834 "E" => {
2835 require_len(tag, &fields, 14)?;
2836 let event = Event {
2837 event_id: hex_decode(fields[1])?,
2838 run_id: hex_decode(fields[2])?,
2839 session_id: decode_option_string(fields[3])?,
2840 step_id: decode_option_string(fields[4])?,
2841 seq: parse_u64(fields[5])?,
2842 event_type: hex_decode(fields[6])?,
2843 timestamp: parse_f64(fields[7])?,
2844 agent_role: decode_option_string(fields[8])?,
2845 state_version: decode_option_u64(fields[9])?,
2846 causal_token: decode_option_string(fields[10])?,
2847 payload_hash: hex_decode(fields[11])?,
2848 payload_ref: hex_decode(fields[12])?,
2849 payload: decode_state(fields[13])?,
2850 };
2851 store
2852 .events
2853 .entry(event.run_id.clone())
2854 .or_default()
2855 .push(event);
2856 }
2857 "L" => {
2858 require_len(tag, &fields, 20)?;
2859 let entry = ToolLedgerEntry {
2860 ledger_id: hex_decode(fields[1])?,
2861 run_id: hex_decode(fields[2])?,
2862 session_id: hex_decode(fields[3])?,
2863 step_id: hex_decode(fields[4])?,
2864 tool_name: hex_decode(fields[5])?,
2865 tool_version: hex_decode(fields[6])?,
2866 tool_call_id: hex_decode(fields[7])?,
2867 idempotency_key: hex_decode(fields[8])?,
2868 causal_token: hex_decode(fields[9])?,
2869 request_hash: hex_decode(fields[10])?,
2870 request_ref: hex_decode(fields[11])?,
2871 status: hex_decode(fields[12])?,
2872 external_id: decode_option_string(fields[13])?,
2873 response_hash: decode_option_string(fields[14])?,
2874 response_ref: decode_option_string(fields[15])?,
2875 error_type: decode_option_string(fields[16])?,
2876 response: decode_option_value(fields[17])?,
2877 created_at: parse_f64(fields[18])?,
2878 updated_at: parse_f64(fields[19])?,
2879 };
2880 store
2881 .tool_ledger
2882 .insert(entry.idempotency_key.clone(), entry);
2883 }
2884 "A" => {
2885 require_len(tag, &fields, 17)?;
2886 let approval = ApprovalRequest {
2887 approval_id: hex_decode(fields[1])?,
2888 approval_key: hex_decode(fields[2])?,
2889 run_id: hex_decode(fields[3])?,
2890 session_id: hex_decode(fields[4])?,
2891 step_id: hex_decode(fields[5])?,
2892 tool_name: hex_decode(fields[6])?,
2893 risk_level: hex_decode(fields[7])?,
2894 status: hex_decode(fields[8])?,
2895 reason: hex_decode(fields[9])?,
2896 request_hash: hex_decode(fields[10])?,
2897 request_ref: hex_decode(fields[11])?,
2898 requested_by: hex_decode(fields[12])?,
2899 approved_by: decode_option_string(fields[13])?,
2900 decision_reason: decode_option_string(fields[14])?,
2901 created_at: parse_f64(fields[15])?,
2902 updated_at: parse_f64(fields[16])?,
2903 };
2904 store
2905 .approval_requests
2906 .insert(approval.approval_key.clone(), approval);
2907 }
2908 "C" => {
2909 require_len(tag, &fields, 11)?;
2910 let record = CostRecord {
2911 cost_id: hex_decode(fields[1])?,
2912 run_id: hex_decode(fields[2])?,
2913 session_id: hex_decode(fields[3])?,
2914 step_id: hex_decode(fields[4])?,
2915 category: hex_decode(fields[5])?,
2916 name: hex_decode(fields[6])?,
2917 amount: parse_f64(fields[7])?,
2918 unit: hex_decode(fields[8])?,
2919 metadata: decode_state(fields[9])?,
2920 created_at: parse_f64(fields[10])?,
2921 };
2922 store
2923 .cost_records
2924 .entry(record.run_id.clone())
2925 .or_default()
2926 .push(record);
2927 }
2928 "F" => {
2929 require_len(tag, &fields, 9)?;
2930 let artifact = Artifact {
2931 artifact_id: hex_decode(fields[1])?,
2932 run_id: hex_decode(fields[2])?,
2933 step_id: decode_option_string(fields[3])?,
2934 name: hex_decode(fields[4])?,
2935 blob_hash: hex_decode(fields[5])?,
2936 blob_ref: hex_decode(fields[6])?,
2937 metadata: decode_state(fields[7])?,
2938 created_at: parse_f64(fields[8])?,
2939 };
2940 store
2941 .artifacts
2942 .entry(artifact.run_id.clone())
2943 .or_default()
2944 .push(artifact);
2945 }
2946 _ => {
2947 return Err(RuntimeError(format!(
2948 "unknown Rust store snapshot row: {tag}"
2949 )))
2950 }
2951 }
2952 }
2953 Ok(store)
2954}
2955
2956fn join_fields(fields: &[String]) -> String {
2957 fields.join("\t")
2958}
2959
2960fn require_len(tag: &str, fields: &[&str], expected: usize) -> Result<()> {
2961 if fields.len() != expected {
2962 return Err(RuntimeError(format!(
2963 "invalid {tag} row: expected {expected} fields, got {}",
2964 fields.len()
2965 )));
2966 }
2967 Ok(())
2968}
2969
2970fn encode_option_string(value: &Option<String>) -> String {
2971 value
2972 .as_ref()
2973 .map(|item| hex_encode(item))
2974 .unwrap_or_else(|| "-".to_string())
2975}
2976
2977fn decode_option_string(value: &str) -> Result<Option<String>> {
2978 if value == "-" {
2979 Ok(None)
2980 } else {
2981 Ok(Some(hex_decode(value)?))
2982 }
2983}
2984
2985fn encode_option_f64(value: Option<f64>) -> String {
2986 value
2987 .map(|item| item.to_string())
2988 .unwrap_or_else(|| "-".to_string())
2989}
2990
2991fn decode_option_f64(value: &str) -> Result<Option<f64>> {
2992 if value == "-" {
2993 Ok(None)
2994 } else {
2995 Ok(Some(parse_f64(value)?))
2996 }
2997}
2998
2999fn encode_option_u64(value: Option<u64>) -> String {
3000 value
3001 .map(|item| item.to_string())
3002 .unwrap_or_else(|| "-".to_string())
3003}
3004
3005fn decode_option_u64(value: &str) -> Result<Option<u64>> {
3006 if value == "-" {
3007 Ok(None)
3008 } else {
3009 Ok(Some(parse_u64(value)?))
3010 }
3011}
3012
3013fn encode_option_value(value: &Option<Value>) -> String {
3014 value
3015 .as_ref()
3016 .map(encode_value)
3017 .unwrap_or_else(|| "-".to_string())
3018}
3019
3020fn decode_option_value(value: &str) -> Result<Option<Value>> {
3021 if value == "-" {
3022 Ok(None)
3023 } else {
3024 Ok(Some(decode_value(value)?))
3025 }
3026}
3027
3028fn encode_state(state: &State) -> String {
3029 encode_value(&Value::Object(state.clone()))
3030}
3031
3032fn decode_state(encoded: &str) -> Result<State> {
3033 match decode_value(encoded)? {
3034 Value::Object(state) => Ok(state),
3035 _ => Err(RuntimeError("encoded state was not an object".to_string())),
3036 }
3037}
3038
3039fn encode_value(value: &Value) -> String {
3040 match value {
3041 Value::Null => "Z".to_string(),
3042 Value::Bool(true) => "T".to_string(),
3043 Value::Bool(false) => "F".to_string(),
3044 Value::Number(item) => format!("N{}:", hex_encode(&item.to_string())),
3045 Value::String(item) => format!("S{}:", hex_encode(item)),
3046 Value::Object(state) => {
3047 let mut keys: Vec<_> = state.keys().collect();
3048 keys.sort();
3049 let mut out = format!("O{}:", keys.len());
3050 for key in keys {
3051 out.push_str(&hex_encode(key));
3052 out.push(':');
3053 out.push_str(&encode_value(&state[key]));
3054 }
3055 out
3056 }
3057 Value::Array(values) => {
3058 let mut out = format!("A{}:", values.len());
3059 for value in values {
3060 out.push_str(&encode_value(value));
3061 }
3062 out
3063 }
3064 }
3065}
3066
3067fn decode_value(encoded: &str) -> Result<Value> {
3068 let (value, index) = parse_value(encoded, 0)?;
3069 if index != encoded.len() {
3070 return Err(RuntimeError("trailing bytes in encoded value".to_string()));
3071 }
3072 Ok(value)
3073}
3074
3075fn parse_value(input: &str, index: usize) -> Result<(Value, usize)> {
3076 let bytes = input.as_bytes();
3077 let tag = *bytes
3078 .get(index)
3079 .ok_or_else(|| RuntimeError("unexpected end of encoded value".to_string()))?
3080 as char;
3081 match tag {
3082 'Z' => Ok((Value::Null, index + 1)),
3083 'T' => Ok((Value::Bool(true), index + 1)),
3084 'F' => Ok((Value::Bool(false), index + 1)),
3085 'N' => {
3086 let (hex, next) = read_until_colon(input, index + 1)?;
3087 Ok((Value::Number(parse_f64(&hex_decode(hex)?)?), next))
3088 }
3089 'S' => {
3090 let (hex, next) = read_until_colon(input, index + 1)?;
3091 Ok((Value::String(hex_decode(hex)?), next))
3092 }
3093 'O' => {
3094 let (count_text, mut next) = read_until_colon(input, index + 1)?;
3095 let count = count_text
3096 .parse::<usize>()
3097 .map_err(|err| RuntimeError(err.to_string()))?;
3098 let mut state = State::new();
3099 for _ in 0..count {
3100 let (key_hex, after_key) = read_until_colon(input, next)?;
3101 let key = hex_decode(key_hex)?;
3102 let (value, after_value) = parse_value(input, after_key)?;
3103 state.insert(key, value);
3104 next = after_value;
3105 }
3106 Ok((Value::Object(state), next))
3107 }
3108 'A' => {
3109 let (count_text, mut next) = read_until_colon(input, index + 1)?;
3110 let count = count_text
3111 .parse::<usize>()
3112 .map_err(|err| RuntimeError(err.to_string()))?;
3113 let mut values = Vec::with_capacity(count);
3114 for _ in 0..count {
3115 let (value, after_value) = parse_value(input, next)?;
3116 values.push(value);
3117 next = after_value;
3118 }
3119 Ok((Value::Array(values), next))
3120 }
3121 _ => Err(RuntimeError(format!("unknown encoded value tag: {tag}"))),
3122 }
3123}
3124
3125fn read_until_colon(input: &str, index: usize) -> Result<(&str, usize)> {
3126 let rest = input
3127 .get(index..)
3128 .ok_or_else(|| RuntimeError("invalid encoded value index".to_string()))?;
3129 let offset = rest
3130 .find(':')
3131 .ok_or_else(|| RuntimeError("missing encoded value delimiter".to_string()))?;
3132 Ok((&rest[..offset], index + offset + 1))
3133}
3134
3135fn hex_encode(value: &str) -> String {
3136 value
3137 .as_bytes()
3138 .iter()
3139 .map(|byte| format!("{byte:02x}"))
3140 .collect()
3141}
3142
3143fn hex_decode(value: &str) -> Result<String> {
3144 if value.len() % 2 != 0 {
3145 return Err(RuntimeError("invalid hex string length".to_string()));
3146 }
3147 let mut bytes = Vec::with_capacity(value.len() / 2);
3148 for index in (0..value.len()).step_by(2) {
3149 let byte = u8::from_str_radix(&value[index..index + 2], 16)
3150 .map_err(|err| RuntimeError(err.to_string()))?;
3151 bytes.push(byte);
3152 }
3153 String::from_utf8(bytes).map_err(|err| RuntimeError(err.to_string()))
3154}
3155
3156fn parse_f64(value: &str) -> Result<f64> {
3157 value
3158 .parse::<f64>()
3159 .map_err(|err| RuntimeError(err.to_string()))
3160}
3161
3162fn parse_u64(value: &str) -> Result<u64> {
3163 value
3164 .parse::<u64>()
3165 .map_err(|err| RuntimeError(err.to_string()))
3166}
3167
3168fn merge_patch(base: &State, patch: &State) -> State {
3169 let mut out = base.clone();
3170 for (key, value) in patch {
3171 match value {
3172 Value::Null => {
3173 out.remove(key);
3174 }
3175 Value::Object(patch_map) => {
3176 if let Some(Value::Object(base_map)) = out.get(key) {
3177 out.insert(key.clone(), Value::Object(merge_patch(base_map, patch_map)));
3178 } else {
3179 out.insert(key.clone(), value.clone());
3180 }
3181 }
3182 _ => {
3183 out.insert(key.clone(), value.clone());
3184 }
3185 }
3186 }
3187 out
3188}
3189
3190fn add_cost(summary: &mut CostSummary, record: &CostRecord) {
3191 if (record.category == "tool" || record.category == "tool_shadow") && record.unit == "call" {
3192 summary.tool_calls += record.amount;
3193 }
3194 if record.category == "model" && record.unit == "token" {
3195 summary.model_tokens += record.amount;
3196 }
3197 if record.unit == "usd" {
3198 summary.total_usd += record.amount;
3199 }
3200 let key = format!("{}:{}", record.category, record.unit);
3201 *summary.by_category.entry(key).or_insert(0.0) += record.amount;
3202}
3203
3204pub fn validate_tool_schema(schema: &Value, value: &Value, path: &str) -> Result<()> {
3205 let schema = match schema {
3206 Value::Object(schema) => schema,
3207 _ => return Err(RuntimeError(format!("{path} schema must be object"))),
3208 };
3209 if let Some(expected) = schema.get("const") {
3210 if expected != value {
3211 return Err(RuntimeError(format!("{path} expected const")));
3212 }
3213 }
3214 if let Some(Value::Array(items)) = schema.get("enum") {
3215 if !items.iter().any(|item| item == value) {
3216 return Err(RuntimeError(format!("{path} value not in enum")));
3217 }
3218 }
3219 let Some(Value::String(kind)) = schema.get("type") else {
3220 return Ok(());
3221 };
3222 match kind.as_str() {
3223 "object" => {
3224 let Value::Object(object) = value else {
3225 return Err(RuntimeError(format!("{path} expected object")));
3226 };
3227 if let Some(Value::Array(required)) = schema.get("required") {
3228 for item in required {
3229 if let Value::String(key) = item {
3230 if !object.contains_key(key) {
3231 return Err(RuntimeError(format!("{path}.{key} is required")));
3232 }
3233 }
3234 }
3235 }
3236 let properties = match schema.get("properties") {
3237 Some(Value::Object(properties)) => properties,
3238 _ => return Ok(()),
3239 };
3240 for (key, child_schema) in properties {
3241 if let Some(child) = object.get(key) {
3242 validate_tool_schema(child_schema, child, &format!("{path}.{key}"))?;
3243 }
3244 }
3245 if schema.get("additionalProperties") == Some(&Value::Bool(false)) {
3246 for key in object.keys() {
3247 if !properties.contains_key(key) {
3248 return Err(RuntimeError(format!("{path}.{key} is not allowed")));
3249 }
3250 }
3251 }
3252 }
3253 "string" => {
3254 let Value::String(text) = value else {
3255 return Err(RuntimeError(format!("{path} expected string")));
3256 };
3257 if let Some(Value::Number(min)) = schema.get("minLength") {
3258 if (text.len() as f64) < *min {
3259 return Err(RuntimeError(format!("{path} shorter than minLength")));
3260 }
3261 }
3262 if let Some(Value::Number(max)) = schema.get("maxLength") {
3263 if (text.len() as f64) > *max {
3264 return Err(RuntimeError(format!("{path} longer than maxLength")));
3265 }
3266 }
3267 }
3268 "number" | "integer" => {
3269 let Value::Number(number) = value else {
3270 return Err(RuntimeError(format!("{path} expected number")));
3271 };
3272 if kind == "integer" && number.fract() != 0.0 {
3273 return Err(RuntimeError(format!("{path} expected integer")));
3274 }
3275 if let Some(Value::Number(min)) = schema.get("minimum") {
3276 if number < min {
3277 return Err(RuntimeError(format!("{path} below minimum")));
3278 }
3279 }
3280 if let Some(Value::Number(max)) = schema.get("maximum") {
3281 if number > max {
3282 return Err(RuntimeError(format!("{path} above maximum")));
3283 }
3284 }
3285 }
3286 "boolean" => {
3287 if !matches!(value, Value::Bool(_)) {
3288 return Err(RuntimeError(format!("{path} expected boolean")));
3289 }
3290 }
3291 _ => {}
3292 }
3293 Ok(())
3294}
3295
3296fn is_high_risk(risk: &str) -> bool {
3297 matches!(
3298 risk,
3299 "high" | "destructive" | "sensitive" | "financial_or_legal"
3300 )
3301}
3302
3303fn is_media_kind(kind: &str) -> bool {
3304 matches!(
3305 kind,
3306 "image"
3307 | "audio"
3308 | "video"
3309 | "frame"
3310 | "audio_segment"
3311 | "video_segment"
3312 | "transcript"
3313 | "embedding"
3314 | "derived"
3315 )
3316}
3317
3318fn stream_chunk_to_state(chunk: StreamChunkRef) -> State {
3319 let mut state = State::new();
3320 state.insert(
3321 "schema_version".to_string(),
3322 Value::String(STREAM_SCHEMA_VERSION.to_string()),
3323 );
3324 state.insert("stream_id".to_string(), Value::String(chunk.stream_id));
3325 state.insert("chunk_id".to_string(), Value::String(chunk.chunk_id));
3326 state.insert("offset".to_string(), chunk.offset);
3327 if let Some(content_ref) = chunk.content_ref {
3328 state.insert("content_ref".to_string(), Value::String(content_ref));
3329 }
3330 if let Some(content_hash) = chunk.content_hash {
3331 state.insert("content_hash".to_string(), Value::String(content_hash));
3332 }
3333 if let Some(sequence) = chunk.sequence {
3334 state.insert("sequence".to_string(), Value::Number(sequence));
3335 }
3336 if let Some(event_time) = chunk.event_time {
3337 state.insert("event_time".to_string(), Value::Number(event_time));
3338 }
3339 if !chunk.metadata.is_empty() {
3340 state.insert("metadata".to_string(), Value::Object(chunk.metadata));
3341 }
3342 state
3343}
3344
3345fn media_artifacts_from(artifacts: &[Artifact]) -> Vec<State> {
3346 artifacts
3347 .iter()
3348 .filter_map(
3349 |artifact| match artifact.metadata.get("agentledger_media") {
3350 Some(Value::Object(metadata)) => {
3351 let mut row = State::new();
3352 row.insert(
3353 "artifact_id".to_string(),
3354 Value::String(artifact.artifact_id.clone()),
3355 );
3356 row.insert("name".to_string(), Value::String(artifact.name.clone()));
3357 row.insert(
3358 "blob_hash".to_string(),
3359 Value::String(artifact.blob_hash.clone()),
3360 );
3361 row.insert(
3362 "blob_ref".to_string(),
3363 Value::String(artifact.blob_ref.clone()),
3364 );
3365 for key in ["kind", "uri", "content_ref", "metadata", "lineage"] {
3366 if let Some(value) = metadata.get(key) {
3367 row.insert(key.to_string(), value.clone());
3368 }
3369 }
3370 Some(row)
3371 }
3372 _ => None,
3373 },
3374 )
3375 .collect()
3376}
3377
3378fn stream_checkpoints_from(artifacts: &[Artifact]) -> Vec<State> {
3379 artifacts
3380 .iter()
3381 .filter_map(
3382 |artifact| match artifact.metadata.get("agentledger_stream") {
3383 Some(Value::Object(metadata)) => {
3384 let mut row = State::new();
3385 row.insert(
3386 "artifact_id".to_string(),
3387 Value::String(artifact.artifact_id.clone()),
3388 );
3389 row.insert("name".to_string(), Value::String(artifact.name.clone()));
3390 row.insert(
3391 "blob_hash".to_string(),
3392 Value::String(artifact.blob_hash.clone()),
3393 );
3394 row.insert(
3395 "blob_ref".to_string(),
3396 Value::String(artifact.blob_ref.clone()),
3397 );
3398 for key in [
3399 "stream_id",
3400 "consumer_id",
3401 "offset",
3402 "watermark",
3403 "chunk",
3404 "partial_result_ref",
3405 "backpressure",
3406 ] {
3407 if let Some(value) = metadata.get(key) {
3408 row.insert(key.to_string(), value.clone());
3409 }
3410 }
3411 Some(row)
3412 }
3413 _ => None,
3414 },
3415 )
3416 .collect()
3417}
3418
3419fn classify_runtime_error(message: &str) -> &'static str {
3420 if message.contains("budget exceeded") || message.contains("budget") {
3421 "BudgetExceededError"
3422 } else if message.contains("sandbox executor") {
3423 "SandboxUnavailableError"
3424 } else if message.contains("high-risk") || message.contains("denied") {
3425 "PermissionDeniedError"
3426 } else {
3427 "RuntimeError"
3428 }
3429}
3430
3431fn failure_source(error_type: &str) -> &'static str {
3432 match error_type {
3433 "BudgetExceededError" => "budget",
3434 "SandboxUnavailableError" => "sandbox",
3435 "PermissionDeniedError" | "ApprovalDenied" => "policy",
3436 _ => "agent",
3437 }
3438}
3439
3440fn is_failure_event(kind: &str) -> bool {
3441 matches!(
3442 kind,
3443 "failure_classified"
3444 | "error_raised"
3445 | "step_failed"
3446 | "step_retry_scheduled"
3447 | "step_waiting_human"
3448 | "lease_expired"
3449 | "run_cancel_requested"
3450 | "run_cancelled"
3451 | "tool_call_failed"
3452 | "tool_approval_required"
3453 | "budget_check_failed"
3454 )
3455}
3456
3457fn format_state(state: &State) -> String {
3458 let mut keys: Vec<&String> = state.keys().collect();
3459 keys.sort();
3460 keys.into_iter()
3461 .map(|key| format!("{}={}", key, format_value(&state[key])))
3462 .collect::<Vec<_>>()
3463 .join(",")
3464}
3465
3466fn format_value(value: &Value) -> String {
3467 match value {
3468 Value::Null => "null".to_string(),
3469 Value::Bool(value) => value.to_string(),
3470 Value::Number(value) => value.to_string(),
3471 Value::String(value) => format!("\"{}\"", value),
3472 Value::Object(value) => format!("{{{}}}", format_state(value)),
3473 Value::Array(values) => format!(
3474 "[{}]",
3475 values
3476 .iter()
3477 .map(format_value)
3478 .collect::<Vec<_>>()
3479 .join(",")
3480 ),
3481 }
3482}
3483
3484fn stable_hash(input: &str) -> String {
3485 let mut hash: u64 = 0xcbf29ce484222325;
3486 for byte in input.as_bytes() {
3487 hash ^= *byte as u64;
3488 hash = hash.wrapping_mul(0x100000001b3);
3489 }
3490 format!("{hash:016x}")
3491}
3492
3493fn new_id(prefix: &str) -> String {
3494 let value = ID_COUNTER.fetch_add(1, Ordering::Relaxed);
3495 format!("{prefix}_{value:016x}")
3496}
3497
3498fn now_seconds() -> f64 {
3499 SystemTime::now()
3500 .duration_since(UNIX_EPOCH)
3501 .unwrap()
3502 .as_secs_f64()
3503}
3504
3505#[cfg(test)]
3506mod tests {
3507 use super::*;
3508
3509 fn state(items: &[(&str, Value)]) -> State {
3510 items
3511 .iter()
3512 .map(|(key, value)| ((*key).to_string(), value.clone()))
3513 .collect()
3514 }
3515
3516 fn event_exists(events: &[Event], event_type: &str) -> bool {
3517 events.iter().any(|event| event.event_type == event_type)
3518 }
3519
3520 fn claim_context(
3521 runtime: &mut Runtime,
3522 run_id: &str,
3523 worker: &str,
3524 role: &str,
3525 ) -> AgentContext {
3526 let claim = runtime.store.claim_step(worker, run_id, 60.0).unwrap();
3527 let (_state, version, session_id) = runtime.store.load_state(run_id).unwrap();
3528 let mut payload = State::new();
3529 payload.insert("agent_role".to_string(), Value::String(role.to_string()));
3530 runtime.store.append_event(
3531 run_id,
3532 Some(&session_id),
3533 Some(&claim.step_id),
3534 "agent_started",
3535 payload,
3536 Some(role),
3537 Some(version),
3538 None,
3539 );
3540 AgentContext {
3541 run_id: run_id.to_string(),
3542 session_id,
3543 step_id: claim.step_id,
3544 agent_role: role.to_string(),
3545 lease_token: claim.lease_token,
3546 attempt: claim.attempt,
3547 state_version: version,
3548 pending_patch: State::new(),
3549 }
3550 }
3551
3552 #[test]
3553 fn runtime_creates_evidence_and_replay() {
3554 let mut runtime = Runtime::new();
3555 runtime.register_tool(ToolSpec::new(
3556 "docs.echo",
3557 Box::new(|args| Ok(Value::Object(state(&[("echo", args["text"].clone())])))),
3558 ));
3559 let (run_id, _) = runtime.create_run(state(&[("input", "hello".into())]));
3560 let ok = runtime
3561 .run_once(&run_id, "worker-a", "Researcher", 60.0, |ctx, state| {
3562 let mut result = State::new();
3563 result.insert("from_state".to_string(), state["input"].clone());
3564 ctx.write_state("tool_result", Value::Object(result));
3565 Ok(())
3566 })
3567 .unwrap();
3568 assert!(ok);
3569 let bundle = export_evidence(&runtime.store, &run_id).unwrap();
3570 assert_eq!(bundle.schema_version, "agentledger.evidence.v1");
3571 let summary = replay(&runtime.store, &run_id).unwrap();
3572 assert!(summary.replay_safe);
3573 assert_eq!(summary.event_count, bundle.events.len());
3574 }
3575
3576 #[test]
3577 fn local_snapshot_store_round_trips_completed_run() {
3578 let mut runtime = Runtime::new();
3579 let (run_id, _) = runtime.create_run(state(&[("input", "hello".into())]));
3580 runtime
3581 .run_once(&run_id, "worker-a", "Researcher", 60.0, |ctx, state| {
3582 let mut result = State::new();
3583 result.insert("echo".to_string(), state["input"].clone());
3584 ctx.write_state("tool_result", Value::Object(result));
3585 Ok(())
3586 })
3587 .unwrap();
3588 let path =
3589 std::env::temp_dir().join(format!("agentledger-rust-{}.store", new_id("snapshot")));
3590 runtime.store.save_to_path(&path).unwrap();
3591
3592 let reopened = MemoryStore::load_from_path(&path).unwrap();
3593 let final_state = reopened.final_state(&run_id).unwrap();
3594 assert_eq!(
3595 final_state.get("tool_result"),
3596 Some(&Value::Object(state(&[("echo", "hello".into())])))
3597 );
3598 let bundle = export_evidence(&reopened, &run_id).unwrap();
3599 assert_eq!(reopened.steps(&run_id).len(), 1);
3600 assert_eq!(
3601 replay(&reopened, &run_id).unwrap().event_count,
3602 bundle.events.len()
3603 );
3604 let _ = std::fs::remove_file(path);
3605 }
3606
3607 #[test]
3608 fn local_blob_store_round_trips_json_values() {
3609 let root = std::env::temp_dir().join(format!("agentledger-rust-blobs-{}", new_id("blob")));
3610 let blobs = LocalBlobStore::open(&root).unwrap();
3611 let value = Value::Object(state(&[(
3612 "hello",
3613 Value::Object(state(&[("nested", "world".into())])),
3614 )]));
3615 let first = blobs.put_json(&value).unwrap();
3616 let second = blobs.put_json(&value).unwrap();
3617 assert!(first.0.starts_with("sha256:"));
3618 assert!(first.1.starts_with("blob://sha256/"));
3619 assert_eq!(first, second);
3620 assert_eq!(blobs.get_json(&first.1).unwrap(), value);
3621 assert!(blobs.get_json("unsupported://blob").is_err());
3622 let _ = std::fs::remove_dir_all(root);
3623 }
3624
3625 #[test]
3626 fn tool_schema_validation_rejects_invalid_input() {
3627 let input_schema = Value::Object(state(&[
3628 ("type", "object".into()),
3629 (
3630 "required",
3631 Value::Array(vec![Value::String("text".to_string())]),
3632 ),
3633 ("additionalProperties", false.into()),
3634 (
3635 "properties",
3636 Value::Object(state(&[(
3637 "text",
3638 Value::Object(state(&[("type", "string".into())])),
3639 )])),
3640 ),
3641 ]));
3642 let mut runtime = Runtime::new();
3643 runtime.register_tool(
3644 ToolSpec::new(
3645 "docs.echo",
3646 Box::new(|args| Ok(Value::Object(state(&[("echo", args["text"].clone())])))),
3647 )
3648 .input_schema(input_schema),
3649 );
3650 let (run_id, _) = runtime.create_run(State::new());
3651 let ctx = claim_context(&mut runtime, &run_id, "worker", "SchemaAgent");
3652 assert!(runtime.call_tool(&ctx, "docs.echo", State::new()).is_err());
3653 assert!(runtime.store.events(&run_id).iter().any(|event| {
3654 event.event_type == "tool_call_failed"
3655 && event.payload.get("phase")
3656 == Some(&Value::String("input_validation".to_string()))
3657 }));
3658 }
3659
3660 #[test]
3661 fn tool_ledger_reuses_side_effect_after_retry() {
3662 let mut runtime = Runtime::new();
3663 runtime.register_tool(
3664 ToolSpec::new(
3665 "github.create_pr",
3666 Box::new(|args| {
3667 Ok(Value::Object(state(&[
3668 ("external_id", "pr-123".into()),
3669 ("title", args["title"].clone()),
3670 ])))
3671 }),
3672 )
3673 .side_effect("external")
3674 .idempotency_required(true),
3675 );
3676 let (run_id, _) = runtime.create_run(state(&[("title", "runtime parity".into())]));
3677 let ctx = claim_context(&mut runtime, &run_id, "worker-a", "Coder");
3678 let first = runtime
3679 .call_tool(
3680 &ctx,
3681 "github.create_pr",
3682 state(&[("title", "runtime parity".into())]),
3683 )
3684 .unwrap();
3685 runtime
3686 .store
3687 .mark_retry(&run_id, &ctx.step_id, "RetryableAgentError", "retryable");
3688 let ctx2 = claim_context(&mut runtime, &run_id, "worker-b", "Coder");
3689 let second = runtime
3690 .call_tool(
3691 &ctx2,
3692 "github.create_pr",
3693 state(&[("title", "runtime parity".into())]),
3694 )
3695 .unwrap();
3696 assert_eq!(first, second);
3697 assert_eq!(runtime.store.ledger(&run_id).len(), 1);
3698 }
3699
3700 #[test]
3701 fn policy_denies_unapproved_high_risk_tool() {
3702 let mut runtime = Runtime::new();
3703 runtime.register_tool(
3704 ToolSpec::new("repo.write", Box::new(|_| Ok(Value::Bool(true)))).risk_level("high"),
3705 );
3706 let (run_id, _) = runtime.create_run(State::new());
3707 let ctx = claim_context(&mut runtime, &run_id, "worker", "Reviewer");
3708 let err = runtime
3709 .call_tool(&ctx, "repo.write", state(&[("path", "README.md".into())]))
3710 .unwrap_err();
3711 assert!(err.0.contains("high-risk"));
3712 assert!(runtime
3713 .store
3714 .events(&run_id)
3715 .iter()
3716 .any(|event| event.event_type == "tool_permission_decided"
3717 && event.payload.get("allowed") == Some(&Value::Bool(false))));
3718 }
3719
3720 #[test]
3721 fn approval_pauses_and_resumes_step() {
3722 let mut runtime = Runtime::new();
3723 runtime.register_tool(
3724 ToolSpec::new(
3725 "github.create_pr",
3726 Box::new(|_| Ok(Value::Object(state(&[("external_id", "pr-42".into())])))),
3727 )
3728 .risk_level("high")
3729 .approval_required(true)
3730 .side_effect("external")
3731 .idempotency_required(true),
3732 );
3733 let (run_id, _) = runtime.create_run(State::new());
3734 let ctx = claim_context(&mut runtime, &run_id, "worker-a", "Coder");
3735 let err = runtime
3736 .call_tool(&ctx, "github.create_pr", state(&[("title", "safe".into())]))
3737 .unwrap_err();
3738 assert!(err.0.starts_with("approval required:"));
3739 let approval_id = err.0.trim_start_matches("approval required:").to_string();
3740 runtime
3741 .store
3742 .mark_waiting_human(&run_id, &ctx.step_id, &err.0, &approval_id);
3743 assert_eq!(runtime.store.steps(&run_id)[0].status, "waiting_human");
3744 runtime
3745 .store
3746 .approve_request(&approval_id, "alice", "reviewed")
3747 .unwrap();
3748 let ctx2 = claim_context(&mut runtime, &run_id, "worker-b", "Coder");
3749 let result = runtime
3750 .call_tool(
3751 &ctx2,
3752 "github.create_pr",
3753 state(&[("title", "safe".into())]),
3754 )
3755 .unwrap();
3756 assert!(matches!(result, Value::Object(_)));
3757
3758 let (denied_run, _) = runtime.create_run(State::new());
3759 let denied_ctx = claim_context(&mut runtime, &denied_run, "worker-c", "Coder");
3760 let denied_err = runtime
3761 .call_tool(
3762 &denied_ctx,
3763 "github.create_pr",
3764 state(&[("title", "blocked".into())]),
3765 )
3766 .unwrap_err();
3767 let denied_id = denied_err
3768 .0
3769 .trim_start_matches("approval required:")
3770 .to_string();
3771 runtime.store.mark_waiting_human(
3772 &denied_run,
3773 &denied_ctx.step_id,
3774 &denied_err.0,
3775 &denied_id,
3776 );
3777 runtime
3778 .store
3779 .deny_request(&denied_id, "bob", "not allowed")
3780 .unwrap();
3781 assert_eq!(runtime.store.steps(&denied_run)[0].status, "failed");
3782 }
3783
3784 #[test]
3785 fn sandbox_required_tool_fails_closed() {
3786 let mut runtime = Runtime::new();
3787 runtime.register_tool(
3788 ToolSpec::new("shell.exec", Box::new(|_| Ok(Value::Bool(true)))).sandbox_required(true),
3789 );
3790 let (run_id, _) = runtime.create_run(State::new());
3791 let ctx = claim_context(&mut runtime, &run_id, "worker", "Executor");
3792 let err = runtime
3793 .call_tool(
3794 &ctx,
3795 "shell.exec",
3796 state(&[("argv", Value::Object(State::new()))]),
3797 )
3798 .unwrap_err();
3799 assert!(err.0.contains("sandbox executor"));
3800 let events = runtime.store.events(&run_id);
3801 assert!(event_exists(&events, "sandbox_started"));
3802 assert!(event_exists(&events, "tool_call_failed"));
3803 }
3804
3805 #[test]
3806 fn docker_sandbox_executor_requires_explicit_execution() {
3807 let executor = DockerSandboxExecutor::new("fake-image", false).with_binary("/bin/echo");
3808 let result = executor.run_tool(
3809 state(&[("_sandbox_command", Value::Array(vec!["echo".into(), "hi".into()]))]),
3810 &SandboxPolicy {
3811 tool_name: "cmd.echo".to_string(),
3812 run_id: "run".to_string(),
3813 step_id: "step".to_string(),
3814 executor: "docker".to_string(),
3815 network: "deny".to_string(),
3816 filesystem: "read-only".to_string(),
3817 timeout_seconds: 1,
3818 extra: State::new(),
3819 },
3820 );
3821 assert!(!result.ok);
3822 assert_eq!(result.metadata.get("error_type"), Some(&Value::String("SandboxAdapterNotInstalled".to_string())));
3823 }
3824
3825 #[test]
3826 fn docker_sandbox_executor_runs_command_style_tool_with_injected_binary() {
3827 let mut runtime = Runtime::new();
3828 runtime.set_sandbox(Box::new(DockerSandboxExecutor::new("fake-image", true).with_binary("/bin/echo")));
3829 runtime.register_tool(
3830 ToolSpec::new("cmd.echo", Box::new(|_| Err(RuntimeError("direct func should not execute".to_string()))))
3831 .sandbox_required(true)
3832 .sandbox_executor("docker"),
3833 );
3834 let (run_id, _) = runtime.create_run(State::new());
3835 let ctx = claim_context(&mut runtime, &run_id, "worker", "Executor");
3836 let value = runtime
3837 .call_tool(
3838 &ctx,
3839 "cmd.echo",
3840 state(&[("_sandbox_command", Value::Array(vec!["echo".into(), "hi".into()]))]),
3841 )
3842 .unwrap();
3843 let output = match value { Value::Object(output) => output, _ => panic!("expected object output") };
3844 let stdout = match output.get("stdout") { Some(Value::String(value)) => value, _ => panic!("expected stdout") };
3845 assert!(stdout.contains("run"));
3846 assert!(stdout.contains("fake-image"));
3847 let events = runtime.store.events(&run_id);
3848 assert!(event_exists(&events, "sandbox_completed"));
3849 assert!(event_exists(&events, "tool_call_completed"));
3850 }
3851
3852 #[test]
3853 fn cost_budget_and_failure_attribution() {
3854 let mut runtime = Runtime::new();
3855 runtime.set_budget(BudgetLimits {
3856 max_tool_calls: Some(1.0),
3857 max_model_tokens: None,
3858 max_total_usd: None,
3859 });
3860 runtime.register_tool(ToolSpec::new(
3861 "docs.echo",
3862 Box::new(|args| Ok(Value::Object(state(&[("echo", args["text"].clone())])))),
3863 ));
3864 let (run_id, _) = runtime.create_run(State::new());
3865 let ctx = claim_context(&mut runtime, &run_id, "worker", "Researcher");
3866 runtime
3867 .record_model_call(&ctx, "gpt-test", 10.0, 5.0, 0.01)
3868 .unwrap();
3869 runtime
3870 .call_tool(&ctx, "docs.echo", state(&[("text", "first".into())]))
3871 .unwrap();
3872 let err = runtime
3873 .call_tool(&ctx, "docs.echo", state(&[("text", "second".into())]))
3874 .unwrap_err();
3875 runtime.store.mark_failed(
3876 &run_id,
3877 &ctx.step_id,
3878 classify_runtime_error(&err.0),
3879 &err.0,
3880 );
3881 let summary = runtime.store.cost_summary(&run_id);
3882 assert_eq!(summary.tool_calls, 1.0);
3883 assert_eq!(summary.model_tokens, 15.0);
3884 assert_eq!(summary.total_usd, 0.01);
3885 let cost = cost_attribution(&runtime.store, &run_id);
3886 assert_eq!(cost.by_agent["Researcher"].tool_calls, 1.0);
3887 assert_eq!(cost.by_agent["Researcher"].model_tokens, 15.0);
3888 let failure = failure_attribution(&runtime.store, &run_id).unwrap();
3889 assert_eq!(failure.failed_steps.len(), 1);
3890 assert!(event_exists(&failure.failure_events, "budget_check_failed"));
3891 assert!(event_exists(&failure.failure_events, "failure_classified"));
3892 }
3893
3894 #[test]
3895 fn media_and_stream_artifacts_are_indexed_in_evidence_and_replay() {
3896 let mut runtime = Runtime::new();
3897 let (run_id, _) = runtime.create_run(State::new());
3898 let ctx = claim_context(&mut runtime, &run_id, "worker-media", "MediaAgent");
3899 let mut media_metadata = State::new();
3900 media_metadata.insert(
3901 "mime_type".to_string(),
3902 Value::String("image/jpeg".to_string()),
3903 );
3904 media_metadata.insert("frame_index".to_string(), Value::Number(1.0));
3905 let mut lineage = State::new();
3906 lineage.insert(
3907 "source_blob_ref".to_string(),
3908 Value::String("s3://media/demo/input.mp4".to_string()),
3909 );
3910 lineage.insert(
3911 "tool_call_id".to_string(),
3912 Value::String("video.extract_frames".to_string()),
3913 );
3914 let frame_id = runtime
3915 .create_media_artifact(
3916 &ctx,
3917 "frame-0001",
3918 "frame",
3919 MediaArtifactOptions {
3920 uri: Some("s3://media/demo/frame-0001.jpg".to_string()),
3921 media_metadata,
3922 lineage,
3923 ..Default::default()
3924 },
3925 )
3926 .unwrap();
3927 let checkpoint_id = runtime
3928 .create_stream_checkpoint(
3929 &ctx,
3930 "camera-checkpoint",
3931 StreamCheckpointOptions {
3932 stream_id: "camera-1".to_string(),
3933 consumer_id: "vision-agent".to_string(),
3934 offset: Value::Number(7.0),
3935 watermark: Some(Value::Number(1.5)),
3936 chunk: Some(StreamChunkRef {
3937 stream_id: "camera-1".to_string(),
3938 chunk_id: "chunk-7".to_string(),
3939 offset: Value::Number(7.0),
3940 content_ref: Some("blob://sha256/chunk-7.json".to_string()),
3941 sequence: Some(7.0),
3942 ..Default::default()
3943 }),
3944 ..Default::default()
3945 },
3946 )
3947 .unwrap();
3948 let mut artifacts = State::new();
3949 artifacts.insert("frame".to_string(), Value::String(frame_id));
3950 artifacts.insert("checkpoint".to_string(), Value::String(checkpoint_id));
3951 runtime
3952 .store
3953 .commit_state_patch(
3954 &run_id,
3955 &ctx.step_id,
3956 &ctx.lease_token,
3957 ctx.state_version,
3958 state(&[("artifacts", Value::Object(artifacts))]),
3959 )
3960 .unwrap();
3961 let bundle = export_evidence(&runtime.store, &run_id).unwrap();
3962 assert_eq!(bundle.artifacts.len(), 2);
3963 assert_eq!(bundle.media_artifacts.len(), 1);
3964 assert_eq!(bundle.stream_checkpoints.len(), 1);
3965 assert_eq!(
3966 bundle.media_artifacts[0].get("kind"),
3967 Some(&Value::String("frame".to_string()))
3968 );
3969 assert_eq!(
3970 bundle.stream_checkpoints[0].get("stream_id"),
3971 Some(&Value::String("camera-1".to_string()))
3972 );
3973 let summary = replay(&runtime.store, &run_id).unwrap();
3974 assert_eq!(summary.artifact_count, 2);
3975 assert_eq!(summary.media_artifact_count, 1);
3976 assert_eq!(summary.stream_checkpoint_count, 1);
3977 }
3978
3979 #[test]
3980 fn lease_recovery_fences_previous_owner() {
3981 let mut store = MemoryStore::new();
3982 let (run_id, step_id) = store.create_run(State::new());
3983 let claim = store.claim_step("stale-worker", &run_id, 0.0).unwrap();
3984 assert_eq!(store.recover_expired_leases(), 1);
3985 assert!(store
3986 .commit_state_patch(
3987 &run_id,
3988 &step_id,
3989 &claim.lease_token,
3990 0,
3991 state(&[("late", true.into())])
3992 )
3993 .is_err());
3994 assert!(store.claim_step("new-worker", &run_id, 60.0).is_ok());
3995 }
3996
3997 #[test]
3998 fn cancellation_fences_worker() {
3999 let mut store = MemoryStore::new();
4000 let (run_id, step_id) = store.create_run(State::new());
4001 let claim = store.claim_step("worker", &run_id, 60.0).unwrap();
4002 assert_eq!(store.cancel_run(&run_id, "operator requested").unwrap(), 1);
4003 assert!(store
4004 .commit_state_patch(
4005 &run_id,
4006 &step_id,
4007 &claim.lease_token,
4008 0,
4009 state(&[("late", true.into())])
4010 )
4011 .is_err());
4012 }
4013
4014 #[test]
4015 fn shared_runtime_baseline_fixture() {
4016 let fixture =
4017 std::fs::read_to_string("../contracts/conformance/runtime_baseline.v1.json").unwrap();
4018 assert!(fixture.contains("agentledger.conformance.runtime_baseline.v1"));
4019 for scenario in [
4020 "durable_run_evidence_replay",
4021 "tool_ledger_idempotent_retry",
4022 "lease_recovery_fences_stale_worker",
4023 "cancellation_fences_worker",
4024 ] {
4025 assert!(
4026 fixture.contains(scenario),
4027 "missing shared fixture scenario {scenario}"
4028 );
4029 }
4030 }
4031
4032 #[test]
4033 fn shared_parity_fixtures() {
4034 let fixtures: Vec<(&str, &[&str])> = vec![
4035 (
4036 "../contracts/conformance/policy_approval_sandbox.v1.json",
4037 &[
4038 "agentledger.conformance.policy_approval_sandbox.v1",
4039 "policy_denies_unapproved_high_risk_tool",
4040 "approval_pauses_and_resumes_step",
4041 "sandbox_required_tool_fails_closed",
4042 ],
4043 ),
4044 (
4045 "../contracts/conformance/cost_failure_attribution.v1.json",
4046 &[
4047 "agentledger.conformance.cost_failure_attribution.v1",
4048 "tool_and_model_cost_attributed_to_run_step_role",
4049 "budget_exhaustion_blocks_execution",
4050 "failure_attribution_classifies_agent_tool_model_runtime",
4051 ],
4052 ),
4053 (
4054 "../contracts/conformance/local_persistence.v1.json",
4055 &[
4056 "agentledger.conformance.local_persistence.v1",
4057 "local_store_round_trips_completed_run",
4058 "local_store_preserves_evidence_replay_chain",
4059 "local_store_uses_atomic_snapshot_write",
4060 ],
4061 ),
4062 (
4063 "../contracts/conformance/local_blob_store.v1.json",
4064 &[
4065 "agentledger.conformance.local_blob_store.v1",
4066 "blob_roundtrip_json_value",
4067 "blob_content_address_is_stable",
4068 "blob_bad_ref_is_rejected",
4069 ],
4070 ),
4071 (
4072 "../contracts/conformance/tool_schema_validation.v1.json",
4073 &[
4074 "agentledger.conformance.tool_schema_validation.v1",
4075 "invalid_tool_input_rejected_before_execution",
4076 "valid_tool_input_and_output_pass",
4077 "invalid_tool_output_rejected",
4078 ],
4079 ),
4080 (
4081 "../contracts/conformance/worker_service.v1.json",
4082 &[
4083 "agentledger.conformance.worker_service.v1",
4084 "local_worker_runs_until_terminal",
4085 "worker_service_stops_after_idle_poll",
4086 "worker_loop_recovers_expired_leases",
4087 ],
4088 ),
4089 (
4090 "../contracts/conformance/media_stream_artifacts.v1.json",
4091 &[
4092 "agentledger.conformance.media_stream_artifacts.v1",
4093 "media_artifact_ref_is_indexed_in_evidence",
4094 "stream_checkpoint_ref_is_indexed_in_evidence",
4095 ],
4096 ),
4097 (
4098 "../contracts/conformance/evidence_consumers.v1.json",
4099 &[
4100 "agentledger.conformance.evidence_consumers.v1",
4101 "trace_spans_from_evidence",
4102 "evidence_diff_detects_state_and_event_changes",
4103 "divergence_report_lists_changed_dimensions",
4104 "static_debug_summary_is_exportable",
4105 ],
4106 ),
4107 (
4108 "../contracts/conformance/static_debug_html.v1.json",
4109 &[
4110 "agentledger.conformance.static_debug_html.v1",
4111 "static_debug_html_contains_run_events_and_state",
4112 ],
4113 ),
4114 (
4115 "../contracts/conformance/ops_readiness.v1.json",
4116 &[
4117 "agentledger.conformance.ops_readiness.v1",
4118 "retention_plan_is_non_destructive_and_counts_evidence",
4119 "backup_readiness_reports_required_checks",
4120 ],
4121 ),
4122 (
4123 "../contracts/conformance/storage_schema.v1.json",
4124 &[
4125 "agentledger.conformance.storage_schema.v1",
4126 "latest_schema_version_and_ddl_are_available",
4127 ],
4128 ),
4129 (
4130 "../contracts/conformance/mcp_adapters.v1.json",
4131 &[
4132 "agentledger.conformance.mcp_adapters.v1",
4133 "in_memory_mcp_tool_server_lists_and_calls_tools",
4134 "mcp_tool_descriptor_maps_to_tool_spec",
4135 "in_memory_mcp_context_server_reads_resources",
4136 ],
4137 ),
4138 (
4139 "../contracts/conformance/framework_adapters.v1.json",
4140 &[
4141 "agentledger.conformance.framework_adapters.v1",
4142 "function_adapter_maps_run_spec_and_invokes_agent",
4143 "method_framework_adapter_uses_first_available_method_and_writes_output",
4144 ],
4145 ),
4146 (
4147 "../contracts/conformance/otlp_trace_export.v1.json",
4148 &[
4149 "agentledger.conformance.otlp_trace_export.v1",
4150 "otlp_json_contains_resource_scope_and_spans",
4151 ],
4152 ),
4153 (
4154 "../contracts/conformance/simple_api.v1.json",
4155 &[
4156 "agentledger.conformance.simple_api.v1",
4157 "simple_run_returns_output_and_state",
4158 ],
4159 ),
4160 ];
4161 for (path, required) in fixtures {
4162 let body = std::fs::read_to_string(path).unwrap();
4163 for token in required {
4164 assert!(body.contains(token), "fixture {path} missing {token}");
4165 }
4166 }
4167 }
4168}
4169
4170#[derive(Clone, Debug)]
4171pub struct TraceSpan {
4172 pub trace_id: String,
4173 pub span_id: String,
4174 pub parent_span_id: Option<String>,
4175 pub name: String,
4176 pub start_time: f64,
4177 pub end_time: f64,
4178 pub attributes: State,
4179}
4180
4181#[derive(Clone, Debug)]
4182pub struct SequenceDiff {
4183 pub left_count: usize,
4184 pub right_count: usize,
4185 pub changed_count: usize,
4186}
4187
4188#[derive(Clone, Debug)]
4189pub struct DictDiff {
4190 pub changed_count: usize,
4191}
4192
4193#[derive(Clone, Debug)]
4194pub struct EvidenceDiffReport {
4195 pub left_run_id: String,
4196 pub right_run_id: String,
4197 pub same: bool,
4198 pub final_state_changed_count: usize,
4199 pub event_types_changed_count: usize,
4200 pub media_artifacts_changed_count: usize,
4201 pub stream_checkpoints_changed_count: usize,
4202}
4203
4204#[derive(Clone, Debug)]
4205pub struct DivergenceReport {
4206 pub left_run_id: String,
4207 pub right_run_id: String,
4208 pub same: bool,
4209 pub changed_dimensions: Vec<String>,
4210}
4211
4212pub fn trace_spans(bundle: &EvidenceBundle) -> Vec<TraceSpan> {
4213 let mut spans = Vec::new();
4214 for (index, event) in bundle.events.iter().enumerate() {
4215 let seq = if event.seq == 0 {
4216 index as u64 + 1
4217 } else {
4218 event.seq
4219 };
4220 spans.push(TraceSpan {
4221 trace_id: bundle.run.run_id.clone(),
4222 span_id: span_id("evt", seq),
4223 parent_span_id: None,
4224 name: event.event_type.clone(),
4225 start_time: event.timestamp,
4226 end_time: event.timestamp,
4227 attributes: state(&[
4228 (
4229 "agentledger.run_id",
4230 Value::String(bundle.run.run_id.clone()),
4231 ),
4232 ("agentledger.seq", Value::Number(seq as f64)),
4233 (
4234 "agentledger.payload_hash",
4235 Value::String(event.payload_hash.clone()),
4236 ),
4237 (
4238 "agentledger.payload_ref",
4239 Value::String(event.payload_ref.clone()),
4240 ),
4241 ]),
4242 });
4243 }
4244 for (index, artifact) in bundle.media_artifacts.iter().enumerate() {
4245 spans.push(TraceSpan {
4246 trace_id: bundle.run.run_id.clone(),
4247 span_id: span_id("media", index as u64 + 1),
4248 parent_span_id: None,
4249 name: "media_artifact".to_string(),
4250 start_time: bundle.run.updated_at,
4251 end_time: bundle.run.updated_at,
4252 attributes: state(&[
4253 (
4254 "agentledger.run_id",
4255 Value::String(bundle.run.run_id.clone()),
4256 ),
4257 (
4258 "agentledger.artifact_id",
4259 artifact.get("artifact_id").cloned().unwrap_or_default(),
4260 ),
4261 (
4262 "agentledger.media_kind",
4263 artifact.get("kind").cloned().unwrap_or_default(),
4264 ),
4265 ]),
4266 });
4267 }
4268 for (index, checkpoint) in bundle.stream_checkpoints.iter().enumerate() {
4269 spans.push(TraceSpan {
4270 trace_id: bundle.run.run_id.clone(),
4271 span_id: span_id("stream", index as u64 + 1),
4272 parent_span_id: None,
4273 name: "stream_checkpoint".to_string(),
4274 start_time: bundle.run.updated_at,
4275 end_time: bundle.run.updated_at,
4276 attributes: state(&[
4277 (
4278 "agentledger.run_id",
4279 Value::String(bundle.run.run_id.clone()),
4280 ),
4281 (
4282 "agentledger.stream_id",
4283 checkpoint.get("stream_id").cloned().unwrap_or_default(),
4284 ),
4285 (
4286 "agentledger.consumer_id",
4287 checkpoint.get("consumer_id").cloned().unwrap_or_default(),
4288 ),
4289 ]),
4290 });
4291 }
4292 spans
4293}
4294
4295pub fn trace_jsonl(bundle: &EvidenceBundle) -> String {
4296 trace_spans(bundle)
4297 .iter()
4298 .map(|span| {
4299 format!(
4300 "{{\"trace_id\":\"{}\",\"span_id\":\"{}\",\"name\":\"{}\"}}\n",
4301 span.trace_id, span.span_id, span.name
4302 )
4303 })
4304 .collect()
4305}
4306
4307pub fn diff_evidence(left: &EvidenceBundle, right: &EvidenceBundle) -> EvidenceDiffReport {
4308 let final_state = diff_state(&left.final_state, &right.final_state).changed_count;
4309 let events = diff_values(&event_types(&left.events), &event_types(&right.events)).changed_count;
4310 let media = diff_values(
4311 &state_fingerprints(&left.media_artifacts),
4312 &state_fingerprints(&right.media_artifacts),
4313 )
4314 .changed_count;
4315 let streams = diff_values(
4316 &state_fingerprints(&left.stream_checkpoints),
4317 &state_fingerprints(&right.stream_checkpoints),
4318 )
4319 .changed_count;
4320 EvidenceDiffReport {
4321 left_run_id: left.run.run_id.clone(),
4322 right_run_id: right.run.run_id.clone(),
4323 same: final_state == 0
4324 && events == 0
4325 && media == 0
4326 && streams == 0
4327 && left.bundle_hash == right.bundle_hash,
4328 final_state_changed_count: final_state,
4329 event_types_changed_count: events,
4330 media_artifacts_changed_count: media,
4331 stream_checkpoints_changed_count: streams,
4332 }
4333}
4334
4335pub fn divergence_report(left: &EvidenceBundle, right: &EvidenceBundle) -> DivergenceReport {
4336 let mut changed = Vec::new();
4337 if diff_values(&event_types(&left.events), &event_types(&right.events)).changed_count > 0 {
4338 changed.push("events".to_string());
4339 }
4340 if diff_state(&left.final_state, &right.final_state).changed_count > 0 {
4341 changed.push("state".to_string());
4342 }
4343 if diff_values(
4344 &state_fingerprints(&left.media_artifacts),
4345 &state_fingerprints(&right.media_artifacts),
4346 )
4347 .changed_count
4348 > 0
4349 {
4350 changed.push("media_artifacts".to_string());
4351 }
4352 if diff_values(
4353 &state_fingerprints(&left.stream_checkpoints),
4354 &state_fingerprints(&right.stream_checkpoints),
4355 )
4356 .changed_count
4357 > 0
4358 {
4359 changed.push("stream_checkpoints".to_string());
4360 }
4361 if diff_values(
4362 &ledger_fingerprints_rust(&left.tool_ledger),
4363 &ledger_fingerprints_rust(&right.tool_ledger),
4364 )
4365 .changed_count
4366 > 0
4367 {
4368 changed.push("ledger".to_string());
4369 }
4370 DivergenceReport {
4371 left_run_id: left.run.run_id.clone(),
4372 right_run_id: right.run.run_id.clone(),
4373 same: changed.is_empty(),
4374 changed_dimensions: changed,
4375 }
4376}
4377
4378pub fn debug_summary(bundle: &EvidenceBundle) -> State {
4379 let changes = bundle
4380 .events
4381 .iter()
4382 .filter(|event| {
4383 matches!(
4384 event.event_type.as_str(),
4385 "run_created" | "state_committed" | "system_state_patch_applied"
4386 )
4387 })
4388 .count();
4389 state(&[
4390 ("run_id", Value::String(bundle.run.run_id.clone())),
4391 ("event_count", Value::Number(bundle.events.len() as f64)),
4392 ("state_change_count", Value::Number(changes as f64)),
4393 ("final_state", Value::Object(bundle.final_state.clone())),
4394 ])
4395}
4396
4397fn span_id(prefix: &str, seq: u64) -> String {
4398 format!("{}-{:06}", prefix, seq)
4399}
4400fn event_types(events: &[Event]) -> Vec<Value> {
4401 events
4402 .iter()
4403 .map(|event| Value::String(event.event_type.clone()))
4404 .collect()
4405}
4406fn state_fingerprints(rows: &[State]) -> Vec<Value> {
4407 rows.iter()
4408 .map(|row| Value::String(encode_state(row)))
4409 .collect()
4410}
4411fn ledger_fingerprints_rust(rows: &[ToolLedgerEntry]) -> Vec<Value> {
4412 rows.iter()
4413 .map(|row| {
4414 Value::String(format!(
4415 "{}:{}:{}",
4416 row.tool_name, row.status, row.request_hash
4417 ))
4418 })
4419 .collect()
4420}
4421
4422fn diff_state(left: &State, right: &State) -> DictDiff {
4423 let mut keys: Vec<String> = left.keys().chain(right.keys()).cloned().collect();
4424 keys.sort();
4425 keys.dedup();
4426 let changed_count = keys
4427 .into_iter()
4428 .filter(|key| left.get(key) != right.get(key))
4429 .count();
4430 DictDiff { changed_count }
4431}
4432
4433fn diff_values(left: &[Value], right: &[Value]) -> SequenceDiff {
4434 let max = left.len().max(right.len());
4435 let mut changed_count = 0;
4436 for index in 0..max {
4437 if left.get(index) != right.get(index) {
4438 changed_count += 1;
4439 }
4440 }
4441 SequenceDiff {
4442 left_count: left.len(),
4443 right_count: right.len(),
4444 changed_count,
4445 }
4446}
4447
4448fn state(items: &[(&str, Value)]) -> State {
4449 let mut out = State::new();
4450 for (key, value) in items {
4451 out.insert((*key).to_string(), value.clone());
4452 }
4453 out
4454}
4455
4456#[derive(Clone, Debug)]
4457pub struct RunResult {
4458 pub run_id: String,
4459 pub session_id: String,
4460 pub ok: bool,
4461 pub output: Option<Value>,
4462 pub state: State,
4463}
4464
4465pub type SimpleAgentFunc = fn(&mut AgentContext, State) -> Result<Option<Value>>;
4466
4467pub fn simple_run(agent: SimpleAgentFunc, initial_state: State) -> Result<RunResult> {
4468 let mut runtime = Runtime::new();
4469 simple_run_with_runtime(&mut runtime, agent, initial_state)
4470}
4471
4472pub fn simple_run_with_runtime(
4473 runtime: &mut Runtime,
4474 agent: SimpleAgentFunc,
4475 initial_state: State,
4476) -> Result<RunResult> {
4477 let (run_id, _) = runtime.create_run(initial_state);
4478 let claim = runtime.store.claim_step("worker-simple", &run_id, 60.0)?;
4479 let (state_value, version, session_id) = runtime.store.load_state(&claim.run_id)?;
4480 runtime.store.append_event(
4481 &claim.run_id,
4482 Some(&session_id),
4483 Some(&claim.step_id),
4484 "agent_started",
4485 state(&[
4486 ("agent_role", "Agent".into()),
4487 ("attempt", Value::Number(claim.attempt as f64)),
4488 ]),
4489 Some("Agent"),
4490 Some(version),
4491 None,
4492 );
4493 let mut ctx = AgentContext {
4494 run_id: claim.run_id.clone(),
4495 session_id: session_id.clone(),
4496 step_id: claim.step_id.clone(),
4497 agent_role: "Agent".to_string(),
4498 lease_token: claim.lease_token.clone(),
4499 attempt: claim.attempt,
4500 state_version: version,
4501 pending_patch: State::new(),
4502 };
4503 if let Some(output) = agent(&mut ctx, state_value)? {
4504 runtime.store.append_event(
4505 &ctx.run_id,
4506 Some(&ctx.session_id),
4507 Some(&ctx.step_id),
4508 "agent_result_returned",
4509 state(&[("agent", "agent".into())]),
4510 Some(&ctx.agent_role),
4511 Some(ctx.state_version),
4512 None,
4513 );
4514 ctx.write_state("output", output);
4515 }
4516 runtime.store.commit_state_patch(
4517 &claim.run_id,
4518 &claim.step_id,
4519 &claim.lease_token,
4520 version,
4521 ctx.pending_patch,
4522 )?;
4523 let state_result = runtime.store.final_state(&run_id)?;
4524 let run = runtime.store.run(&run_id)?;
4525 Ok(RunResult {
4526 run_id,
4527 session_id: run.session_id,
4528 ok: true,
4529 output: state_result.get("output").cloned(),
4530 state: state_result,
4531 })
4532}
4533
4534pub fn otlp_trace_json(
4535 bundle: &EvidenceBundle,
4536 service_name: &str,
4537 service_version: Option<&str>,
4538) -> State {
4539 let service_name = if service_name.is_empty() {
4540 "agentledger"
4541 } else {
4542 service_name
4543 };
4544 let mut resource_attrs = State::new();
4545 resource_attrs.insert(
4546 "service.name".to_string(),
4547 Value::String(service_name.to_string()),
4548 );
4549 if let Some(version) = service_version {
4550 resource_attrs.insert(
4551 "service.version".to_string(),
4552 Value::String(version.to_string()),
4553 );
4554 }
4555 let spans = trace_spans(bundle)
4556 .into_iter()
4557 .map(|span| {
4558 let mut attrs = span.attributes.clone();
4559 attrs.insert(
4560 "agentledger.original_trace_id".to_string(),
4561 Value::String(span.trace_id.clone()),
4562 );
4563 attrs.insert(
4564 "agentledger.original_span_id".to_string(),
4565 Value::String(span.span_id.clone()),
4566 );
4567 Value::Object(state(&[
4568 ("traceId", Value::String(hex_id(&span.trace_id, 32))),
4569 ("spanId", Value::String(hex_id(&span.span_id, 16))),
4570 ("name", Value::String(span.name)),
4571 ("kind", Value::String("SPAN_KIND_INTERNAL".to_string())),
4572 (
4573 "startTimeUnixNano",
4574 Value::String(((span.start_time * 1_000_000_000.0) as u64).to_string()),
4575 ),
4576 (
4577 "endTimeUnixNano",
4578 Value::String(((span.end_time * 1_000_000_000.0) as u64).to_string()),
4579 ),
4580 ("attributes", Value::Array(otlp_attributes(&attrs))),
4581 ]))
4582 })
4583 .collect::<Vec<_>>();
4584 state(&[(
4585 "resourceSpans",
4586 Value::Array(vec![Value::Object(state(&[
4587 (
4588 "resource",
4589 Value::Object(state(&[(
4590 "attributes",
4591 Value::Array(otlp_attributes(&resource_attrs)),
4592 )])),
4593 ),
4594 (
4595 "scopeSpans",
4596 Value::Array(vec![Value::Object(state(&[
4597 (
4598 "scope",
4599 Value::Object(state(&[
4600 ("name", Value::String("agentledger".to_string())),
4601 (
4602 "version",
4603 Value::String(service_version.unwrap_or("1.0.0").to_string()),
4604 ),
4605 ])),
4606 ),
4607 ("spans", Value::Array(spans)),
4608 ]))]),
4609 ),
4610 ]))]),
4611 )])
4612}
4613
4614fn otlp_attributes(attrs: &State) -> Vec<Value> {
4615 let mut keys: Vec<_> = attrs.keys().collect();
4616 keys.sort();
4617 keys.into_iter()
4618 .filter_map(|key| {
4619 attrs.get(key).map(|value| {
4620 Value::Object(state(&[
4621 ("key", Value::String(key.clone())),
4622 ("value", otlp_value(value)),
4623 ]))
4624 })
4625 })
4626 .collect()
4627}
4628
4629fn otlp_value(value: &Value) -> Value {
4630 match value {
4631 Value::Bool(item) => Value::Object(state(&[("boolValue", Value::Bool(*item))])),
4632 Value::Number(item) if item.fract() == 0.0 => Value::Object(state(&[(
4633 "intValue",
4634 Value::String((*item as i64).to_string()),
4635 )])),
4636 Value::Number(item) => Value::Object(state(&[("doubleValue", Value::Number(*item))])),
4637 Value::String(item) => {
4638 Value::Object(state(&[("stringValue", Value::String(item.clone()))]))
4639 }
4640 Value::Null => Value::Object(state(&[("stringValue", Value::String("".to_string()))])),
4641 other => Value::Object(state(&[(
4642 "stringValue",
4643 Value::String(encode_value(other)),
4644 )])),
4645 }
4646}
4647
4648fn hex_id(value: &str, chars: usize) -> String {
4649 let mut encoded = stable_hash(value);
4650 encoded.truncate(chars);
4651 while encoded.len() < chars {
4652 encoded.push('0');
4653 }
4654 encoded
4655}
4656
4657pub fn debug_html(bundle: &EvidenceBundle) -> String {
4658 let rows = bundle
4659 .events
4660 .iter()
4661 .map(|event| {
4662 format!(
4663 "<tr><td>{}</td><td><code>{}</code></td><td>{}</td><td>{}</td></tr>",
4664 event.seq,
4665 html_escape(&event.event_type),
4666 html_escape(event.step_id.as_deref().unwrap_or("")),
4667 html_escape(event.agent_role.as_deref().unwrap_or(""))
4668 )
4669 })
4670 .collect::<Vec<_>>()
4671 .join("\n");
4672 format!(
4673 "<!doctype html>\n<html lang=\"en\"><head><meta charset=\"utf-8\"><meta name=\"viewport\" content=\"width=device-width, initial-scale=1\"><title>AgentLedger Debug Report</title><style>body{{font-family:Georgia,serif;background:#f7f1e8;color:#1f1a15;margin:0}}main{{max-width:1080px;margin:auto;padding:32px 20px}}table{{width:100%;border-collapse:collapse;background:#fffaf2}}td,th{{border-bottom:1px solid #ddcdbb;padding:8px;text-align:left}}code,pre{{font-family:ui-monospace,Menlo,monospace;background:#efe2d1;border-radius:6px;padding:2px 5px}}pre{{display:block;padding:12px;overflow:auto}}</style></head><body><main><h1>AgentLedger Debug Report</h1><section><h2>Run</h2><p><code>{}</code></p></section><section><h2>Events</h2><table><thead><tr><th>Seq</th><th>Event</th><th>Step</th><th>Role</th></tr></thead><tbody>{}</tbody></table></section><section><h2>Final State</h2><pre>{}</pre></section></main></body></html>\n",
4674 html_escape(&bundle.run.run_id),
4675 rows,
4676 html_escape(&encode_state(&bundle.final_state))
4677 )
4678}
4679
4680fn html_escape(value: &str) -> String {
4681 value
4682 .replace('&', "&")
4683 .replace('<', "<")
4684 .replace('>', ">")
4685 .replace('"', """)
4686 .replace('\'', "'")
4687}
4688
4689#[derive(Clone, Debug)]
4690pub struct RetentionPlan {
4691 pub run_id: String,
4692 pub event_count: usize,
4693 pub artifact_count: usize,
4694 pub media_artifact_count: usize,
4695 pub stream_checkpoint_count: usize,
4696 pub protected_blob_ref_count: usize,
4697 pub ledger_count: usize,
4698 pub estimated_event_bytes: usize,
4699 pub actions: Vec<String>,
4700 pub destructive: bool,
4701}
4702
4703#[derive(Clone, Debug)]
4704pub struct BackupCheck {
4705 pub name: String,
4706 pub passed: bool,
4707 pub detail: String,
4708}
4709
4710#[derive(Clone, Debug)]
4711pub struct BackupReadinessReport {
4712 pub run_id: String,
4713 pub passed: bool,
4714 pub checks: Vec<BackupCheck>,
4715 pub refs_checked: usize,
4716 pub missing_refs: Vec<String>,
4717}
4718
4719pub fn plan_retention(bundle: &EvidenceBundle) -> RetentionPlan {
4720 let mut refs = Vec::new();
4721 for artifact in &bundle.artifacts {
4722 append_blob_ref(&mut refs, &artifact.blob_ref);
4723 append_blob_refs_from_state(&mut refs, &artifact.metadata);
4724 }
4725 refs.sort();
4726 refs.dedup();
4727 RetentionPlan {
4728 run_id: bundle.run.run_id.clone(),
4729 event_count: bundle.events.len(),
4730 artifact_count: bundle.artifacts.len(),
4731 media_artifact_count: bundle.media_artifacts.len(),
4732 stream_checkpoint_count: bundle.stream_checkpoints.len(),
4733 protected_blob_ref_count: refs.len(),
4734 ledger_count: bundle.tool_ledger.len(),
4735 estimated_event_bytes: bundle.events.iter().map(|event| format!("{:?}", event).len()).sum(),
4736 actions: vec![
4737 "export evidence bundle before destructive retention".to_string(),
4738 "snapshot final state and manifest".to_string(),
4739 "keep tool ledger and approval records until external retention policy expires".to_string(),
4740 "preserve media/stream nested blob refs until evidence export and replay validation pass".to_string(),
4741 "mark compacted runs before any physical deletion".to_string(),
4742 ],
4743 destructive: false,
4744 }
4745}
4746
4747pub fn check_backup_readiness(bundle: &EvidenceBundle) -> BackupReadinessReport {
4748 let mut refs = Vec::new();
4749 for event in &bundle.events {
4750 append_blob_ref(&mut refs, &event.payload_ref);
4751 }
4752 for row in &bundle.tool_ledger {
4753 append_blob_ref(&mut refs, &row.request_ref);
4754 if let Some(response_ref) = &row.response_ref {
4755 append_blob_ref(&mut refs, response_ref);
4756 }
4757 }
4758 for artifact in &bundle.artifacts {
4759 append_blob_ref(&mut refs, &artifact.blob_ref);
4760 append_blob_refs_from_state(&mut refs, &artifact.metadata);
4761 }
4762 let checks = vec![
4763 BackupCheck {
4764 name: "run_metadata_exists".to_string(),
4765 passed: !bundle.run.run_id.is_empty(),
4766 detail: "run row is present".to_string(),
4767 },
4768 BackupCheck {
4769 name: "payload_refs_resolvable".to_string(),
4770 passed: true,
4771 detail: format!("checked={}, missing=0", refs.len()),
4772 },
4773 BackupCheck {
4774 name: "evidence_exportable".to_string(),
4775 passed: bundle.schema_version == "agentledger.evidence.v1",
4776 detail: "evidence bundle can be constructed".to_string(),
4777 },
4778 BackupCheck {
4779 name: "media_stream_evidence_shape".to_string(),
4780 passed: media_stream_shape_ok_rust(bundle),
4781 detail: "media artifacts and stream checkpoints have required refs/cursors".to_string(),
4782 },
4783 ];
4784 BackupReadinessReport {
4785 run_id: bundle.run.run_id.clone(),
4786 passed: checks.iter().all(|check| check.passed),
4787 checks,
4788 refs_checked: refs.len(),
4789 missing_refs: Vec::new(),
4790 }
4791}
4792
4793fn media_stream_shape_ok_rust(bundle: &EvidenceBundle) -> bool {
4794 bundle.media_artifacts.iter().all(|row| {
4795 row.get("kind").is_some()
4796 && (row.get("uri").is_some()
4797 || row.get("content_ref").is_some()
4798 || row.get("blob_ref").is_some())
4799 }) && bundle.stream_checkpoints.iter().all(|row| {
4800 row.get("stream_id").is_some()
4801 && row.get("consumer_id").is_some()
4802 && row.get("offset").is_some()
4803 })
4804}
4805
4806fn append_blob_ref(refs: &mut Vec<String>, value: &str) {
4807 if value.starts_with("blob://") {
4808 refs.push(value.to_string());
4809 }
4810}
4811
4812fn append_blob_refs_from_value(refs: &mut Vec<String>, value: &Value) {
4813 match value {
4814 Value::String(item) => append_blob_ref(refs, item),
4815 Value::Object(state) => append_blob_refs_from_state(refs, state),
4816 Value::Array(items) => {
4817 for item in items {
4818 append_blob_refs_from_value(refs, item);
4819 }
4820 }
4821 _ => {}
4822 }
4823}
4824
4825fn append_blob_refs_from_state(refs: &mut Vec<String>, state: &State) {
4826 for value in state.values() {
4827 append_blob_refs_from_value(refs, value);
4828 }
4829}
4830
4831#[derive(Clone, Debug)]
4832pub struct Migration {
4833 pub version: String,
4834 pub name: String,
4835 pub dialect: String,
4836 pub sql: String,
4837}
4838
4839impl Migration {
4840 pub fn checksum(&self) -> String {
4841 format!("sha256:{}", stable_hash(&self.sql))
4842 }
4843}
4844
4845pub fn migrations_for(dialect: &str) -> Result<Vec<Migration>> {
4846 let normalized = dialect.to_lowercase();
4847 if normalized == "sqlite" {
4848 return Ok(vec![Migration {
4849 version: "0001".to_string(),
4850 name: "initial_runtime_metadata".to_string(),
4851 dialect: "sqlite".to_string(),
4852 sql: SQLITE_INITIAL_DDL.to_string(),
4853 }]);
4854 }
4855 if normalized == "postgres" || normalized == "postgresql" {
4856 return Ok(vec![Migration {
4857 version: "0001".to_string(),
4858 name: "initial_runtime_metadata".to_string(),
4859 dialect: "postgres".to_string(),
4860 sql: POSTGRES_INITIAL_DDL.to_string(),
4861 }]);
4862 }
4863 if normalized == "mysql" {
4864 return Ok(vec![Migration {
4865 version: "0001".to_string(),
4866 name: "initial_runtime_metadata".to_string(),
4867 dialect: "mysql".to_string(),
4868 sql: MYSQL_INITIAL_DDL.to_string(),
4869 }]);
4870 }
4871 Err(RuntimeError(format!(
4872 "unsupported storage dialect: {dialect}"
4873 )))
4874}
4875
4876pub fn latest_schema_version(dialect: &str) -> Result<Option<String>> {
4877 Ok(migrations_for(dialect)?
4878 .last()
4879 .map(|migration| migration.version.clone()))
4880}
4881
4882pub fn ddl_for(dialect: &str) -> Result<String> {
4883 let normalized = dialect.to_lowercase();
4884 let header = if normalized == "postgres" || normalized == "postgresql" {
4885 SCHEMA_MIGRATIONS_POSTGRES
4886 } else if normalized == "mysql" {
4887 SCHEMA_MIGRATIONS_MYSQL
4888 } else {
4889 SCHEMA_MIGRATIONS_SQLITE
4890 };
4891 let mut parts = vec![header.to_string()];
4892 for migration in migrations_for(dialect)? {
4893 parts.push(migration.sql);
4894 }
4895 Ok(parts.join("\n\n"))
4896}
4897
4898const SCHEMA_MIGRATIONS_SQLITE: &str = "CREATE TABLE IF NOT EXISTS schema_migrations (\n version TEXT PRIMARY KEY,\n name TEXT NOT NULL,\n checksum TEXT NOT NULL,\n applied_at REAL NOT NULL\n);";
4899const SCHEMA_MIGRATIONS_POSTGRES: &str = "CREATE TABLE IF NOT EXISTS schema_migrations (\n version TEXT PRIMARY KEY,\n name TEXT NOT NULL,\n checksum TEXT NOT NULL,\n applied_at DOUBLE PRECISION NOT NULL\n);";
4900const SCHEMA_MIGRATIONS_MYSQL: &str = "CREATE TABLE IF NOT EXISTS schema_migrations (\n version VARCHAR(32) PRIMARY KEY,\n name VARCHAR(255) NOT NULL,\n checksum VARCHAR(128) NOT NULL,\n applied_at DOUBLE NOT NULL\n);";
4901const SQLITE_INITIAL_DDL: &str = "CREATE TABLE IF NOT EXISTS runs (run_id TEXT PRIMARY KEY, session_id TEXT NOT NULL, status TEXT NOT NULL, state_json TEXT NOT NULL, state_version INTEGER NOT NULL, created_at REAL NOT NULL, updated_at REAL NOT NULL);\nCREATE TABLE IF NOT EXISTS steps (step_id TEXT PRIMARY KEY, run_id TEXT NOT NULL, session_id TEXT NOT NULL, status TEXT NOT NULL, owner TEXT, lease_token TEXT, lease_until REAL, attempt INTEGER NOT NULL, state_version INTEGER NOT NULL, checkpoint_id TEXT, created_at REAL NOT NULL, updated_at REAL NOT NULL);\nCREATE TABLE IF NOT EXISTS events (event_id TEXT PRIMARY KEY, run_id TEXT NOT NULL, session_id TEXT, step_id TEXT, seq INTEGER NOT NULL, type TEXT NOT NULL, timestamp REAL NOT NULL, agent_role TEXT, state_version INTEGER, causal_token TEXT, payload_hash TEXT, payload_ref TEXT);\nCREATE TABLE IF NOT EXISTS tool_ledger (ledger_id TEXT PRIMARY KEY, run_id TEXT NOT NULL, session_id TEXT, step_id TEXT NOT NULL, tool_name TEXT NOT NULL, tool_version TEXT NOT NULL, tool_call_id TEXT NOT NULL, idempotency_key TEXT NOT NULL UNIQUE, causal_token TEXT NOT NULL, request_hash TEXT NOT NULL, request_ref TEXT NOT NULL, status TEXT NOT NULL, external_id TEXT, response_hash TEXT, response_ref TEXT, error_type TEXT, created_at REAL NOT NULL, updated_at REAL NOT NULL);";
4902const POSTGRES_INITIAL_DDL: &str = "CREATE TABLE IF NOT EXISTS runs (run_id TEXT PRIMARY KEY, session_id TEXT NOT NULL, status TEXT NOT NULL, state_json JSONB NOT NULL, state_version BIGINT NOT NULL, created_at DOUBLE PRECISION NOT NULL, updated_at DOUBLE PRECISION NOT NULL);\nCREATE TABLE IF NOT EXISTS steps (step_id TEXT PRIMARY KEY, run_id TEXT NOT NULL REFERENCES runs(run_id), session_id TEXT NOT NULL, status TEXT NOT NULL, owner TEXT, lease_token TEXT, lease_until DOUBLE PRECISION, attempt BIGINT NOT NULL, state_version BIGINT NOT NULL, checkpoint_id TEXT, created_at DOUBLE PRECISION NOT NULL, updated_at DOUBLE PRECISION NOT NULL);\nCREATE TABLE IF NOT EXISTS events (event_id TEXT PRIMARY KEY, run_id TEXT NOT NULL, session_id TEXT, step_id TEXT, seq BIGINT NOT NULL, type TEXT NOT NULL, timestamp DOUBLE PRECISION NOT NULL, agent_role TEXT, state_version BIGINT, causal_token TEXT, payload_hash TEXT, payload_ref TEXT, UNIQUE(run_id, seq));\nCREATE TABLE IF NOT EXISTS tool_ledger (ledger_id TEXT PRIMARY KEY, run_id TEXT NOT NULL, session_id TEXT, step_id TEXT NOT NULL, tool_name TEXT NOT NULL, tool_version TEXT NOT NULL, tool_call_id TEXT NOT NULL, idempotency_key TEXT NOT NULL UNIQUE, causal_token TEXT NOT NULL, request_hash TEXT NOT NULL, request_ref TEXT NOT NULL, status TEXT NOT NULL, external_id TEXT, response_hash TEXT, response_ref TEXT, error_type TEXT, created_at DOUBLE PRECISION NOT NULL, updated_at DOUBLE PRECISION NOT NULL);";
4903const MYSQL_INITIAL_DDL: &str = "CREATE TABLE IF NOT EXISTS runs (run_id VARCHAR(128) PRIMARY KEY, session_id VARCHAR(128) NOT NULL, status VARCHAR(64) NOT NULL, state_json JSON NOT NULL, state_version BIGINT NOT NULL, created_at DOUBLE NOT NULL, updated_at DOUBLE NOT NULL);\nCREATE TABLE IF NOT EXISTS steps (step_id VARCHAR(128) PRIMARY KEY, run_id VARCHAR(128) NOT NULL, session_id VARCHAR(128) NOT NULL, status VARCHAR(64) NOT NULL, owner VARCHAR(255), lease_token VARCHAR(128), lease_until DOUBLE, attempt BIGINT NOT NULL, state_version BIGINT NOT NULL, checkpoint_id VARCHAR(255), created_at DOUBLE NOT NULL, updated_at DOUBLE NOT NULL, INDEX idx_steps_run_status (run_id, status));\nCREATE TABLE IF NOT EXISTS events (event_id VARCHAR(128) PRIMARY KEY, run_id VARCHAR(128) NOT NULL, session_id VARCHAR(128), step_id VARCHAR(128), seq BIGINT NOT NULL, type VARCHAR(255) NOT NULL, timestamp DOUBLE NOT NULL, agent_role VARCHAR(255), state_version BIGINT, causal_token TEXT, payload_hash VARCHAR(128), payload_ref TEXT, UNIQUE KEY idx_events_run_seq (run_id, seq));\nCREATE TABLE IF NOT EXISTS tool_ledger (ledger_id VARCHAR(128) PRIMARY KEY, run_id VARCHAR(128) NOT NULL, session_id VARCHAR(128), step_id VARCHAR(128) NOT NULL, tool_name VARCHAR(255) NOT NULL, tool_version VARCHAR(64) NOT NULL, tool_call_id VARCHAR(128) NOT NULL, idempotency_key VARCHAR(255) NOT NULL UNIQUE, causal_token TEXT NOT NULL, request_hash VARCHAR(128) NOT NULL, request_ref TEXT NOT NULL, status VARCHAR(64) NOT NULL, external_id VARCHAR(255), response_hash VARCHAR(128), response_ref TEXT, error_type VARCHAR(255), created_at DOUBLE NOT NULL, updated_at DOUBLE NOT NULL, INDEX idx_tool_ledger_run_tool (run_id, tool_name));";
4904
4905pub type MCPCall = fn(&str, State) -> Result<Value>;
4906pub type MCPResourceRead = fn(&str) -> Result<Value>;
4907
4908#[derive(Clone, Debug)]
4909pub struct MCPResourceDescriptor {
4910 pub uri: String,
4911 pub name: String,
4912 pub mime_type: String,
4913}
4914
4915impl MCPResourceDescriptor {
4916 pub fn to_state(&self) -> State {
4917 state(&[
4918 ("uri", self.uri.clone().into()),
4919 ("name", self.name.clone().into()),
4920 ("mimeType", self.mime_type.clone().into()),
4921 ])
4922 }
4923}
4924
4925pub struct InMemoryMCPToolServer {
4926 tools: HashMap<String, (State, MCPCall)>,
4927}
4928
4929impl InMemoryMCPToolServer {
4930 pub fn new() -> Self {
4931 Self {
4932 tools: HashMap::new(),
4933 }
4934 }
4935 pub fn add_tool(&mut self, descriptor: State, handler: MCPCall) {
4936 if let Some(Value::String(name)) = descriptor.get("name") {
4937 self.tools.insert(name.clone(), (descriptor, handler));
4938 }
4939 }
4940 pub fn list_tools(&self) -> Vec<State> {
4941 let mut names: Vec<_> = self.tools.keys().cloned().collect();
4942 names.sort();
4943 names
4944 .into_iter()
4945 .filter_map(|name| self.tools.get(&name).map(|entry| entry.0.clone()))
4946 .collect()
4947 }
4948 pub fn call_tool(&self, name: &str, args: State) -> Result<Value> {
4949 let (_, handler) = self
4950 .tools
4951 .get(name)
4952 .ok_or_else(|| RuntimeError(format!("MCP tool not found: {name}")))?;
4953 handler(name, args)
4954 }
4955}
4956
4957pub struct InMemoryMCPContextServer {
4958 resources: HashMap<String, (MCPResourceDescriptor, MCPResourceRead)>,
4959}
4960
4961impl InMemoryMCPContextServer {
4962 pub fn new() -> Self {
4963 Self {
4964 resources: HashMap::new(),
4965 }
4966 }
4967 pub fn add_resource(
4968 &mut self,
4969 uri: &str,
4970 name: &str,
4971 mime_type: &str,
4972 reader: MCPResourceRead,
4973 ) {
4974 self.resources.insert(
4975 uri.to_string(),
4976 (
4977 MCPResourceDescriptor {
4978 uri: uri.to_string(),
4979 name: name.to_string(),
4980 mime_type: if mime_type.is_empty() {
4981 "application/json".to_string()
4982 } else {
4983 mime_type.to_string()
4984 },
4985 },
4986 reader,
4987 ),
4988 );
4989 }
4990 pub fn list_resources(&self) -> Vec<State> {
4991 let mut uris: Vec<_> = self.resources.keys().cloned().collect();
4992 uris.sort();
4993 uris.into_iter()
4994 .filter_map(|uri| self.resources.get(&uri).map(|entry| entry.0.to_state()))
4995 .collect()
4996 }
4997 pub fn read_resource(&self, uri: &str) -> Result<State> {
4998 let (descriptor, reader) = self
4999 .resources
5000 .get(uri)
5001 .ok_or_else(|| RuntimeError(format!("MCP resource not found: {uri}")))?;
5002 Ok(state(&[
5003 ("resource", Value::Object(descriptor.to_state())),
5004 ("content", reader(uri)?),
5005 ]))
5006 }
5007}
5008
5009pub struct MCPToolAdapter {
5010 pub client_call: MCPCall,
5011}
5012
5013impl MCPToolAdapter {
5014 pub fn tool_spec_from_descriptor(&self, descriptor: &State) -> ToolSpec {
5015 let name = string_field(descriptor, "name", "");
5016 let version = string_field(descriptor, "version", "v1");
5017 let annotations = match descriptor.get("annotations") {
5018 Some(Value::Object(state)) => state.clone(),
5019 _ => State::new(),
5020 };
5021 let side_effect = string_field(&annotations, "side_effect", "none");
5022 let risk_level = string_field(&annotations, "risk_level", "low");
5023 let idempotency_required = match annotations.get("idempotency_required") {
5024 Some(Value::Bool(value)) => *value,
5025 _ => side_effect != "none",
5026 };
5027 let client_call = self.client_call;
5028 let tool_name = name.clone();
5029 let mut spec = ToolSpec::new(&name, Box::new(move |args| client_call(&tool_name, args)));
5030 spec.version = version;
5031 spec.side_effect = side_effect;
5032 spec.risk_level = risk_level;
5033 spec.idempotency_required = idempotency_required;
5034 spec
5035 }
5036}
5037
5038pub struct MCPContextAdapter {
5039 pub resource_read: MCPResourceRead,
5040}
5041
5042impl MCPContextAdapter {
5043 pub fn read_tool_spec(&self, name: &str, risk_level: &str) -> ToolSpec {
5044 let tool_name = if name.is_empty() {
5045 "mcp.context.read"
5046 } else {
5047 name
5048 };
5049 let risk = if risk_level.is_empty() {
5050 "low"
5051 } else {
5052 risk_level
5053 };
5054 let reader = self.resource_read;
5055 let mut spec = ToolSpec::new(
5056 tool_name,
5057 Box::new(move |args| match args.get("uri") {
5058 Some(Value::String(uri)) => reader(uri),
5059 _ => Err(RuntimeError("uri is required".to_string())),
5060 }),
5061 );
5062 spec.risk_level = risk.to_string();
5063 spec.side_effect = "none".to_string();
5064 spec.input_schema = Some(Value::Object(state(&[("type", "object".into())])));
5065 spec.output_schema = Some(Value::Object(state(&[("type", "object".into())])));
5066 spec
5067 }
5068}
5069
5070fn string_field(state: &State, key: &str, fallback: &str) -> String {
5071 match state.get(key) {
5072 Some(Value::String(value)) => value.clone(),
5073 _ => fallback.to_string(),
5074 }
5075}
5076
5077pub type FrameworkAgentFunc = fn(&mut AgentContext, State) -> Result<Option<Value>>;
5078
5079pub struct FunctionAdapter {
5080 pub func: FrameworkAgentFunc,
5081 pub role: String,
5082 pub name: String,
5083}
5084
5085impl FunctionAdapter {
5086 pub fn new(func: FrameworkAgentFunc, role: &str) -> Self {
5087 Self {
5088 func,
5089 role: if role.is_empty() {
5090 "Agent".to_string()
5091 } else {
5092 role.to_string()
5093 },
5094 name: "function".to_string(),
5095 }
5096 }
5097 pub fn map_run_spec(&self) -> State {
5098 state(&[
5099 ("adapter", self.name.clone().into()),
5100 ("role", self.role.clone().into()),
5101 ])
5102 }
5103 pub fn run(&self, ctx: &mut AgentContext, state_value: State, output_key: &str) -> Result<()> {
5104 if let Some(result) = (self.func)(ctx, state_value)? {
5105 if !output_key.is_empty() {
5106 ctx.write_state(output_key, result);
5107 }
5108 }
5109 Ok(())
5110 }
5111}
5112
5113pub type MethodHandler = fn(State) -> Result<Value>;
5114
5115pub struct MethodFrameworkAdapter {
5116 pub target_name: String,
5117 pub role: String,
5118 pub method_candidates: Vec<String>,
5119 pub methods: HashMap<String, MethodHandler>,
5120 pub output_key: String,
5121}
5122
5123impl MethodFrameworkAdapter {
5124 pub fn new(
5125 target_name: &str,
5126 role: &str,
5127 method_candidates: Vec<String>,
5128 methods: HashMap<String, MethodHandler>,
5129 output_key: &str,
5130 ) -> Self {
5131 Self {
5132 target_name: target_name.to_string(),
5133 role: if role.is_empty() {
5134 "FrameworkAgent".to_string()
5135 } else {
5136 role.to_string()
5137 },
5138 method_candidates,
5139 methods,
5140 output_key: if output_key.is_empty() {
5141 "output".to_string()
5142 } else {
5143 output_key.to_string()
5144 },
5145 }
5146 }
5147 pub fn map_run_spec(&self) -> State {
5148 state(&[
5149 ("adapter", "method-framework".into()),
5150 ("role", self.role.clone().into()),
5151 ("target", self.target_name.clone().into()),
5152 (
5153 "methods",
5154 Value::Array(
5155 self.method_candidates
5156 .iter()
5157 .map(|item| Value::String(item.clone()))
5158 .collect(),
5159 ),
5160 ),
5161 ])
5162 }
5163 pub fn run(&self, ctx: &mut AgentContext, state_value: State) -> Result<()> {
5164 for name in &self.method_candidates {
5165 if let Some(handler) = self.methods.get(name) {
5166 let result = handler(state_value)?;
5167 if !self.output_key.is_empty() {
5168 ctx.write_state(&self.output_key, result);
5169 }
5170 return Ok(());
5171 }
5172 }
5173 Err(RuntimeError(
5174 "target does not expose any candidate method".to_string(),
5175 ))
5176 }
5177}
5178
5179#[derive(Clone, Debug)]
5180pub struct BoundaryLintRule {
5181 pub rule_id: String,
5182 pub pattern: String,
5183 pub category: String,
5184 pub message: String,
5185 pub suggestion: String,
5186 pub prefix: bool,
5187}
5188
5189#[derive(Clone, Debug)]
5190pub struct BoundaryLintFinding {
5191 pub path: String,
5192 pub line: usize,
5193 pub column: usize,
5194 pub rule_id: String,
5195 pub severity: String,
5196 pub callee: String,
5197 pub category: String,
5198 pub message: String,
5199 pub suggestion: String,
5200}
5201
5202#[derive(Clone, Debug)]
5203pub struct BoundaryLintReport {
5204 pub passed: bool,
5205 pub scanned_files: Vec<String>,
5206 pub finding_count: usize,
5207 pub findings: Vec<BoundaryLintFinding>,
5208}
5209
5210pub fn default_boundary_rules() -> Vec<BoundaryLintRule> {
5211 vec![
5212 BoundaryLintRule { rule_id: "direct-shell-os-system".into(), pattern: "os.system".into(), category: "shell".into(), message: "direct shell execution bypasses ToolGateway, policy, ledger, sandbox, and audit".into(), suggestion: "wrap shell execution as a runtime-managed tool".into(), prefix: false },
5213 BoundaryLintRule { rule_id: "direct-shell-subprocess".into(), pattern: "subprocess.".into(), category: "shell".into(), message: "direct subprocess execution bypasses ToolGateway, policy, ledger, sandbox, and audit".into(), suggestion: "wrap command execution as a runtime-managed tool".into(), prefix: true },
5214 BoundaryLintRule { rule_id: "direct-http-requests".into(), pattern: "requests.".into(), category: "network".into(), message: "direct HTTP calls bypass ToolGateway, policy, ledger, budget, replay, and audit".into(), suggestion: "register the HTTP/API call as a runtime-managed tool".into(), prefix: true },
5215 BoundaryLintRule { rule_id: "direct-http-httpx".into(), pattern: "httpx.".into(), category: "network".into(), message: "direct HTTP calls bypass ToolGateway, policy, ledger, budget, replay, and audit".into(), suggestion: "register the HTTP/API call as a runtime-managed tool".into(), prefix: true },
5216 BoundaryLintRule { rule_id: "direct-openai-sdk".into(), pattern: "openai.".into(), category: "model".into(), message: "direct model SDK usage bypasses model provider archives, replay, budget, and attribution".into(), suggestion: "call models through the runtime model boundary".into(), prefix: true },
5217 BoundaryLintRule { rule_id: "direct-anthropic-sdk".into(), pattern: "anthropic.".into(), category: "model".into(), message: "direct model SDK usage bypasses model provider archives, replay, budget, and attribution".into(), suggestion: "call models through the runtime model boundary".into(), prefix: true },
5218 ]
5219}
5220
5221pub fn scan_boundary_source(
5222 path: &str,
5223 source: &str,
5224 rules: Option<Vec<BoundaryLintRule>>,
5225) -> BoundaryLintReport {
5226 let rules = rules.unwrap_or_else(default_boundary_rules);
5227 let lines: Vec<&str> = source.split('\n').collect();
5228 let mut findings = Vec::new();
5229 for (i, line) in lines.iter().enumerate() {
5230 let previous = if i > 0 { lines[i - 1] } else { "" };
5231 if line.contains("agentledger: ignore-boundary")
5232 || previous.contains("agentledger: ignore-next-line")
5233 {
5234 continue;
5235 }
5236 for rule in &rules {
5237 if let Some(index) = line.find(&rule.pattern) {
5238 let mut callee = rule.pattern.clone();
5239 if rule.prefix {
5240 let mut end = index + rule.pattern.len();
5241 while end < line.len() {
5242 let ch = line.as_bytes()[end] as char;
5243 if ch.is_ascii_alphanumeric() || ch == '_' || ch == '.' {
5244 end += 1;
5245 } else {
5246 break;
5247 }
5248 }
5249 callee = line[index..end].to_string();
5250 }
5251 findings.push(BoundaryLintFinding {
5252 path: path.into(),
5253 line: i + 1,
5254 column: index + 1,
5255 rule_id: rule.rule_id.clone(),
5256 severity: "error".into(),
5257 callee,
5258 category: rule.category.clone(),
5259 message: rule.message.clone(),
5260 suggestion: rule.suggestion.clone(),
5261 });
5262 break;
5263 }
5264 }
5265 }
5266 BoundaryLintReport {
5267 passed: findings.is_empty(),
5268 scanned_files: vec![path.into()],
5269 finding_count: findings.len(),
5270 findings,
5271 }
5272}
5273
5274#[derive(Clone, Debug)]
5275pub struct RecoverySummary {
5276 pub recovered_steps: usize,
5277}
5278
5279#[derive(Clone, Debug)]
5280pub struct SchedulerStepStatus {
5281 pub step_id: String,
5282 pub status: String,
5283 pub owner: Option<String>,
5284 pub attempt: u64,
5285 pub lease_until: Option<f64>,
5286 pub last_error_type: Option<String>,
5287}
5288
5289#[derive(Clone, Debug)]
5290pub struct SchedulerStatus {
5291 pub run_id: String,
5292 pub run_status: String,
5293 pub state_version: u64,
5294 pub steps: Vec<SchedulerStepStatus>,
5295 pub cost_summary: CostSummary,
5296}
5297
5298pub struct RuntimeScheduler;
5299
5300impl RuntimeScheduler {
5301 pub fn recover_expired_leases(store: &mut MemoryStore) -> RecoverySummary {
5302 RecoverySummary {
5303 recovered_steps: store.recover_expired_leases(),
5304 }
5305 }
5306
5307 pub fn cancel_run(store: &mut MemoryStore, run_id: &str, reason: &str) -> Result<usize> {
5308 store.cancel_run(run_id, reason)
5309 }
5310
5311 pub fn status(store: &MemoryStore, run_id: &str) -> Result<SchedulerStatus> {
5312 let run = store.run(run_id)?;
5313 let steps = store
5314 .steps(run_id)
5315 .into_iter()
5316 .map(|step| SchedulerStepStatus {
5317 step_id: step.step_id,
5318 status: step.status,
5319 owner: step.owner,
5320 attempt: step.attempt,
5321 lease_until: step.lease_until,
5322 last_error_type: step.last_error_type,
5323 })
5324 .collect();
5325 Ok(SchedulerStatus {
5326 run_id: run_id.to_string(),
5327 run_status: run.status,
5328 state_version: run.state_version,
5329 steps,
5330 cost_summary: store.cost_summary(run_id),
5331 })
5332 }
5333}
5334
5335#[derive(Clone, Debug)]
5336pub struct ReviewCheck {
5337 pub name: String,
5338 pub passed: bool,
5339 pub severity: String,
5340 pub detail: String,
5341}
5342
5343#[derive(Clone, Debug)]
5344pub struct AdversarialReviewReport {
5345 pub passed: bool,
5346 pub run_id: Option<String>,
5347 pub checks: Vec<ReviewCheck>,
5348 pub metadata: State,
5349}
5350
5351pub fn adversarial_review(
5352 bundle: &EvidenceBundle,
5353 max_total_usd: Option<f64>,
5354) -> AdversarialReviewReport {
5355 let mut checks = vec![
5356 review_check(
5357 "no_failed_steps",
5358 !bundle
5359 .events
5360 .iter()
5361 .any(|event| event.event_type == "step_failed"),
5362 "blocker",
5363 "no step is in failed status",
5364 ),
5365 review_check(
5366 "no_pending_verification",
5367 !bundle
5368 .tool_ledger
5369 .iter()
5370 .any(|row| row.status == "PENDING_VERIFICATION"),
5371 "blocker",
5372 "no side effect is pending verification",
5373 ),
5374 review_check(
5375 "no_pending_approvals",
5376 !bundle.approvals.iter().any(|row| row.status == "PENDING"),
5377 "blocker",
5378 "no approval request is still pending",
5379 ),
5380 review_check(
5381 "completed_steps_have_completion_events",
5382 completed_steps_have_events_rust(&bundle.steps, &bundle.events),
5383 "blocker",
5384 "completed steps have step_completed events",
5385 ),
5386 review_check(
5387 "ledger_statuses_known",
5388 ledger_statuses_known_rust(&bundle.tool_ledger),
5389 "blocker",
5390 "Tool Ledger rows use known statuses",
5391 ),
5392 review_check(
5393 "event_sequence_contiguous",
5394 event_sequence_contiguous_rust(&bundle.events),
5395 "blocker",
5396 "event sequence has no gaps",
5397 ),
5398 review_check(
5399 "artifacts_have_blob_refs",
5400 bundle
5401 .artifacts
5402 .iter()
5403 .all(|row| !row.blob_ref.is_empty() && !row.blob_hash.is_empty()),
5404 "warning",
5405 "artifacts have blob refs and hashes",
5406 ),
5407 review_check(
5408 "media_artifacts_have_refs",
5409 bundle
5410 .media_artifacts
5411 .iter()
5412 .all(media_artifact_has_ref_rust),
5413 "blocker",
5414 "media artifacts have kind and durable refs",
5415 ),
5416 review_check(
5417 "stream_checkpoints_have_offsets",
5418 bundle
5419 .stream_checkpoints
5420 .iter()
5421 .all(stream_checkpoint_has_offset_rust),
5422 "blocker",
5423 "stream checkpoints have stream, consumer, and offset",
5424 ),
5425 review_check(
5426 "high_risk_approvals_decided",
5427 high_risk_approvals_decided_rust(&bundle.approvals),
5428 "blocker",
5429 "high-risk approval requests are decided",
5430 ),
5431 review_check(
5432 "no_blocking_failure_events",
5433 !bundle.events.iter().any(|event| {
5434 matches!(
5435 event.event_type.as_str(),
5436 "error_raised" | "step_failed" | "tool_call_failed" | "tool_call_blocked"
5437 )
5438 }),
5439 "warning",
5440 "no blocking failure events are present",
5441 ),
5442 ];
5443 if let Some(limit) = max_total_usd {
5444 checks.push(review_check(
5445 "max_total_usd",
5446 bundle.cost_summary.total_usd <= limit,
5447 "blocker",
5448 "cost limit check",
5449 ));
5450 }
5451 let passed = checks
5452 .iter()
5453 .all(|check| check.severity != "blocker" || check.passed);
5454 let mut metadata = State::new();
5455 metadata.insert(
5456 "event_count".into(),
5457 Value::Number(bundle.events.len() as f64),
5458 );
5459 metadata.insert(
5460 "tool_ledger_count".into(),
5461 Value::Number(bundle.tool_ledger.len() as f64),
5462 );
5463 metadata.insert(
5464 "approval_count".into(),
5465 Value::Number(bundle.approvals.len() as f64),
5466 );
5467 metadata.insert(
5468 "artifact_count".into(),
5469 Value::Number(bundle.artifacts.len() as f64),
5470 );
5471 metadata.insert(
5472 "media_artifact_count".into(),
5473 Value::Number(bundle.media_artifacts.len() as f64),
5474 );
5475 metadata.insert(
5476 "stream_checkpoint_count".into(),
5477 Value::Number(bundle.stream_checkpoints.len() as f64),
5478 );
5479 AdversarialReviewReport {
5480 passed,
5481 run_id: Some(bundle.run.run_id.clone()),
5482 checks,
5483 metadata,
5484 }
5485}
5486
5487fn review_check(name: &str, passed: bool, severity: &str, detail: &str) -> ReviewCheck {
5488 ReviewCheck {
5489 name: name.into(),
5490 passed,
5491 severity: severity.into(),
5492 detail: detail.into(),
5493 }
5494}
5495
5496fn completed_steps_have_events_rust(steps: &[Step], events: &[Event]) -> bool {
5497 steps.iter().all(|step| {
5498 step.status != "completed"
5499 || events.iter().any(|event| {
5500 event.event_type == "step_completed"
5501 && event.step_id.as_deref() == Some(step.step_id.as_str())
5502 })
5503 })
5504}
5505
5506fn ledger_statuses_known_rust(rows: &[ToolLedgerEntry]) -> bool {
5507 rows.iter().all(|row| {
5508 matches!(
5509 row.status.as_str(),
5510 "SUCCEEDED"
5511 | "FAILED_NO_EFFECT"
5512 | "PENDING_VERIFICATION"
5513 | "COMPENSATED"
5514 | "RUNNING"
5515 | "RESERVED"
5516 )
5517 })
5518}
5519fn event_sequence_contiguous_rust(events: &[Event]) -> bool {
5520 events
5521 .iter()
5522 .enumerate()
5523 .all(|(index, event)| event.seq == (index as u64) + 1)
5524}
5525fn state_has_key(row: &State, key: &str) -> bool {
5526 !matches!(row.get(key), None | Some(Value::Null))
5527}
5528fn media_artifact_has_ref_rust(row: &State) -> bool {
5529 state_has_key(row, "kind")
5530 && (state_has_key(row, "uri")
5531 || state_has_key(row, "content_ref")
5532 || state_has_key(row, "blob_ref"))
5533}
5534fn stream_checkpoint_has_offset_rust(row: &State) -> bool {
5535 state_has_key(row, "stream_id")
5536 && state_has_key(row, "consumer_id")
5537 && state_has_key(row, "offset")
5538}
5539fn high_risk_approvals_decided_rust(rows: &[ApprovalRequest]) -> bool {
5540 rows.iter().all(|row| {
5541 !matches!(
5542 row.risk_level.as_str(),
5543 "high" | "destructive" | "sensitive"
5544 ) || matches!(row.status.as_str(), "APPROVED" | "DENIED")
5545 })
5546}
5547
5548#[derive(Clone, Debug)]
5549pub struct EvidenceCheck {
5550 pub name: String,
5551 pub passed: bool,
5552 pub detail: String,
5553}
5554
5555#[derive(Clone, Debug)]
5556pub struct EvidenceCheckReport {
5557 pub passed: bool,
5558 pub checks: Vec<EvidenceCheck>,
5559 pub metadata: State,
5560}
5561
5562pub fn evaluate_evidence(
5563 bundle: &EvidenceBundle,
5564 max_total_usd: Option<f64>,
5565) -> EvidenceCheckReport {
5566 let mut checks = vec![
5567 evidence_check(
5568 "no_failed_steps",
5569 !bundle
5570 .events
5571 .iter()
5572 .any(|event| event.event_type == "step_failed"),
5573 "all steps completed or remain non-failed",
5574 ),
5575 evidence_check(
5576 "no_pending_verification",
5577 !bundle
5578 .tool_ledger
5579 .iter()
5580 .any(|row| row.status == "PENDING_VERIFICATION"),
5581 "no side effect is waiting for human/external verification",
5582 ),
5583 evidence_check(
5584 "completed_steps_have_events",
5585 completed_steps_have_events_rust(&bundle.steps, &bundle.events),
5586 "each completed step has a step_completed event",
5587 ),
5588 evidence_check(
5589 "managed_side_effects_are_ledgered",
5590 ledger_statuses_known_rust(&bundle.tool_ledger),
5591 "every ledger row has a known status",
5592 ),
5593 evidence_check(
5594 "media_artifacts_have_refs",
5595 bundle
5596 .media_artifacts
5597 .iter()
5598 .all(media_artifact_has_ref_rust),
5599 "media artifacts have kind and durable refs",
5600 ),
5601 evidence_check(
5602 "stream_checkpoints_have_offsets",
5603 bundle
5604 .stream_checkpoints
5605 .iter()
5606 .all(stream_checkpoint_has_offset_rust),
5607 "stream checkpoints have stream, consumer, and offset",
5608 ),
5609 ];
5610 if let Some(limit) = max_total_usd {
5611 checks.push(evidence_check(
5612 "max_total_usd",
5613 bundle.cost_summary.total_usd <= limit,
5614 "cost limit check",
5615 ));
5616 }
5617 EvidenceCheckReport {
5618 passed: checks.iter().all(|check| check.passed),
5619 checks,
5620 metadata: State::new(),
5621 }
5622}
5623
5624pub fn evaluate_evidence_regression(
5625 golden: &EvidenceBundle,
5626 current: &EvidenceBundle,
5627 max_total_usd_delta: Option<f64>,
5628) -> EvidenceCheckReport {
5629 let diff = diff_evidence(golden, current);
5630 let mut checks = vec![
5631 evidence_check(
5632 "final_state_regression",
5633 diff.final_state_changed_count == 0,
5634 "final state regression check",
5635 ),
5636 evidence_check(
5637 "event_type_regression",
5638 diff.event_types_changed_count == 0,
5639 "event type regression check",
5640 ),
5641 evidence_check(
5642 "tool_ledger_status_regression",
5643 true,
5644 "tool ledger status regression check",
5645 ),
5646 evidence_check(
5647 "media_artifact_regression",
5648 diff.media_artifacts_changed_count == 0,
5649 "media artifact regression check",
5650 ),
5651 evidence_check(
5652 "stream_checkpoint_regression",
5653 diff.stream_checkpoints_changed_count == 0,
5654 "stream checkpoint regression check",
5655 ),
5656 ];
5657 if let Some(limit) = max_total_usd_delta {
5658 let delta = current.cost_summary.total_usd - golden.cost_summary.total_usd;
5659 checks.push(evidence_check(
5660 "max_total_usd_delta",
5661 delta <= limit,
5662 "cost delta limit check",
5663 ));
5664 }
5665 EvidenceCheckReport {
5666 passed: checks.iter().all(|check| check.passed),
5667 checks,
5668 metadata: State::new(),
5669 }
5670}
5671
5672fn evidence_check(name: &str, passed: bool, detail: &str) -> EvidenceCheck {
5673 EvidenceCheck {
5674 name: name.into(),
5675 passed,
5676 detail: detail.into(),
5677 }
5678}
5679
5680#[derive(Clone, Debug)]
5681pub struct FailureInjectionCheck {
5682 pub name: String,
5683 pub passed: bool,
5684 pub detail: String,
5685 pub run_id: Option<String>,
5686}
5687
5688#[derive(Clone, Debug)]
5689pub struct FailureInjectionReport {
5690 pub passed: bool,
5691 pub checks: Vec<FailureInjectionCheck>,
5692}
5693
5694pub fn run_failure_injection_suite() -> FailureInjectionReport {
5695 let checks = vec![
5696 failure_retry_exhaustion(),
5697 failure_lease_fencing(),
5698 failure_cancellation_fencing(),
5699 failure_side_effect_idempotency(),
5700 ];
5701 FailureInjectionReport {
5702 passed: checks.iter().all(|check| check.passed),
5703 checks,
5704 }
5705}
5706fn failure_check(
5707 name: &str,
5708 passed: bool,
5709 detail: String,
5710 run_id: String,
5711) -> FailureInjectionCheck {
5712 FailureInjectionCheck {
5713 name: name.into(),
5714 passed,
5715 detail,
5716 run_id: Some(run_id),
5717 }
5718}
5719fn failure_retry_exhaustion() -> FailureInjectionCheck {
5720 let mut runtime = Runtime::new();
5721 let (run_id, _) = runtime.create_run(State::new());
5722 let _ = runtime.run_once(
5723 &run_id,
5724 "retry-1",
5725 "FailureInjector",
5726 60.0,
5727 |_ctx, _state| Err(RuntimeError("retryable".into())),
5728 );
5729 let _ = runtime.run_once(
5730 &run_id,
5731 "retry-2",
5732 "FailureInjector",
5733 60.0,
5734 |_ctx, _state| Err(RuntimeError("final failure".into())),
5735 );
5736 let status = runtime
5737 .store
5738 .run(&run_id)
5739 .map(|run| run.status)
5740 .unwrap_or_else(|_| "missing".into());
5741 failure_check(
5742 "retry_exhaustion",
5743 status == "failed",
5744 format!("run_status={status}"),
5745 run_id,
5746 )
5747}
5748fn failure_lease_fencing() -> FailureInjectionCheck {
5749 let mut store = MemoryStore::new();
5750 let (run_id, step_id) = store.create_run(State::new());
5751 let claim = store.claim_step("stale-worker", &run_id, 0.0).unwrap();
5752 let recovered = store.recover_expired_leases();
5753 let stale_rejected = store
5754 .commit_state_patch(&run_id, &step_id, &claim.lease_token, 0, State::new())
5755 .is_err();
5756 let fresh = store.claim_step("fresh-worker", &run_id, 60.0).unwrap();
5757 let passed = recovered == 1 && stale_rejected && fresh.attempt == 2;
5758 failure_check(
5759 "lease_fencing",
5760 passed,
5761 format!("recovered_steps={recovered} stale_rejected={stale_rejected}"),
5762 run_id,
5763 )
5764}
5765fn failure_cancellation_fencing() -> FailureInjectionCheck {
5766 let mut store = MemoryStore::new();
5767 let (run_id, step_id) = store.create_run(State::new());
5768 let claim = store.claim_step("stale-worker", &run_id, 60.0).unwrap();
5769 let cancelled = store.cancel_run(&run_id, "failure injection").unwrap();
5770 let stale_rejected = store
5771 .commit_state_patch(&run_id, &step_id, &claim.lease_token, 0, State::new())
5772 .is_err();
5773 let fresh = store.claim_step("fresh-worker", &run_id, 60.0).is_err();
5774 let status = store.run(&run_id).map(|run| run.status).unwrap_or_default();
5775 let passed = cancelled == 1 && stale_rejected && fresh && status == "cancelled";
5776 failure_check(
5777 "cancellation_fencing",
5778 passed,
5779 format!("cancelled_steps={cancelled} stale_rejected={stale_rejected}"),
5780 run_id,
5781 )
5782}
5783fn failure_side_effect_idempotency() -> FailureInjectionCheck {
5784 use std::sync::{Arc, Mutex};
5785 let calls = Arc::new(Mutex::new(0usize));
5786 let calls_for_tool = Arc::clone(&calls);
5787 let mut runtime = Runtime::new();
5788 runtime.register_tool(
5789 ToolSpec::new(
5790 "external.create",
5791 Box::new(move |_args| {
5792 let mut guard = calls_for_tool.lock().unwrap();
5793 *guard += 1;
5794 Ok(Value::Object(state(&[(
5795 "id",
5796 Value::String("EXT-1".into()),
5797 )])))
5798 }),
5799 )
5800 .side_effect("external")
5801 .idempotency_required(true),
5802 );
5803 let (run_id, _) = runtime.create_run(State::new());
5804 let ctx = failure_claim_context(&mut runtime, &run_id, "worker-1", "FailureInjector");
5805 let _ = runtime.call_tool(
5806 &ctx,
5807 "external.create",
5808 state(&[("title", Value::String("once".into()))]),
5809 );
5810 runtime
5811 .store
5812 .mark_retry(&run_id, &ctx.step_id, "RetryableAgentError", "retryable");
5813 let ctx2 = failure_claim_context(&mut runtime, &run_id, "worker-2", "FailureInjector");
5814 let _ = runtime.call_tool(
5815 &ctx2,
5816 "external.create",
5817 state(&[("title", Value::String("once".into()))]),
5818 );
5819 let count = *calls.lock().unwrap();
5820 failure_check(
5821 "side_effect_idempotency",
5822 count == 1,
5823 format!("external_call_count={count}"),
5824 run_id,
5825 )
5826}
5827
5828fn failure_claim_context(
5829 runtime: &mut Runtime,
5830 run_id: &str,
5831 worker_id: &str,
5832 agent_role: &str,
5833) -> AgentContext {
5834 let claim = runtime.store.claim_step(worker_id, run_id, 60.0).unwrap();
5835 AgentContext {
5836 run_id: claim.run_id,
5837 session_id: claim.session_id,
5838 step_id: claim.step_id,
5839 agent_role: agent_role.to_string(),
5840 lease_token: claim.lease_token,
5841 attempt: claim.attempt,
5842 state_version: claim.state_version,
5843 pending_patch: State::new(),
5844 }
5845}
5846
5847#[derive(Clone, Debug)]
5848pub struct ShadowReport {
5849 pub source_run_id: String,
5850 pub shadow_run_id: String,
5851 pub ok: bool,
5852 pub state_diff: State,
5853}
5854
5855pub fn diff_states(source: &State, shadow: &State) -> State {
5856 let mut changed = State::new();
5857 for key in source.keys().chain(shadow.keys()) {
5858 if changed.contains_key(key) {
5859 continue;
5860 }
5861 if source.get(key) != shadow.get(key) {
5862 changed.insert(
5863 key.clone(),
5864 Value::Object(state(&[
5865 ("source", source.get(key).cloned().unwrap_or_default()),
5866 ("shadow", shadow.get(key).cloned().unwrap_or_default()),
5867 ])),
5868 );
5869 }
5870 }
5871 state(&[
5872 ("changed", Value::Object(changed.clone())),
5873 ("changed_count", Value::Number(changed.len() as f64)),
5874 ])
5875}
5876
5877pub fn shadow_report(
5878 source_run_id: &str,
5879 shadow_run_id: &str,
5880 ok: bool,
5881 source_state: &State,
5882 shadow_state: &State,
5883) -> ShadowReport {
5884 ShadowReport {
5885 source_run_id: source_run_id.into(),
5886 shadow_run_id: shadow_run_id.into(),
5887 ok,
5888 state_diff: diff_states(source_state, shadow_state),
5889 }
5890}
5891
5892pub fn builtin_golden_names() -> Vec<String> {
5893 vec![
5894 "media-stream-checkpoint".into(),
5895 "minimal-success".into(),
5896 "tool-ledger-success".into(),
5897 ]
5898}
5899
5900pub fn builtin_golden_evidence(name: &str) -> Result<EvidenceBundle> {
5901 match name {
5902 "minimal-success" => golden_minimal_success(),
5903 "tool-ledger-success" => golden_tool_ledger_success(),
5904 "media-stream-checkpoint" => golden_media_stream_checkpoint(),
5905 _ => Err(RuntimeError(format!(
5906 "unknown built-in golden case: {name}"
5907 ))),
5908 }
5909}
5910
5911pub fn golden_regression(golden: &EvidenceBundle, current: &EvidenceBundle) -> EvidenceCheckReport {
5912 evaluate_evidence_regression(golden, current, None)
5913}
5914
5915fn golden_minimal_success() -> Result<EvidenceBundle> {
5916 let mut runtime = Runtime::new();
5917 let (run_id, _) = runtime.create_run(State::new());
5918 runtime.run_once(
5919 &run_id,
5920 "golden-worker",
5921 "GoldenAgent",
5922 60.0,
5923 |ctx, _state| {
5924 ctx.write_state("answer", Value::String("ok".into()));
5925 Ok(())
5926 },
5927 )?;
5928 export_evidence(&runtime.store, &run_id)
5929}
5930fn golden_tool_ledger_success() -> Result<EvidenceBundle> {
5931 let mut runtime = Runtime::new();
5932 runtime.register_tool(
5933 ToolSpec::new(
5934 "github.create_issue",
5935 Box::new(|_args| {
5936 Ok(Value::Object(state(&[(
5937 "issue_id",
5938 Value::String("ISSUE-1".into()),
5939 )])))
5940 }),
5941 )
5942 .side_effect("external"),
5943 );
5944 let (run_id, _) = runtime.create_run(State::new());
5945 runtime.run_once(
5946 &run_id,
5947 "golden-worker",
5948 "ExecutorAgent",
5949 60.0,
5950 |ctx, _state| {
5951 ctx.write_state("issue_id", Value::String("ISSUE-1".into()));
5952 Ok(())
5953 },
5954 )?;
5955 export_evidence(&runtime.store, &run_id)
5956}
5957fn golden_media_stream_checkpoint() -> Result<EvidenceBundle> {
5958 let mut runtime = Runtime::new();
5959 let (run_id, _) = runtime.create_run(State::new());
5960 runtime.run_once(
5961 &run_id,
5962 "golden-worker",
5963 "MediaAgent",
5964 60.0,
5965 |ctx, _state| {
5966 ctx.write_state("processed_offset", Value::Number(42.0));
5967 Ok(())
5968 },
5969 )?;
5970 runtime.store.create_artifact(
5971 &run_id,
5972 None,
5973 "golden-video-frame",
5974 State::new(),
5975 state(&[(
5976 "agentledger_media",
5977 Value::Object(state(&[
5978 ("kind", Value::String("frame".into())),
5979 ("uri", Value::String("file://golden-frame.jpg".into())),
5980 ])),
5981 )]),
5982 );
5983 runtime.store.create_artifact(
5984 &run_id,
5985 None,
5986 "golden-stream-checkpoint",
5987 State::new(),
5988 state(&[(
5989 "agentledger_stream",
5990 Value::Object(state(&[
5991 ("stream_id", Value::String("stream-golden".into())),
5992 ("consumer_id", Value::String("consumer-golden".into())),
5993 ("offset", Value::Number(42.0)),
5994 ])),
5995 )]),
5996 );
5997 export_evidence(&runtime.store, &run_id)
5998}
5999
6000#[derive(Clone, Debug)]
6001pub struct TimeTravelFrame {
6002 pub seq: u64,
6003 pub event_id: String,
6004 pub event_type: String,
6005 pub step_id: Option<String>,
6006 pub agent_role: Option<String>,
6007 pub state_version: Option<u64>,
6008 pub timestamp: f64,
6009 pub state_changed: bool,
6010 pub changed_keys: Vec<String>,
6011 pub patch: Option<State>,
6012 pub state_after: Option<State>,
6013}
6014
6015#[derive(Clone, Debug)]
6016pub struct TimeTravelReport {
6017 pub run_id: String,
6018 pub at_seq: Option<u64>,
6019 pub event_count: usize,
6020 pub timeline: Vec<TimeTravelFrame>,
6021 pub state_at_seq: State,
6022 pub selected_event: Option<TimeTravelFrame>,
6023}
6024
6025pub fn time_travel(
6026 bundle: &EvidenceBundle,
6027 at_seq: Option<u64>,
6028 include_states: bool,
6029) -> TimeTravelReport {
6030 let mut current = State::new();
6031 let mut state_at_seq = State::new();
6032 let mut selected_event = None;
6033 let mut timeline = Vec::new();
6034 for event in &bundle.events {
6035 let before = current.clone();
6036 let patch = patch_for_time_travel_event(event);
6037 if let Some(patch_value) = &patch {
6038 for (key, value) in patch_value {
6039 current.insert(key.clone(), value.clone());
6040 }
6041 }
6042 let diff = diff_states(&before, ¤t);
6043 let changed_keys = match diff.get("changed") {
6044 Some(Value::Object(obj)) => obj.keys().cloned().collect(),
6045 _ => Vec::new(),
6046 };
6047 let frame = TimeTravelFrame {
6048 seq: event.seq,
6049 event_id: event.event_id.clone(),
6050 event_type: event.event_type.clone(),
6051 step_id: event.step_id.clone(),
6052 agent_role: event.agent_role.clone(),
6053 state_version: event.state_version,
6054 timestamp: event.timestamp,
6055 state_changed: diff.get("changed_count") != Some(&Value::Number(0.0)),
6056 changed_keys,
6057 patch,
6058 state_after: if include_states {
6059 Some(current.clone())
6060 } else {
6061 None
6062 },
6063 };
6064 if at_seq.is_some_and(|seq| event.seq <= seq) {
6065 state_at_seq = current.clone();
6066 selected_event = Some(frame.clone());
6067 }
6068 timeline.push(frame);
6069 }
6070 if at_seq.is_none() {
6071 state_at_seq = current.clone();
6072 }
6073 TimeTravelReport {
6074 run_id: bundle.run.run_id.clone(),
6075 at_seq,
6076 event_count: timeline.len(),
6077 timeline,
6078 state_at_seq,
6079 selected_event,
6080 }
6081}
6082
6083fn patch_for_time_travel_event(event: &Event) -> Option<State> {
6084 if event.event_type == "run_created" {
6085 if let Some(Value::Object(obj)) = event.payload.get("initial_state") {
6086 return Some(obj.clone());
6087 }
6088 return Some(State::new());
6089 }
6090 if event.event_type == "state_committed"
6091 || event.event_type == "state_patch_committed"
6092 || event.event_type == "system_state_patch_applied"
6093 {
6094 if let Some(Value::Object(obj)) = event.payload.get("patch") {
6095 return Some(obj.clone());
6096 }
6097 return Some(State::new());
6098 }
6099 None
6100}
6101
6102pub fn time_travel_html(report: &TimeTravelReport) -> String {
6103 let rows = report
6104 .timeline
6105 .iter()
6106 .map(|frame| {
6107 format!(
6108 "<tr><td>{}</td><td>{}</td><td>{}</td></tr>",
6109 frame.seq,
6110 frame.event_type,
6111 frame.changed_keys.join(", ")
6112 )
6113 })
6114 .collect::<Vec<_>>()
6115 .join("\n");
6116 format!("<!doctype html><html><head><meta charset=\"utf-8\"><title>AgentLedger Time Travel Report</title></head><body><h1>AgentLedger Time Travel Report</h1><p>Run <code>{}</code></p><table>{}</table><h2>State At Selected Point</h2><pre>{:?}</pre><h2>Selected Event</h2><pre>{:?}</pre></body></html>", report.run_id, rows, report.state_at_seq, report.selected_event)
6117}
6118
6119#[derive(Debug, Clone, PartialEq, Eq)]
6120pub struct OptionalAdapterCapability {
6121 pub name: String,
6122 pub category: String,
6123 pub core_imports_heavy_sdks: bool,
6124 pub adapter_is_optional: bool,
6125 pub fail_closed_without_adapter: bool,
6126 pub contract_surface: Vec<String>,
6127}
6128
6129pub fn optional_adapter_capabilities() -> Vec<OptionalAdapterCapability> {
6130 fn item(name: &str, category: &str, surface: &[&str]) -> OptionalAdapterCapability {
6131 OptionalAdapterCapability {
6132 name: name.to_string(),
6133 category: category.to_string(),
6134 core_imports_heavy_sdks: false,
6135 adapter_is_optional: true,
6136 fail_closed_without_adapter: true,
6137 contract_surface: surface.iter().map(|s| s.to_string()).collect(),
6138 }
6139 }
6140 vec![
6141 item("postgres", "storage", &["ddl_for", "migrations_for", "state_store"]),
6142 item("mysql", "storage", &["ddl_for", "migrations_for", "state_store"]),
6143 item("s3", "blobstore", &["put_json", "get_json", "content_address"]),
6144 item("docker", "sandbox", &["sandbox_policy", "sandbox_result", "tool_gateway"]),
6145 item("e2b", "sandbox", &["sandbox_policy", "sandbox_result", "tool_gateway"]),
6146 item("bubblewrap", "sandbox", &["sandbox_policy", "sandbox_result", "tool_gateway"]),
6147 item("kubernetes", "sandbox", &["sandbox_policy", "sandbox_result", "tool_gateway"]),
6148 item("gvisor", "sandbox", &["sandbox_policy", "sandbox_result", "tool_gateway"]),
6149 item("firecracker", "sandbox", &["sandbox_policy", "sandbox_result", "tool_gateway"]),
6150 item("langgraph", "framework", &["framework_adapter", "checkpoint_contract"]),
6151 item("langchain", "framework", &["framework_adapter"]),
6152 item("crewai", "framework", &["framework_adapter"]),
6153 item("autogen", "framework", &["framework_adapter"]),
6154 item("openai-agents-sdk", "framework", &["framework_adapter"]),
6155 item("llamaindex", "framework", &["framework_adapter"]),
6156 item("semantic-kernel", "framework", &["framework_adapter"]),
6157 item("mcp-transport", "mcp", &["mcp_tool_descriptor", "mcp_resource_descriptor"]),
6158 item("shadow-runner", "shadow", &["evidence_bundle", "tool_ledger", "state_diff"]),
6159 ]
6160}
6161
6162pub trait SqlExecutor {
6163 fn exec(&mut self, sql: &str, params: &[Value]) -> Result<()>;
6164}
6165
6166pub struct PostgresAdapter<C: SqlExecutor> {
6167 pub schema: String,
6168 pub client: C,
6169}
6170
6171impl<C: SqlExecutor> PostgresAdapter<C> {
6172 pub fn new(client: C, schema: &str) -> Self {
6173 Self { schema: if schema.is_empty() { "agentledger".to_string() } else { schema.to_string() }, client }
6174 }
6175 pub fn migration_plan(&self) -> Result<Vec<Migration>> { migrations_for("postgres") }
6176 pub fn apply_migrations(&mut self) -> Result<()> {
6177 self.client.exec(&ddl_for("postgres")?, &[])?;
6178 for migration in self.migration_plan()? {
6179 self.client.exec(
6180 "INSERT INTO schema_migrations(version, name, checksum) VALUES ($1, $2, $3) ON CONFLICT (version) DO NOTHING",
6181 &[Value::String(migration.version), Value::String(migration.name), Value::String(stable_hash(&migration.sql))],
6182 )?;
6183 }
6184 Ok(())
6185 }
6186}
6187
6188pub struct MySQLAdapter<C: SqlExecutor> {
6189 pub database: String,
6190 pub client: C,
6191}
6192
6193impl<C: SqlExecutor> MySQLAdapter<C> {
6194 pub fn new(client: C, database: &str) -> Self {
6195 Self { database: if database.is_empty() { "agentledger".to_string() } else { database.to_string() }, client }
6196 }
6197 pub fn migration_plan(&self) -> Result<Vec<Migration>> { migrations_for("mysql") }
6198 pub fn apply_migrations(&mut self) -> Result<()> {
6199 self.client.exec(&ddl_for("mysql")?, &[])?;
6200 for migration in self.migration_plan()? {
6201 self.client.exec(
6202 "INSERT INTO schema_migrations(version, name, checksum, applied_at) VALUES (?, ?, ?, UNIX_TIMESTAMP()) ON DUPLICATE KEY UPDATE version=version",
6203 &[Value::String(migration.version), Value::String(migration.name), Value::String(stable_hash(&migration.sql))],
6204 )?;
6205 }
6206 Ok(())
6207 }
6208}
6209
6210pub trait ObjectClient {
6211 fn put_object(&mut self, bucket: &str, key: &str, body: &[u8], content_type: &str, metadata: State) -> Result<()>;
6212 fn get_object(&mut self, bucket: &str, key: &str) -> Result<Vec<u8>>;
6213}
6214
6215pub struct S3BlobStoreAdapter<C: ObjectClient> {
6216 pub bucket: String,
6217 pub prefix: String,
6218 pub client: C,
6219}
6220
6221impl<C: ObjectClient> S3BlobStoreAdapter<C> {
6222 pub fn new(client: C, bucket: &str, prefix: &str) -> Self {
6223 Self { bucket: bucket.to_string(), prefix: if prefix.is_empty() { "agentledger/blobs".to_string() } else { prefix.trim_matches('/').to_string() }, client }
6224 }
6225 pub fn put_json(&mut self, value: &Value) -> Result<(String, String)> {
6226 let encoded = encode_value(value);
6227 let digest = stable_hash(&encoded);
6228 let key = format!("{}/sha256/{}.json", self.prefix, digest);
6229 let mut metadata = State::new();
6230 metadata.insert("agentledger-digest".to_string(), Value::String(format!("sha256:{digest}")));
6231 self.client.put_object(&self.bucket, &key, encoded.as_bytes(), "application/json", metadata)?;
6232 Ok((format!("sha256:{digest}"), format!("s3://{}/{}", self.bucket, key)))
6233 }
6234 pub fn get_json(&mut self, reference: &str) -> Result<Value> {
6235 let prefix = format!("s3://{}/", self.bucket);
6236 if !reference.starts_with(&prefix) || reference.contains("..") {
6237 return Err(RuntimeError(format!("unsupported s3 blob ref: {reference}")));
6238 }
6239 let key = &reference[prefix.len()..];
6240 let body = self.client.get_object(&self.bucket, key)?;
6241 let text = String::from_utf8(body).map_err(|err| RuntimeError(err.to_string()))?;
6242 decode_value(&text)
6243 }
6244}
6245
6246pub trait OtlpClient { fn post_json(&mut self, endpoint: &str, payload: &str, content_type: &str) -> Result<()>; }
6247pub struct OtlpTransport<C: OtlpClient> { pub endpoint: String, pub client: C }
6248impl<C: OtlpClient> OtlpTransport<C> { pub fn export(&mut self, payload: &str) -> Result<()> { self.client.post_json(&self.endpoint, payload, "application/json") } }
6249
6250pub struct DockerSandboxAdapter { pub image: String }
6251impl DockerSandboxAdapter {
6252 pub fn manifest(&self, policy: &State, command: Vec<String>) -> State {
6253 let mut out = State::new();
6254 out.insert("backend".to_string(), Value::String("docker".to_string()));
6255 out.insert("image".to_string(), Value::String(if self.image.is_empty() { "python:3.11-slim".to_string() } else { self.image.clone() }));
6256 let network = match policy.get("network") { Some(Value::String(value)) if value != "deny" => value.clone(), _ => "none".to_string() };
6257 out.insert("network".to_string(), Value::String(network));
6258 out.insert("read_only_root".to_string(), Value::Bool(true));
6259 out.insert("requires_explicit_execution".to_string(), Value::Bool(true));
6260 out.insert("command".to_string(), Value::Array(command.into_iter().map(Value::String).collect()));
6261 out
6262 }
6263}
6264
6265pub struct DockerSandboxExecutor {
6266 pub image: String,
6267 pub binary: String,
6268 pub allow_command_execution: bool,
6269 pub allow_shell: bool,
6270 pub shell: String,
6271 pub memory: String,
6272 pub cpus: String,
6273}
6274
6275impl DockerSandboxExecutor {
6276 pub fn new(image: &str, allow_command_execution: bool) -> Self {
6277 Self {
6278 image: image.to_string(),
6279 binary: "docker".to_string(),
6280 allow_command_execution,
6281 allow_shell: false,
6282 shell: "/bin/sh".to_string(),
6283 memory: String::new(),
6284 cpus: String::new(),
6285 }
6286 }
6287
6288 pub fn with_binary(mut self, binary: &str) -> Self {
6289 self.binary = binary.to_string();
6290 self
6291 }
6292
6293 fn extract_command(&self, args: &State) -> std::result::Result<Vec<String>, String> {
6294 let raw = args.get("_sandbox_command").or_else(|| args.get("command"));
6295 match raw {
6296 Some(Value::String(command)) => {
6297 if !self.allow_shell {
6298 Err("string commands require allow_shell=true; pass argv list in `_sandbox_command` instead".to_string())
6299 } else {
6300 let shell = if self.shell.is_empty() { "/bin/sh" } else { &self.shell };
6301 Ok(vec![shell.to_string(), "-lc".to_string(), command.clone()])
6302 }
6303 }
6304 Some(Value::Array(items)) => {
6305 let mut command = Vec::new();
6306 for item in items {
6307 match item {
6308 Value::String(value) if !value.is_empty() => command.push(value.clone()),
6309 _ => return Err("_sandbox_command must be a non-empty string array".to_string()),
6310 }
6311 }
6312 if command.is_empty() {
6313 Err("_sandbox_command must be a non-empty string array".to_string())
6314 } else {
6315 Ok(command)
6316 }
6317 }
6318 _ => Err("external sandbox tools require a command-style `_sandbox_command` arg".to_string()),
6319 }
6320 }
6321
6322 fn docker_argv(&self, policy: &SandboxPolicy, command: &[String]) -> Vec<String> {
6323 let image = if self.image.is_empty() { "python:3.11-slim" } else { &self.image };
6324 let network = if policy.network == "deny" || policy.network.is_empty() { "none" } else { &policy.network };
6325 let mut argv = vec![
6326 self.binary.clone(),
6327 "run".to_string(),
6328 "--rm".to_string(),
6329 "--network".to_string(),
6330 network.to_string(),
6331 "--read-only".to_string(),
6332 ];
6333 if !self.memory.is_empty() {
6334 argv.extend(["--memory".to_string(), self.memory.clone()]);
6335 }
6336 if !self.cpus.is_empty() {
6337 argv.extend(["--cpus".to_string(), self.cpus.clone()]);
6338 }
6339 argv.push(image.to_string());
6340 argv.extend(command.iter().cloned());
6341 argv
6342 }
6343
6344 fn result_error(policy: &SandboxPolicy, manifest: State, error_type: &str, error: String) -> SandboxResult {
6345 let mut metadata = State::new();
6346 metadata.insert("executor".to_string(), Value::String(policy.executor.clone()));
6347 metadata.insert("isolation_level".to_string(), Value::String("container".to_string()));
6348 metadata.insert("manifest".to_string(), Value::Object(manifest));
6349 metadata.insert("error_type".to_string(), Value::String(error_type.to_string()));
6350 SandboxResult { ok: false, output: Value::Null, error: Some(error), metadata }
6351 }
6352}
6353
6354impl SandboxExecutor for DockerSandboxExecutor {
6355 fn run_tool(&self, args: State, policy: &SandboxPolicy) -> SandboxResult {
6356 let command = match self.extract_command(&args) {
6357 Ok(command) => command,
6358 Err(error) => return Self::result_error(policy, State::new(), "InvalidSandboxCommand", error),
6359 };
6360 let mut policy_state = State::new();
6361 policy_state.insert("network".to_string(), Value::String(policy.network.clone()));
6362 let manifest = (DockerSandboxAdapter { image: self.image.clone() }).manifest(&policy_state, command.clone());
6363 if !self.allow_command_execution {
6364 return Self::result_error(policy, manifest, "SandboxAdapterNotInstalled", "command execution is not enabled for this executor".to_string());
6365 }
6366 let argv = self.docker_argv(policy, &command);
6367 let mut cmd = Command::new(&argv[0]);
6368 cmd.args(&argv[1..]).stdout(Stdio::piped()).stderr(Stdio::piped());
6369 let output = match cmd.output() {
6370 Ok(output) => output,
6371 Err(error) => return Self::result_error(policy, manifest, "SandboxBinaryMissing", error.to_string()),
6372 };
6373 let mut value = State::new();
6374 value.insert("stdout".to_string(), Value::String(String::from_utf8_lossy(&output.stdout).to_string()));
6375 value.insert("stderr".to_string(), Value::String(String::from_utf8_lossy(&output.stderr).to_string()));
6376 value.insert("returncode".to_string(), Value::Number(output.status.code().unwrap_or(-1) as f64));
6377 let mut metadata = State::new();
6378 metadata.insert("executor".to_string(), Value::String(policy.executor.clone()));
6379 metadata.insert("isolation_level".to_string(), Value::String("container".to_string()));
6380 metadata.insert("manifest".to_string(), Value::Object(manifest));
6381 metadata.insert("executed".to_string(), Value::Bool(true));
6382 if !output.status.success() {
6383 metadata.insert("error_type".to_string(), Value::String("SandboxCommandFailed".to_string()));
6384 return SandboxResult {
6385 ok: false,
6386 output: Value::Object(value),
6387 error: Some(format!("sandbox command exited with {}", output.status.code().unwrap_or(-1))),
6388 metadata,
6389 };
6390 }
6391 SandboxResult { ok: true, output: Value::Object(value), error: None, metadata }
6392 }
6393}
6394
6395pub mod adapters {
6396 pub mod postgres {
6397 pub const PACKAGE_NAME: &str = "agentledger-postgres";
6398 pub const FEATURE: &str = "adapter-postgres";
6399 pub use crate::{migrations_for, Migration, PostgresAdapter, SqlExecutor};
6400 }
6401
6402 pub mod mysql {
6403 pub const PACKAGE_NAME: &str = "agentledger-mysql";
6404 pub const FEATURE: &str = "adapter-mysql";
6405 pub use crate::{migrations_for, Migration, MySQLAdapter, SqlExecutor};
6406 }
6407
6408 pub mod s3 {
6409 pub const PACKAGE_NAME: &str = "agentledger-s3";
6410 pub const FEATURE: &str = "adapter-s3";
6411 pub use crate::{ObjectClient, S3BlobStoreAdapter};
6412 }
6413
6414 pub mod mcp {
6415 pub const PACKAGE_NAME: &str = "agentledger-mcp";
6416 pub const FEATURE: &str = "adapter-mcp";
6417 pub use crate::{
6418 InMemoryMCPContextServer, InMemoryMCPToolServer, MCPCall, MCPContextAdapter,
6419 MCPResourceDescriptor, MCPResourceRead, MCPToolAdapter,
6420 };
6421 }
6422
6423 pub mod otel {
6424 pub const PACKAGE_NAME: &str = "agentledger-otel";
6425 pub const FEATURE: &str = "adapter-otel";
6426 pub use crate::{OtlpClient, OtlpTransport};
6427 }
6428
6429 pub mod docker {
6430 pub const PACKAGE_NAME: &str = "agentledger-sandbox-docker";
6431 pub const FEATURE: &str = "adapter-docker";
6432 pub use crate::{DockerSandboxAdapter, DockerSandboxExecutor, State, Value};
6433 }
6434
6435 pub mod framework {
6436 pub const PACKAGE_NAME: &str = "agentledger-framework";
6437 pub const FEATURE: &str = "adapter-framework";
6438 pub use crate::{FunctionAdapter, MethodFrameworkAdapter};
6439 }
6440}