car_engine/executor.rs
1//! Core execution engine — propose → validate → execute → commit.
2
3use crate::cache::ResultCache;
4use crate::capabilities::CapabilitySet;
5use crate::checkpoint::Checkpoint;
6use crate::rate_limit::{RateLimit, RateLimiter};
7use car_eventlog::{EventKind, EventLog, SpanStatus};
8use car_ir::{
9 build_dag, AcceptedProposalPreimage, Action, ActionProposal, ActionResult, ActionStatus,
10 ActionType, CostSummary, FailureBehavior, ProposalLineageEntry, ProposalLineageStatus,
11 ProposalResult, StateMutation, ToolFailure, ToolFailureClassification, ToolSchema,
12};
13use car_policy::{ApprovalDecision, ApprovalLedger, ApprovalRecord, PermissionTier, PolicyEngine};
14use car_state::{RestoreDurability, StateStore};
15use car_validator::validate_action;
16use serde_json::Value;
17use sha2::{Digest, Sha256};
18use std::collections::HashMap;
19use std::sync::Arc;
20use std::time::Duration;
21use tokio::sync::{Mutex as TokioMutex, RwLock as TokioRwLock};
22use tokio::time::timeout;
23use tracing::instrument;
24use uuid::Uuid;
25
26/// Retry backoff constants.
27const RETRY_BASE_DELAY_MS: u64 = 100;
28const RETRY_BACKOFF_FACTOR: u64 = 2;
29const ROLLBACK_WARNING: &str =
30 "proposal aborted; state effects were rolled back; external effects may remain and were not undone";
31const PLAN_FALLBACK_ROLLBACK_WARNING: &str =
32 "planning candidate rejected; state effects were rolled back; external effects may remain and were not undone";
33const ROLLBACK_DURABILITY_ERROR: &str =
34 "durable state rollback failed before publication; in-memory state and idempotency entries were preserved";
35const ROLLBACK_DURABILITY_UNKNOWN: &str =
36 "state rollback was published but parent-directory durability is unknown; idempotency entries were invalidated for exact retry";
37
38// ---------------------------------------------------------------------------
39// Replan types — failure recovery via model callback
40// ---------------------------------------------------------------------------
41
42/// Callback trait for replanning failed proposals.
43/// Implement this to let the runtime ask the model for an alternative plan
44/// when a proposal aborts.
45#[async_trait::async_trait]
46pub trait ReplanCallback: Send + Sync {
47 async fn replan(&self, ctx: &ReplanContext) -> Result<ActionProposal, String>;
48}
49
50/// Context provided to the replan callback so the model can generate an alternative.
51#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
52pub struct ReplanContext {
53 /// Original proposal ID.
54 pub proposal_id: String,
55 /// Which replan attempt this is (1-indexed; attempt 1 = first replan after initial failure).
56 pub attempt: u32,
57 /// Actions that failed and caused the abort.
58 pub failed_actions: Vec<FailedActionSummary>,
59 /// Action IDs that succeeded before the abort (now rolled back).
60 pub completed_action_ids: Vec<String>,
61 /// State snapshot after rollback.
62 pub state_snapshot: HashMap<String, Value>,
63 /// How many replans remain.
64 pub replans_remaining: u32,
65 /// Original proposal source (model name, agent, etc.).
66 pub original_source: String,
67 /// Total actions in the original proposal.
68 pub original_action_count: usize,
69 /// The original goal/context from the proposal (for generating alternatives).
70 pub original_context: HashMap<String, Value>,
71}
72
73/// Summary of a failed action, included in ReplanContext.
74#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
75pub struct FailedActionSummary {
76 pub action_id: String,
77 pub tool: Option<String>,
78 pub error: String,
79 pub parameters: HashMap<String, Value>,
80}
81
82/// Configuration for the replan loop.
83#[derive(Debug, Clone)]
84pub struct ReplanConfig {
85 /// Maximum number of replan attempts. 0 = disabled (default).
86 pub max_replans: u32,
87 /// Delay in milliseconds between replan attempts. Prevents burning through
88 /// attempts instantly if the model returns garbage fast. 0 = no delay.
89 pub delay_ms: u64,
90 /// If true, replan proposals are scored via car-planner's verify() before
91 /// execution. Proposals with errors are rejected without executing.
92 /// Prevents the engine from running a worse plan than the one that failed.
93 pub verify_before_execute: bool,
94 /// When true, validator/policy/capability rejections (ActionStatus::Rejected)
95 /// also trigger rollback + replan, not just runtime Failed actions.
96 /// Default false (conservative: preserves prior abort-only-on-Failed behavior).
97 pub replan_on_rejected: bool,
98}
99
100/// How the runtime treats a proposal that conflicts with the current
101/// shared state on a pre-execution transactional check (survey §4.3/§5.2.4
102/// — `car_verify::check_transaction` against the versioned `StateStore`).
103#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
104pub enum TransactionCheckMode {
105 /// Don't run the check (default — preserves prior behavior exactly).
106 #[default]
107 Off,
108 /// Run it and emit any conflicts as `TransactionConflict` telemetry, but
109 /// execute anyway.
110 Warn,
111 /// Run it; if any conflict is found, reject the proposal without
112 /// executing (the conflicting actions become `Rejected` results).
113 Strict,
114}
115
116impl Default for ReplanConfig {
117 fn default() -> Self {
118 Self {
119 max_replans: 0,
120 delay_ms: 0,
121 verify_before_execute: true,
122 replan_on_rejected: false,
123 }
124 }
125}
126
127/// Trait for tool execution. Implement this to provide tools to the runtime.
128///
129/// In-process: implement directly with function calls.
130/// Daemon mode: implement by sending JSON-RPC to the client.
131#[async_trait::async_trait]
132pub trait ToolExecutor: Send + Sync {
133 async fn execute(&self, tool: &str, params: &Value) -> Result<Value, String>;
134
135 /// Variant that also carries the originating proposal `Action.id`
136 /// and its `timeout_ms` budget.
137 ///
138 /// `action_id`: WS-based executors (`car-server-core::WsToolExecutor`)
139 /// use it so the daemon-initiated `tools.execute` request to the client
140 /// carries the same id the host's process-wide handler is keyed on
141 /// — without this round-trip the host can't disambiguate concurrent
142 /// callbacks for the same tool (Parslee-ai/car-releases#43 follow-up).
143 ///
144 /// `timeout_ms`: the action's per-call budget. WS executors MUST bound
145 /// their callback wait by this (falling back to a default when `None`)
146 /// so the daemon→host wait and the executor's own action deadline stay
147 /// coordinated — otherwise a hardcoded inner wait reaps a call the outer
148 /// deadline still permits (Parslee-ai/car#259). In-process executors
149 /// that don't need either can keep the default forward to [`execute`](crate::executor::ToolExecutor::execute).
150 async fn execute_with_action(
151 &self,
152 tool: &str,
153 params: &Value,
154 _action_id: &str,
155 _timeout_ms: Option<u64>,
156 ) -> Result<Value, String> {
157 self.execute(tool, params).await
158 }
159
160 /// Variant that also carries the Runtime execution session and the retry
161 /// attempt. Executors that retain safety-relevant state across calls (such
162 /// as a read-before-edit ledger) override this to keep that state isolated
163 /// by session. Existing executors retain their current behavior through the
164 /// default delegation.
165 ///
166 /// `attempt` is **1-based**: `1` on the first try, `2` on the first retry.
167 /// WS executors surface it on the `tools.execute` payload so a host can
168 /// tell which retry it is serving — `action_id` cannot, being neither
169 /// unique across attempts nor varying between them.
170 ///
171 /// It is threaded from `execute_with_retry`'s own counter rather than
172 /// synthesized here. It was previously hardcoded to `1` at the one place
173 /// that put it on the wire, which made the field a constant and any join
174 /// built on it silently degenerate (Parslee-ai/car#928).
175 async fn execute_with_action_in_session(
176 &self,
177 tool: &str,
178 params: &Value,
179 action_id: &str,
180 timeout_ms: Option<u64>,
181 _session_id: Option<&str>,
182 _attempt: u32,
183 ) -> Result<Value, String> {
184 self.execute_with_action(tool, params, action_id, timeout_ms)
185 .await
186 }
187
188 /// Execute one action while allowing a transport to return runtime-observed
189 /// state mutations alongside its ordinary tool output.
190 ///
191 /// The default preserves every direct/internal executor: its existing
192 /// output is wrapped with no mutations, and declared `expected_effects`
193 /// remain assertions rather than writes. A transport that overrides this
194 /// method must validate its mutation envelope against `expected_effects`
195 /// before returning. The Runtime repeats the key-set and I-JSON checks and
196 /// applies accepted changes through its own [`StateStore`].
197 async fn execute_with_action_state_in_session(
198 &self,
199 tool: &str,
200 params: &Value,
201 action_id: &str,
202 timeout_ms: Option<u64>,
203 session_id: Option<&str>,
204 attempt: u32,
205 _expected_effects: &HashMap<String, Value>,
206 _return_schema: Option<&Value>,
207 ) -> Result<ToolExecution, String> {
208 self.execute_with_action_in_session(
209 tool, params, action_id, timeout_ms, session_id, attempt,
210 )
211 .await
212 .map(ToolExecution::output_only)
213 }
214
215 /// Execute one action with typed failure evidence.
216 ///
217 /// This is a new default-delegating layer rather than a signature change
218 /// to the existing execution methods. Existing executors therefore retain
219 /// their exact behavior: every legacy string error becomes an ordinary,
220 /// non-terminal failure. Executors opt into fail-stop behavior only by
221 /// overriding this method and returning [`ToolFailure::terminal`].
222 async fn execute_classified(
223 &self,
224 tool: &str,
225 params: &Value,
226 action_id: &str,
227 timeout_ms: Option<u64>,
228 session_id: Option<&str>,
229 attempt: u32,
230 expected_effects: &HashMap<String, Value>,
231 return_schema: Option<&Value>,
232 ) -> Result<ToolExecution, ToolFailure> {
233 self.execute_with_action_state_in_session(
234 tool,
235 params,
236 action_id,
237 timeout_ms,
238 session_id,
239 attempt,
240 expected_effects,
241 return_schema,
242 )
243 .await
244 .map_err(ToolFailure::from)
245 }
246
247 /// Streaming entry point for detached invocation modes (C2). Start
248 /// the tool and return a channel of [`car_ir::ToolStreamChunk`]s; the
249 /// runtime drains it into the per-runtime handle registry while the
250 /// DAG proceeds. End the stream with a terminal chunk (`done` /
251 /// `error`); dropping the sender without one is reported as failure.
252 /// A cooperative executor should stop work when the receiver returned
253 /// here is dropped (that's what cancellation looks like from its side).
254 ///
255 /// Default: unsupported — existing one-shot executors compile and
256 /// behave unchanged; a detached action against them is rejected with
257 /// this error.
258 async fn execute_stream(
259 &self,
260 tool: &str,
261 _params: &Value,
262 _action_id: &str,
263 ) -> Result<tokio::sync::mpsc::Receiver<car_ir::ToolStreamChunk>, String> {
264 Err(format!(
265 "tool '{tool}': this executor does not support streaming/long-running invocation"
266 ))
267 }
268}
269
270/// Result of a tool dispatch before the Runtime commits observed state.
271#[derive(Debug, Clone, PartialEq)]
272pub struct ToolExecution {
273 pub output: Value,
274 pub state_changes: HashMap<String, Value>,
275}
276
277impl ToolExecution {
278 pub fn output_only(output: Value) -> Self {
279 Self {
280 output,
281 state_changes: HashMap::new(),
282 }
283 }
284}
285
286/// Deterministic key for idempotency deduplication.
287/// Deterministic key for idempotency deduplication. Carries the tenant
288/// dimension (linus review): without it, tenant A's cached result for
289/// an identical idempotent action was served to tenant B — a
290/// cross-tenant data leak through the dedup cache. Unscoped executions
291/// keep their historical keys (empty tenant segment).
292fn idempotency_key(action: &Action, scope: Option<&crate::scope::RuntimeScope>) -> String {
293 let sorted: std::collections::BTreeMap<_, _> = action.parameters.iter().collect();
294 let params = serde_json::to_string(&sorted).unwrap_or_default();
295 let tenant = scope.and_then(|s| s.tenant_id.as_deref()).unwrap_or("");
296 format!(
297 "{}:{}:{}:{}",
298 tenant,
299 serde_json::to_string(&action.action_type).unwrap_or_default(),
300 action.tool.as_deref().unwrap_or(""),
301 params
302 )
303}
304
305/// Exact normal-serde proposal preimage plus its lowercase RFC 8785/JCS
306/// SHA-256 identity. The shared inference helper is the repository's settled
307/// canonicalizer; journal identity must not drift onto a second JSON sorter.
308fn proposal_journal_identity(proposal: &ActionProposal) -> Result<(Value, String), String> {
309 let preimage = serde_json::to_value(proposal)
310 .map_err(|error| format!("proposal serialization failed: {error}"))?;
311 let canonical = car_inference::catalog_identity::canonical_json(&preimage)?;
312 let digest = format!("{:x}", Sha256::digest(canonical.as_bytes()));
313 Ok((preimage, digest))
314}
315
316fn proposal_digest(proposal: &ActionProposal) -> Result<String, String> {
317 proposal_journal_identity(proposal).map(|(_, digest)| digest)
318}
319
320fn proposal_rejection_boundary_data(
321 proposal: &ActionProposal,
322 proposal_digest: Option<&str>,
323 reason: &str,
324) -> HashMap<String, Value> {
325 let mut data: HashMap<String, Value> = [
326 ("affected_actions".to_string(), Value::Array(Vec::new())),
327 (
328 "rolled_back_changes".to_string(),
329 Value::Object(serde_json::Map::new()),
330 ),
331 (
332 "changes_semantics".to_string(),
333 Value::from("proposal_rejected_no_state_commit"),
334 ),
335 ("attempted".to_string(), Value::from(false)),
336 ("stage".to_string(), Value::from("proposal_rejection")),
337 ("rejection_reason".to_string(), Value::from(reason)),
338 ]
339 .into();
340 if let Ok(preimage) = serde_json::to_value(proposal) {
341 data.insert("proposal".to_string(), preimage);
342 }
343 if let Some(digest) = proposal_digest {
344 data.insert("proposal_digest".to_string(), Value::from(digest));
345 }
346 data
347}
348
349fn proposal_lineage_entry(
350 proposal: &ActionProposal,
351 generation: u32,
352 status: ProposalLineageStatus,
353 rejection_reason: Option<String>,
354) -> ProposalLineageEntry {
355 ProposalLineageEntry {
356 generation,
357 proposal_id: proposal.id.clone(),
358 proposal_digest: proposal_digest(proposal).ok(),
359 status,
360 rejection_reason,
361 }
362}
363
364fn finalize_proposal_result(
365 mut result: ProposalResult,
366 original_proposal_id: &str,
367 lineage: &[ProposalLineageEntry],
368 accepted_proposal_preimages: &[AcceptedProposalPreimage],
369) -> ProposalResult {
370 result.original_proposal_id = original_proposal_id.to_string();
371 result.replan_lineage = lineage.to_vec();
372 result.accepted_proposal_preimages = accepted_proposal_preimages.to_vec();
373 result
374}
375
376/// Validate the proposal-local action identity boundary used by DAG execution,
377/// action receipts, results, and state-transition attribution. A retry reuses
378/// one admitted action id across attempts; two declared actions may not share
379/// an id because every downstream join would become ambiguous.
380pub fn validate_proposal_action_ids(proposal: &ActionProposal) -> Result<(), String> {
381 let mut seen = std::collections::HashSet::with_capacity(proposal.actions.len());
382 for action in &proposal.actions {
383 if !seen.insert(action.id.as_str()) {
384 return Err(format!("duplicate action id '{}'", action.id));
385 }
386 }
387 Ok(())
388}
389
390fn validate_proposal_retry_limits(proposal: &ActionProposal) -> Result<(), String> {
391 for action in &proposal.actions {
392 if action.failure_behavior == FailureBehavior::Retry
393 && action.max_retries.checked_add(1).is_none()
394 {
395 return Err(format!(
396 "action '{}' retry attempt count overflows u32",
397 action.id
398 ));
399 }
400 }
401 Ok(())
402}
403
404fn requires_integrity_rollback(result: &ActionResult) -> bool {
405 result.status == ActionStatus::Failed
406 && result.error.as_deref().is_some_and(|error| {
407 error.starts_with("tool output failed JCS/I-JSON validation after dispatch")
408 })
409}
410
411fn record_rollback_durability_error(
412 results: &mut [ActionResult],
413 prefix: &str,
414 error: &str,
415) -> String {
416 let detail = format!("{prefix}: {error}");
417 if let Some(result) = results
418 .iter_mut()
419 .find(|result| result.status != ActionStatus::Succeeded)
420 {
421 result.error = Some(match result.error.take() {
422 Some(existing) => format!("{existing}; {detail}"),
423 None => detail.clone(),
424 });
425 }
426 detail
427}
428
429fn rejected_result(action_id: &str, error: String) -> ActionResult {
430 ActionResult {
431 action_id: action_id.to_string(),
432 status: ActionStatus::Rejected,
433 output: None,
434 error: Some(error),
435 terminal: false,
436 state_changes: HashMap::new(),
437 rolled_back: false,
438 duration_ms: None,
439 timestamp: chrono::Utc::now(),
440 }
441}
442
443/// Capture only the state keys relevant to an action (state_dependencies + expected_effects).
444/// Returns an empty map if the action declares no relevant keys (e.g., tool calls
445/// that don't interact with state).
446fn snapshot_relevant_keys(
447 state: &car_state::StateStore,
448 action: &Action,
449) -> HashMap<String, Value> {
450 let mut keys: std::collections::HashSet<&str> = std::collections::HashSet::new();
451 for dep in &action.state_dependencies {
452 keys.insert(dep.as_str());
453 }
454 for key in action.expected_effects.keys() {
455 keys.insert(key.as_str());
456 }
457 // For state_write actions, capture the key being written
458 if action.action_type == ActionType::StateWrite {
459 if let Some(key) = action.parameters.get("key").and_then(|v| v.as_str()) {
460 keys.insert(key);
461 }
462 }
463
464 if keys.is_empty() {
465 // No declared state interaction — skip snapshot (empty map)
466 return HashMap::new();
467 }
468
469 keys.iter()
470 .filter_map(|&k| state.get(k).map(|v| (k.to_string(), v)))
471 .collect()
472}
473
474fn skipped_result(action_id: &str, reason: &str) -> ActionResult {
475 ActionResult {
476 action_id: action_id.to_string(),
477 status: ActionStatus::Skipped,
478 output: None,
479 error: Some(reason.to_string()),
480 terminal: false,
481 state_changes: HashMap::new(),
482 rolled_back: false,
483 duration_ms: None,
484 timestamp: chrono::Utc::now(),
485 }
486}
487
488fn insert_tool_event_provenance(
489 data: &mut HashMap<String, Value>,
490 action: &Action,
491 source: Option<car_ir::ToolSourceKind>,
492) {
493 if let Some(tool) = action.tool.as_deref() {
494 data.insert("tool".to_string(), Value::from(tool));
495 data.insert(
496 "tool_source".to_string(),
497 Value::from(
498 source
499 .unwrap_or(car_ir::ToolSourceKind::UserDefined)
500 .as_str(),
501 ),
502 );
503 }
504}
505
506/// SHA-256 over the RFC 8785/JCS rendering of only the action parameters.
507///
508/// The accepted proposal has already crossed `proposal_journal_identity`, so
509/// canonicalization cannot fail here without violating that admission
510/// invariant. Keeping only the digest on outcome events lets a later join find
511/// the raw parameters in `ProposalReceived` without copying them into every
512/// failure record.
513fn action_params_digest(action: &Action) -> String {
514 // Serialize the map directly: RFC 8785/JCS sorts object keys
515 // recursively at render time (see `canonical_json` and the
516 // `canonical_json_sorts_object_keys_recursively` test), so the map's
517 // iteration order does not affect the digest and cloning into a
518 // `Value::Object` would be pure per-action allocation overhead.
519 let canonical = car_inference::catalog_identity::canonical_json(&action.parameters)
520 .expect("accepted action parameters must remain RFC 8785/JCS canonicalizable");
521 format!("{:x}", Sha256::digest(canonical.as_bytes()))
522}
523
524fn insert_action_outcome_signal(
525 data: &mut HashMap<String, Value>,
526 action: &Action,
527 params_digest: &str,
528 error_class: Option<&str>,
529) {
530 data.insert(
531 "params_digest".to_string(),
532 Value::from(params_digest.to_string()),
533 );
534 data.insert(
535 "expected_effects".to_string(),
536 serde_json::to_value(&action.expected_effects)
537 .expect("action expected_effects must serialize as JSON"),
538 );
539 if let Some(error_class) = error_class {
540 data.insert("error_class".to_string(), Value::from(error_class));
541 }
542}
543
544/// Normalize execution failures into a stable, deliberately low-cardinality
545/// event-log vocabulary. Validation and policy checks that happen before
546/// dispatch remain `ActionRejected`; this maps failures reached after an
547/// action began executing.
548pub(crate) fn action_error_class(
549 action: &Action,
550 error: &str,
551 engine_timeout: bool,
552) -> &'static str {
553 // Classification is by substring over prose and is a published wire
554 // contract (docs/agent-ir-spec.md `error_class`). Each literal is pinned
555 // to its producer below; changing a producer's wording must update the
556 // matching literal (and `action_failure_error_class_mapping_is_stable`).
557 let error = error.to_ascii_lowercase();
558 // "callback timed out" — the engine deadline itself is typed via
559 // `engine_timeout`, so this literal only covers the detached-dispatch
560 // producer `tool '{tool}' callback timed out ({s}s)`
561 // (car-server-core/src/session.rs, `reap_detached_calls`/wait path).
562 // A tool faithfully relaying a *downstream* timeout in its own message
563 // also lands here; accepted drift, low-cardinality vocabulary.
564 if engine_timeout || error.contains("callback timed out") {
565 "timeout"
566 } else if error.starts_with("denied by policy:") || error.starts_with("rejected by policy:") {
567 // Exact producers return this prefix unwrapped from the tool's
568 // guard check: car-server-core/src/assistant/executor.rs and
569 // car-server-core/src/coder/shell_tool.rs
570 // (`format!("denied by policy: {reason}")`). An executor that
571 // *wraps* the denial (e.g. `tool 'x' failed: denied by policy: …`)
572 // deliberately falls through to `tool_error` — see the wrapped
573 // case in `action_failure_error_class_mapping_is_stable`.
574 "rejected_by_policy"
575 } else if error.contains("jcs/i-json validation")
576 || error.contains("output validation:")
577 || error.contains("invalid return json schema")
578 || error.contains("must return exact envelope {output,state_changes}")
579 || error.contains("state_changes must be an object")
580 || error.contains("state_changes keys do not match")
581 || error.contains("state_changes serialization failed")
582 {
583 "validation"
584 } else if action.tool.is_some() {
585 "tool_error"
586 } else {
587 "unknown"
588 }
589}
590
591fn action_outcome_data(
592 stage: &str,
593 reason: &str,
594 attempted: bool,
595 attempt: Option<u32>,
596) -> HashMap<String, Value> {
597 let mut data: HashMap<String, Value> = [
598 ("stage".to_string(), Value::from(stage)),
599 ("reason".to_string(), Value::from(reason)),
600 ("attempted".to_string(), Value::from(attempted)),
601 ]
602 .into();
603 if let Some(attempt) = attempt {
604 data.insert("attempt".to_string(), Value::from(attempt));
605 }
606 data
607}
608
609/// Prefix on the `error` field of an `ActionResult` that distinguishes
610/// "the user pulled the plug" from "earlier abort cascaded." The
611/// `ActionStatus` itself is `Skipped` in both cases (introducing a
612/// new variant ripples through every IR consumer + FFI binding); the
613/// prefix lets callers like the A2A bridge tell the cases apart
614/// without string-matching a magic literal.
615pub const CANCELED_PREFIX: &str = "canceled: ";
616
617/// Result for an action that didn't run because the proposal was
618/// cancelled mid-flight. Distinct from `Skipped` so callers can tell
619/// "didn't run because of an earlier abort" from "didn't run because
620/// the user pulled the plug."
621fn canceled_result(action_id: &str, reason: &str) -> ActionResult {
622 ActionResult {
623 action_id: action_id.to_string(),
624 status: ActionStatus::Skipped,
625 output: None,
626 error: Some(format!("{}{}", CANCELED_PREFIX, reason)),
627 terminal: false,
628 state_changes: HashMap::new(),
629 rolled_back: false,
630 duration_ms: None,
631 timestamp: chrono::Utc::now(),
632 }
633}
634
635/// Format a tool result for feeding back to a model.
636pub fn format_tool_result(result: &ActionResult) -> String {
637 match result.status {
638 ActionStatus::Succeeded => match &result.output {
639 Some(v) => serde_json::to_string(v).unwrap_or_else(|_| v.to_string()),
640 None => String::new(),
641 },
642 ActionStatus::Rejected => format!("[REJECTED] {}", result.error.as_deref().unwrap_or("")),
643 ActionStatus::Failed => format!("[FAILED] {}", result.error.as_deref().unwrap_or("")),
644 _ => format!(
645 "[{:?}] {}",
646 result.status,
647 result.error.as_deref().unwrap_or("")
648 ),
649 }
650}
651
652/// Budget constraints for proposal execution.
653#[derive(Debug, Clone)]
654pub struct CostBudget {
655 pub max_tool_calls: Option<u32>,
656 pub max_duration_ms: Option<f64>,
657 pub max_actions: Option<u32>,
658}
659
660/// Common Agent Runtime — deterministic execution layer.
661///
662/// Lock ordering discipline (never hold multiple simultaneously, never hold sync locks across .await):
663/// 1. capabilities (RwLock, read-only during execution)
664/// 2. tools (RwLock, read-only during execution)
665/// 3. policies (RwLock, read-only during execution)
666/// 4. session_policies (RwLock to find the per-session engine, then read inner Arc)
667/// 5. cost_budget (RwLock, read-only during execution)
668/// 6. log (TokioMutex, acquired/released per event)
669/// 7. tool_executor (TokioMutex, clone Arc and drop before await)
670/// 8. idempotency_cache (TokioMutex, acquired/released per check)
671///
672/// StateStore uses parking_lot::Mutex (sync) — NEVER hold across .await points.
673pub struct Runtime {
674 pub state: Arc<StateStore>,
675 pub tools: Arc<TokioRwLock<HashMap<String, ToolSchema>>>,
676 pub policies: Arc<TokioRwLock<PolicyEngine>>,
677 /// Per-session policy registries. Hosts that multiplex multiple
678 /// concurrent agent sessions over a single `Runtime` (IDE-style
679 /// frontends with per-project rules, multi-tenant servers) call
680 /// [`Runtime::open_session`] to mint an id, then
681 /// [`Runtime::register_policy_in_session`] to attach session-scoped
682 /// rules. Validation under that session walks the global registry
683 /// AND the session's — both must pass. Sessions can deny what
684 /// global allows; sessions cannot allow what global denies.
685 /// Closing a session drops its registry and any closures it holds.
686 /// See `docs/proposals/per-session-policy-scoping.md`.
687 pub session_policies: Arc<TokioRwLock<HashMap<String, Arc<TokioRwLock<PolicyEngine>>>>>,
688 pub log: Arc<TokioMutex<EventLog>>,
689 pub rate_limiter: Arc<RateLimiter>,
690 pub result_cache: Arc<ResultCache>,
691 /// Per-execution-session read ledgers backing the read-before-edit / staleness guard on
692 /// the built-in file tools (H1/F4-remainder, audit 2026-07-06). Threaded into
693 /// `agent_basics::execute_with_ledger` on the built-in fallback path so a raw
694 /// Runtime session (no configured executor) still requires reading a file
695 /// before editing or overwriting it. Configured executors
696 /// (coder/assistant/bench) own their own ledger, so this one only governs the
697 /// built-in fallback path.
698 read_ledgers: crate::agent_basics::SessionReadLedgers,
699 tool_executor: TokioMutex<Option<Arc<dyn ToolExecutor>>>,
700 idempotency_cache: TokioMutex<HashMap<String, ActionResult>>,
701 cost_budget: TokioRwLock<Option<CostBudget>>,
702 capabilities: TokioRwLock<Option<CapabilitySet>>,
703 inference_engine: Option<Arc<car_inference::InferenceEngine>>,
704 /// Optional outbound transport backing the `messaging.send` built-in.
705 /// `None` by default — a runtime with no sink refuses `messaging.send`
706 /// outright rather than falling through to the host executor, and never
707 /// advertises the tool. Attach one with
708 /// [`Runtime::with_message_sink`]; see [`crate::messaging`] for why
709 /// human-directed messaging belongs inside the governed chain at all.
710 message_sink: Option<Arc<dyn crate::messaging::MessageSink>>,
711 /// Optional memgine for skill learning (auto-distillation after execution).
712 memgine: Option<Arc<TokioMutex<car_memgine::MemgineEngine>>>,
713 /// Whether to auto-distill skills after each proposal execution.
714 auto_distill: bool,
715 /// Optional trajectory store for persisting execution traces.
716 trajectory_store: Option<Arc<car_memgine::TrajectoryStore>>,
717 /// Optional replan callback for failure recovery.
718 replan_callback: TokioMutex<Option<Arc<dyn ReplanCallback>>>,
719 /// Replan configuration.
720 replan_config: TokioRwLock<ReplanConfig>,
721 /// Pre-execution transactional conflict check mode (survey §4.3/§5.2.4).
722 /// `Off` by default. When `Warn`/`Strict`, each proposal is checked
723 /// against the versioned shared state before execution; `Strict`
724 /// rejects on conflict. See [`Runtime::set_transaction_check_mode`].
725 transaction_check: TokioRwLock<TransactionCheckMode>,
726 /// The live harness operating config the Evolution Agent tunes (survey
727 /// §3.5). `None` until one is installed via [`Runtime::set_harness_config`]
728 /// — so default behavior is byte-identical to before (no cap on
729 /// per-action retries, built-in backoff). Once installed, the runtime
730 /// *reads* it: `max_retries` caps per-action retry budgets and
731 /// `retry_backoff_ms` sets the inter-attempt delay; the setter also maps
732 /// `planning_max_replans` onto the replan config. This is what makes an
733 /// applied [`car_memgine::HarnessConfigPatch`] take effect.
734 harness_config: TokioRwLock<Option<car_memgine::HarnessConfig>>,
735 /// Canonical tool registry (optional — new code should use this).
736 pub registry: Arc<crate::registry::ToolRegistry>,
737 /// The environment the agent's side-effecting built-in tools act within.
738 /// Defaults to [`crate::substrate::LocalSubstrate`] (host fs/process), which
739 /// reproduces the historic `agent_basics` host behavior byte-for-byte.
740 /// Bind a different environment (e.g. a VM via `McpSubstrate`) with
741 /// [`Runtime::with_substrate`] / [`Runtime::set_substrate`]. `calculate`
742 /// stays pure and never consults the substrate.
743 substrate: TokioRwLock<Arc<dyn crate::substrate::Substrate>>,
744 /// Proposal-admission gates — the pre-execution safety seam (EPIC A /
745 /// task A1). Each registered [`crate::admission::AdmissionGate`] runs
746 /// during admission, before any action executes; a proposal that any
747 /// gate blocks (or escalates to approval) is refused. Empty by default,
748 /// so a runtime that registers no gates behaves exactly as before.
749 /// Individual gates (information-flow, concurrency, blocking-policy)
750 /// are layered on via [`Runtime::register_admission_gate`].
751 admission_gates: TokioRwLock<Vec<Arc<dyn crate::admission::AdmissionGate>>>,
752 /// Runtime taint provenance for the VIGIL intent gate
753 /// ([`crate::taint::TaintLedger`]). Installed by
754 /// [`Runtime::install_intent_gate`] and by nothing else: with no intent
755 /// gate there is no ledger, so a runtime that never configures VIGIL
756 /// pays no per-action cost and behaves exactly as before. When present,
757 /// each successful action records which state keys its result wrote and
758 /// whether that result was tainted, and the gate reads it back at
759 /// admission to mark actions that READ a tainted key — the cross-
760 /// proposal (replan) and trusted-tool-laundering cases the proposal-
761 /// local dependency DAG cannot see.
762 taint_ledger: TokioRwLock<Option<Arc<crate::taint::TaintLedger>>>,
763 /// Durable human-in-the-loop approval ledger (EPIC A / A7). When set,
764 /// an admission gate's `NeedsApproval` verdict is resolved against this
765 /// ledger by fingerprint: a prior Approved decision admits the
766 /// proposal, a Rejected decision blocks it, an unseen one stays pending
767 /// (fail-closed). `None` by default — escalations fail closed with an
768 /// explanatory reason until a ledger is installed.
769 approval_ledger: TokioRwLock<Option<ApprovalLedger>>,
770 /// Optional JSONL journal backing the idempotency cache (EPIC A / C3).
771 /// `None` by default — the cache is in-memory and lost on restart. When
772 /// set via [`Runtime::set_idempotency_cache_path`], idempotent results
773 /// are persisted and reloaded so a crash-restart doesn't re-execute a
774 /// completed idempotent action (avoiding duplicate side effects).
775 idempotency_journal: TokioRwLock<Option<std::path::PathBuf>>,
776 /// Detached tool invocations (C2): a `ToolCall` with a
777 /// `streaming`/`long_running` invocation mode is registered here and
778 /// its handle returned as the action's output while the DAG proceeds.
779 /// Chunks are drained via [`Runtime::tool_poll`], cancelled via
780 /// [`Runtime::tool_cancel`], and fanned out to
781 /// [`Runtime::subscribe_tool_events`] subscribers.
782 pub tool_handles: Arc<crate::tool_handles::ToolHandleRegistry>,
783}
784
785/// A durable idempotency-cache record (C3). `result: None` is a tombstone
786/// recorded when a cached entry is invalidated by a rollback.
787#[derive(serde::Serialize, serde::Deserialize)]
788struct IdempotencyEntry {
789 key: String,
790 result: Option<ActionResult>,
791}
792
793impl Runtime {
794 pub fn new() -> Self {
795 Self {
796 state: Arc::new(StateStore::new()),
797 tools: Arc::new(TokioRwLock::new(HashMap::new())),
798 policies: Arc::new(TokioRwLock::new(PolicyEngine::new())),
799 session_policies: Arc::new(TokioRwLock::new(HashMap::new())),
800 log: Arc::new(TokioMutex::new(EventLog::new())),
801 rate_limiter: Arc::new(RateLimiter::new()),
802 result_cache: Arc::new(ResultCache::new()),
803 read_ledgers: crate::agent_basics::SessionReadLedgers::new(),
804 tool_executor: TokioMutex::new(None),
805 idempotency_cache: TokioMutex::new(HashMap::new()),
806 cost_budget: TokioRwLock::new(None),
807 capabilities: TokioRwLock::new(None),
808 inference_engine: None,
809 message_sink: None,
810 memgine: None,
811 auto_distill: false,
812 trajectory_store: None,
813 replan_callback: TokioMutex::new(None),
814 replan_config: TokioRwLock::new(ReplanConfig::default()),
815 transaction_check: TokioRwLock::new(TransactionCheckMode::Off),
816 harness_config: TokioRwLock::new(None),
817 registry: Arc::new(crate::registry::ToolRegistry::new()),
818 substrate: TokioRwLock::new(Arc::new(crate::substrate::LocalSubstrate::new())),
819 admission_gates: TokioRwLock::new(Vec::new()),
820 taint_ledger: TokioRwLock::new(None),
821 approval_ledger: TokioRwLock::new(None),
822 idempotency_journal: TokioRwLock::new(None),
823 tool_handles: Arc::new(crate::tool_handles::ToolHandleRegistry::new()),
824 }
825 }
826
827 /// Create a runtime with shared state, event log, and policies.
828 /// Each runtime gets its own tool set, executor, and idempotency cache.
829 pub fn with_shared(
830 state: Arc<StateStore>,
831 log: Arc<TokioMutex<EventLog>>,
832 policies: Arc<TokioRwLock<PolicyEngine>>,
833 ) -> Self {
834 Self {
835 state,
836 tools: Arc::new(TokioRwLock::new(HashMap::new())),
837 policies,
838 // Session-policy registries are per-runtime — sharing them
839 // across embedders that share global policies would defeat
840 // the isolation point. Hosts that genuinely want shared
841 // sessions should drive them through one shared Runtime.
842 session_policies: Arc::new(TokioRwLock::new(HashMap::new())),
843 log,
844 rate_limiter: Arc::new(RateLimiter::new()),
845 result_cache: Arc::new(ResultCache::new()),
846 read_ledgers: crate::agent_basics::SessionReadLedgers::new(),
847 tool_executor: TokioMutex::new(None),
848 idempotency_cache: TokioMutex::new(HashMap::new()),
849 cost_budget: TokioRwLock::new(None),
850 capabilities: TokioRwLock::new(None),
851 inference_engine: None,
852 message_sink: None,
853 memgine: None,
854 auto_distill: false,
855 trajectory_store: None,
856 replan_callback: TokioMutex::new(None),
857 replan_config: TokioRwLock::new(ReplanConfig::default()),
858 transaction_check: TokioRwLock::new(TransactionCheckMode::Off),
859 harness_config: TokioRwLock::new(None),
860 registry: Arc::new(crate::registry::ToolRegistry::new()),
861 substrate: TokioRwLock::new(Arc::new(crate::substrate::LocalSubstrate::new())),
862 admission_gates: TokioRwLock::new(Vec::new()),
863 taint_ledger: TokioRwLock::new(None),
864 approval_ledger: TokioRwLock::new(None),
865 idempotency_journal: TokioRwLock::new(None),
866 tool_handles: Arc::new(crate::tool_handles::ToolHandleRegistry::new()),
867 }
868 }
869
870 // ─── Session policy lifecycle ───────────────────────────────────
871 //
872 // Per-session policy scoping. Policies registered against a
873 // session apply to proposals executed under that session id (via
874 // [`Self::execute_with_session`] / [`Self::execute_with_session_and_cancel`]).
875 // Policies registered globally always apply, on top of any
876 // session-scoped layer. See `docs/proposals/per-session-policy-scoping.md`.
877
878 /// Mint a new session id and pre-register an empty policy engine
879 /// under it. Hosts call this once per concurrent agent context
880 /// (an IDE project window, a multi-tenant client, etc.) and pair
881 /// it with [`Self::close_session`] when the context ends.
882 ///
883 /// Returns the opaque id to pass to subsequent
884 /// [`Self::register_policy_in_session`] / [`Self::execute_with_session`]
885 /// calls. Ids are UUIDs so collisions across concurrent calls
886 /// don't matter.
887 pub async fn open_session(&self) -> String {
888 let id = Uuid::new_v4().to_string();
889 let mut sessions = self.session_policies.write().await;
890 sessions.insert(id.clone(), Arc::new(TokioRwLock::new(PolicyEngine::new())));
891 self.read_ledgers.ledger_for(Some(&id));
892 id
893 }
894
895 /// Drop the session and every policy scoped to it. Returns true
896 /// if a session by that id existed; false if it didn't (already
897 /// closed, never opened, etc.). Idempotent in effect — closing a
898 /// missing session is a no-op the caller is free to ignore.
899 pub async fn close_session(&self, session_id: &str) -> bool {
900 let mut sessions = self.session_policies.write().await;
901 let removed = sessions.remove(session_id).is_some();
902 if removed {
903 self.read_ledgers.remove(session_id);
904 }
905 removed
906 }
907
908 /// Register a policy under a specific session id. The policy
909 /// applies only when a proposal is executed under that session;
910 /// proposals executed without a session (the default) only see
911 /// global policies.
912 ///
913 /// Returns `Err(...)` if the session is unknown — callers either
914 /// forgot to call [`Self::open_session`] or are using a
915 /// stale/closed id.
916 pub async fn register_policy_in_session(
917 &self,
918 session_id: &str,
919 name: &str,
920 check: car_policy::PolicyCheck,
921 description: &str,
922 ) -> Result<(), String> {
923 let engine = {
924 let sessions = self.session_policies.read().await;
925 sessions
926 .get(session_id)
927 .cloned()
928 .ok_or_else(|| format!("unknown session id '{session_id}'"))?
929 };
930 let mut engine = engine.write().await;
931 engine.register(name, check, description);
932 Ok(())
933 }
934
935 /// [`Self::register_policy_in_session`] for a check that forbids `tool`
936 /// outright, so `PolicyEngine::blanket_denied_tools` can read it back.
937 /// Separate method rather than an extra parameter: the existing signature
938 /// is public and crosses the bindings, and this is purely additive.
939 pub async fn register_tool_deny_in_session(
940 &self,
941 session_id: &str,
942 name: &str,
943 tool: &str,
944 check: car_policy::PolicyCheck,
945 description: &str,
946 ) -> Result<(), String> {
947 let engine = {
948 let sessions = self.session_policies.read().await;
949 sessions
950 .get(session_id)
951 .cloned()
952 .ok_or_else(|| format!("unknown session id '{session_id}'"))?
953 };
954 let mut engine = engine.write().await;
955 engine.register_tool_deny(name, tool, check, description);
956 Ok(())
957 }
958
959 /// Remove a policy by name. `session_id` targets a session's policy set;
960 /// `None` targets the global one.
961 ///
962 /// Returns how many policies were dropped, or `Err` if `session_id` names a
963 /// session that doesn't exist. Session-scoped policies could always be
964 /// dropped wholesale by [`Self::close_session`], but a *global* policy had
965 /// no removal path at all — once registered it lived until the process
966 /// exited, so a mistyped or over-broad rule could only be cleared by
967 /// restarting the daemon (Parslee-ai/car#623).
968 pub async fn unregister_policy(
969 &self,
970 name: &str,
971 session_id: Option<&str>,
972 ) -> Result<usize, String> {
973 match session_id {
974 Some(sid) => {
975 let engine = {
976 let sessions = self.session_policies.read().await;
977 sessions
978 .get(sid)
979 .cloned()
980 .ok_or_else(|| format!("unknown session id '{sid}'"))?
981 };
982 let mut engine = engine.write().await;
983 Ok(engine.unregister(name))
984 }
985 None => {
986 let mut engine = self.policies.write().await;
987 Ok(engine.unregister(name))
988 }
989 }
990 }
991
992 /// Registered policies as `(name, description)`. `session_id` lists a
993 /// session's set; `None` lists the global one. Without this a client could
994 /// register a policy but never ask what was in force, so a rejection could
995 /// not be explained beyond its single message (Parslee-ai/car#623).
996 pub async fn list_policies(
997 &self,
998 session_id: Option<&str>,
999 ) -> Result<Vec<(String, String)>, String> {
1000 match session_id {
1001 Some(sid) => {
1002 let engine = {
1003 let sessions = self.session_policies.read().await;
1004 sessions
1005 .get(sid)
1006 .cloned()
1007 .ok_or_else(|| format!("unknown session id '{sid}'"))?
1008 };
1009 let engine = engine.read().await;
1010 Ok(engine.policy_details())
1011 }
1012 None => {
1013 let engine = self.policies.read().await;
1014 Ok(engine.policy_details())
1015 }
1016 }
1017 }
1018
1019 /// Load declarative deny rules from a project's `.car/policies/`
1020 /// directory and register them on the global policy engine (EPIC A /
1021 /// task A2).
1022 ///
1023 /// `car_dir` is a `.car` directory; this looks for
1024 /// `car_dir/policies/*.toml`. **The caller chooses the directory and
1025 /// there is no walk-up here** — this joins the path it is given, once.
1026 /// The two production callers pick differently: the daemon passes
1027 /// `$HOME/.car`, and the assistant passes its working directory's
1028 /// `.car` (`--dir`, else cwd). So a rule file at a repository root does
1029 /// not govern a `car do` run started from a subdirectory. Say which
1030 /// directory you mean at the call site rather than assuming discovery.
1031 ///
1032 /// A missing directory is not an error (returns 0). A malformed rule
1033 /// file *is* an error — a dropped security rule must surface loudly.
1034 /// Returns the number of rules registered.
1035 ///
1036 /// These rules are additive on top of any code-registered policies, and
1037 /// every one of them is a prohibition — though `allow_tool_param` states
1038 /// its prohibition as an allowlist, denying everything about its tool
1039 /// that it does not name. Once A9 makes policy violations blocking at
1040 /// admission, a matching action refuses the proposal.
1041 pub async fn load_project_policies(
1042 &self,
1043 car_dir: impl AsRef<std::path::Path>,
1044 ) -> Result<usize, car_policy::PolicyLoadError> {
1045 let dir = car_dir.as_ref().join("policies");
1046 let rules = car_policy::load_policy_dir(&dir)?;
1047 // `PolicyRules::len` counts every kind. Summing a hand-picked subset
1048 // here is what made this under-report once already — it covered the
1049 // three kinds that existed at the time and was not revisited when
1050 // more landed, so a file full of allowlists reported zero rules
1051 // loaded while `apply` registered all of them.
1052 let count = rules.len();
1053 let mut engine = self.policies.write().await;
1054 rules.apply(&mut engine);
1055 Ok(count)
1056 }
1057
1058 /// Load information-flow tool labels from a project's `.car` directory
1059 /// and register the information-flow admission gate (EPIC A / A3+A4).
1060 ///
1061 /// Reads `car_dir/tool-labels.json` (merged over built-in defaults) and
1062 /// registers an [`crate::flow::InformationFlowGate`] so every admitted
1063 /// proposal is checked for data exfiltration (blocked) and forbidden
1064 /// tool orderings (escalated to approval). A missing labels file is
1065 /// fine — the built-in defaults still mark the network tools as sinks.
1066 /// A malformed file is a loud error.
1067 pub async fn install_information_flow_gate(
1068 &self,
1069 car_dir: impl AsRef<std::path::Path>,
1070 ) -> Result<(), crate::flow::FlowLoadError> {
1071 let config = crate::flow::load_tool_labels(car_dir)?;
1072 let gate = Arc::new(crate::flow::InformationFlowGate::new(config));
1073 self.register_admission_gate(gate).await;
1074 Ok(())
1075 }
1076
1077 /// True if a session with this id is currently open. Mostly for
1078 /// tests and FFI surface validation — production code should
1079 /// trust the id it just opened.
1080 pub async fn session_exists(&self, session_id: &str) -> bool {
1081 self.session_policies.read().await.contains_key(session_id)
1082 }
1083
1084 /// Attach a local inference engine. Registers `infer`, `embed`, `classify`
1085 /// as built-in tools with real implementations.
1086 pub fn with_inference(mut self, engine: Arc<car_inference::InferenceEngine>) -> Self {
1087 self.inference_engine = Some(engine);
1088 // Register inference tool schemas (non-async init, use try_lock).
1089 // Keep the canonical registry populated too: execution-event provenance
1090 // is read from ToolEntry.source rather than inferred from a tool name.
1091 //
1092 // Contention contract: this runs at construction time on a private
1093 // runtime, so both try_locks are expected to succeed. If either fails
1094 // the two stores would disagree (validator reads `tools`, provenance
1095 // reads the registry first) — loud in debug, logged in release.
1096 if let Ok(mut tools) = self.tools.try_write() {
1097 for schema in car_inference::service::all_schemas() {
1098 let name = schema.name.clone();
1099 let entry = crate::registry::ToolEntry::builtin(schema);
1100 tools.insert(name.clone(), entry.schema.clone());
1101 if !self.registry.try_register(entry) {
1102 tracing::warn!(
1103 tool = %name,
1104 "with_inference: canonical registry contention; \
1105 schema map registered the inference tool but the \
1106 registry did not — provenance falls back to the map"
1107 );
1108 }
1109 }
1110 } else {
1111 debug_assert!(
1112 false,
1113 "with_inference must populate both stores; schema-map lock was contended \
1114 at construction time — inference tools stay unregistered"
1115 );
1116 tracing::warn!(
1117 "with_inference: schema map lock contended at construction; \
1118 inference tools were NOT registered"
1119 );
1120 }
1121 self
1122 }
1123
1124 /// Attach an outbound message sink, making `messaging.send` a real tool on
1125 /// this runtime.
1126 ///
1127 /// Registering the schema here — and ONLY here — is deliberate. A tool the
1128 /// runtime cannot execute must not be advertised to the model: without a
1129 /// sink the dispatch arm can only return an error, and a model that keeps
1130 /// seeing "message the human" in its tool list will keep trying to use it.
1131 /// So sink and schema arrive together, and neither exists alone.
1132 ///
1133 /// The entry is `AskUser` with side effects: reaching a human is
1134 /// irreversible, so the conservative default is the right one, and hosts
1135 /// that have their own consent model can re-register with a different
1136 /// permission. `category = "messaging"` groups it for the capability
1137 /// surfaces that filter by category.
1138 ///
1139 /// Follows [`Self::with_inference`]'s non-async registration pattern
1140 /// (`try_write` during construction, while the locks are uncontended) —
1141 /// making the builders async would break every `Runtime::new().with_…()`
1142 /// chain in the workspace for no gain.
1143 pub fn with_message_sink(mut self, sink: Arc<dyn crate::messaging::MessageSink>) -> Self {
1144 self.message_sink = Some(sink);
1145
1146 let entry = crate::registry::ToolEntry::builtin(car_ir::builtins::messaging_send())
1147 .with_permission(crate::registry::ToolPermission::AskUser)
1148 .with_side_effects(true)
1149 .with_category("messaging");
1150 let schema = entry.schema.clone();
1151
1152 // Honour the schema's declared rate limit, the way the async
1153 // `register_tool_schema` path does — otherwise the backstop on
1154 // outbound human messaging would silently not exist.
1155 if let Some(ref rl) = schema.rate_limit {
1156 self.rate_limiter.try_set_limit(
1157 &schema.name,
1158 RateLimit {
1159 max_calls: rl.max_calls,
1160 interval_secs: rl.interval_secs,
1161 },
1162 );
1163 }
1164 self.registry.try_register(entry);
1165 // The legacy schema map is what `validate_action` reads, so the
1166 // validator can only check `messaging.send` parameters once it lands
1167 // here.
1168 if let Ok(mut tools) = self.tools.try_write() {
1169 tools.insert(schema.name.clone(), schema);
1170 }
1171 self
1172 }
1173
1174 /// Attach a memgine for automatic skill learning after execution.
1175 /// When `auto_distill` is true, execution traces are automatically distilled
1176 /// into skills and domains are evolved when underperforming.
1177 pub fn with_learning(
1178 mut self,
1179 memgine: Arc<TokioMutex<car_memgine::MemgineEngine>>,
1180 auto_distill: bool,
1181 ) -> Self {
1182 self.memgine = Some(memgine);
1183 self.auto_distill = auto_distill;
1184 self
1185 }
1186
1187 /// Attach a memgine with auto-distillation enabled (recommended default).
1188 pub fn with_memgine(self, memgine: Arc<TokioMutex<car_memgine::MemgineEngine>>) -> Self {
1189 self.with_learning(memgine, true)
1190 }
1191
1192 /// Attach a trajectory store for persisting execution traces.
1193 pub fn with_trajectory_store(mut self, store: Arc<car_memgine::TrajectoryStore>) -> Self {
1194 self.trajectory_store = Some(store);
1195 self
1196 }
1197
1198 /// The attached trajectory store, if any.
1199 ///
1200 /// Writing traces was always the point of the store; reading them back is
1201 /// what makes them a feedback signal rather than an audit log.
1202 pub fn trajectory_store(&self) -> Option<&Arc<car_memgine::TrajectoryStore>> {
1203 self.trajectory_store.as_ref()
1204 }
1205
1206 /// Per-tool success rates observed over the last `window_days`, or `None`
1207 /// when no trajectory store is attached.
1208 ///
1209 /// Dispatch-conditional (see
1210 /// [`ToolFeedback::dispatched_from_trajectories`](car_planner::ToolFeedback::dispatched_from_trajectories))
1211 /// — rejected and skipped actions are excluded, because the tool never ran
1212 /// and a consumer that models rejection separately would otherwise count
1213 /// it twice.
1214 ///
1215 /// The window exists because a success rate is a claim about how a tool
1216 /// behaves *now*. An API that was broken for a week six months ago and has
1217 /// been fixed since should not still be dragging its own rate down, and an
1218 /// unbounded history makes the rate progressively less responsive to
1219 /// exactly the recent change an operator is trying to see. 30 days is the
1220 /// suggested default at the call sites.
1221 ///
1222 /// Reads from disk on every call — parsing is bounded by the window (day
1223 /// files outside it are skipped by filename, never opened), and the callers
1224 /// are interactive rather than hot-path. If that stops being true, cache
1225 /// here rather than at each call site.
1226 pub fn tool_feedback(&self, window_days: u32) -> Option<car_planner::ToolFeedback> {
1227 let store = self.trajectory_store.as_ref()?;
1228 let cutoff = chrono::Utc::now() - chrono::Duration::days(i64::from(window_days));
1229 let trajectories = store.load_since(cutoff);
1230 Some(car_planner::ToolFeedback::dispatched_from_trajectories(
1231 &trajectories,
1232 ))
1233 }
1234
1235 pub fn with_executor(self, executor: Arc<dyn ToolExecutor>) -> Self {
1236 // Use try_lock for non-async init context. Safe because we just created the mutex.
1237 if let Ok(mut guard) = self.tool_executor.try_lock() {
1238 *guard = Some(executor);
1239 }
1240 self
1241 }
1242
1243 /// Set a tool executor for the next execute() call.
1244 /// Used by NAPI bindings where executor varies per call.
1245 pub async fn set_executor(&self, executor: Arc<dyn ToolExecutor>) {
1246 *self.tool_executor.lock().await = Some(executor);
1247 }
1248
1249 /// Bind the execution substrate the side-effecting built-in tools
1250 /// (`read_file`/`write_file`/`edit_file`/`list_dir`/`find_files`/
1251 /// `grep_files`) act within (builder). Defaults to
1252 /// [`crate::substrate::LocalSubstrate`]; bind e.g.
1253 /// [`crate::substrate::McpSubstrate`] to make those tools hit a VM.
1254 /// `calculate` stays pure and ignores the substrate.
1255 pub fn with_substrate(self, substrate: Arc<dyn crate::substrate::Substrate>) -> Self {
1256 // Use try_write for non-async init context. Safe because we just created the lock.
1257 if let Ok(mut guard) = self.substrate.try_write() {
1258 *guard = substrate;
1259 }
1260 self
1261 }
1262
1263 /// Set the execution substrate at runtime.
1264 pub async fn set_substrate(&self, substrate: Arc<dyn crate::substrate::Substrate>) {
1265 {
1266 *self.substrate.write().await = substrate;
1267 }
1268 self.read_ledgers.clear();
1269 }
1270
1271 /// Clone the currently bound substrate.
1272 pub async fn substrate(&self) -> Arc<dyn crate::substrate::Substrate> {
1273 self.substrate.read().await.clone()
1274 }
1275
1276 pub fn with_event_log(mut self, log: EventLog) -> Self {
1277 self.log = Arc::new(TokioMutex::new(log));
1278 self
1279 }
1280
1281 /// Bind an event log the caller already holds a handle to.
1282 ///
1283 /// [`Self::with_event_log`] takes ownership, which leaves no way for
1284 /// anything outside the runtime to read what was recorded. That was fine
1285 /// while the log was observability-only, but a model-callable events
1286 /// surface has to run inside a `ToolExecutor` — and the executor is
1287 /// constructed *before* the runtime that would own it, so the two can only
1288 /// meet through a handle created first and shared into both
1289 /// (Parslee-ai/car#815).
1290 pub fn with_shared_event_log(mut self, log: Arc<TokioMutex<EventLog>>) -> Self {
1291 self.log = log;
1292 self
1293 }
1294
1295 /// The runtime's event log handle, for callers that need to read the record
1296 /// the runtime is writing.
1297 pub fn event_log_handle(&self) -> Arc<TokioMutex<EventLog>> {
1298 Arc::clone(&self.log)
1299 }
1300
1301 /// Attach a replan callback for failure recovery (builder).
1302 pub fn with_replan(self, callback: Arc<dyn ReplanCallback>, config: ReplanConfig) -> Self {
1303 if let Ok(mut guard) = self.replan_callback.try_lock() {
1304 *guard = Some(callback);
1305 }
1306 if let Ok(mut guard) = self.replan_config.try_write() {
1307 *guard = config;
1308 }
1309 self
1310 }
1311
1312 /// Set a replan callback at runtime.
1313 pub async fn set_replan_callback(&self, callback: Arc<dyn ReplanCallback>) {
1314 *self.replan_callback.lock().await = Some(callback);
1315 }
1316
1317 /// Set replan configuration at runtime.
1318 pub async fn set_replan_config(&self, config: ReplanConfig) {
1319 *self.replan_config.write().await = config;
1320 }
1321
1322 /// Set the pre-execution transactional conflict-check mode (survey
1323 /// §4.3/§5.2.4). `Off` (default) preserves prior behavior; `Warn`
1324 /// records conflicts as telemetry; `Strict` rejects a conflicting
1325 /// proposal before executing it.
1326 pub async fn set_transaction_check_mode(&self, mode: TransactionCheckMode) {
1327 *self.transaction_check.write().await = mode;
1328 }
1329
1330 /// Register a proposal-admission gate (EPIC A / task A1).
1331 ///
1332 /// Gates run during admission, before any action executes, in the
1333 /// order they were registered. Each is a verified pre-execution safety
1334 /// check — information-flow (A4), concurrency (A5), blocking-policy
1335 /// (A9) — that can block a proposal or escalate it to human approval.
1336 /// Registering no gates leaves behavior unchanged.
1337 pub async fn register_admission_gate(&self, gate: Arc<dyn crate::admission::AdmissionGate>) {
1338 self.admission_gates.write().await.push(gate);
1339 }
1340
1341 /// Register `gate`, **replacing** any already-registered gate answering
1342 /// to the same [`crate::admission::AdmissionGate::name`] instead of
1343 /// appending a second one. For gates that are *reconfigured* rather than
1344 /// layered — the VIGIL intent gate is reloaded whenever an operator
1345 /// re-reads `.car/intent.json` — appending is a bug, not a no-op: gate
1346 /// aggregation is fail-closed, so the stale gate's verdict still wins
1347 /// while it holds state (a [`crate::taint::TaintLedger`]) nothing writes
1348 /// to any more, and two gates with the same name emit two
1349 /// `AdmissionGateDecision` events under that name.
1350 ///
1351 /// The replaced gate keeps its **position**, so a reconfigure never
1352 /// silently reorders admission relative to the other gates. Any further
1353 /// gate carrying the same name is a stale duplicate from an earlier
1354 /// registration and is dropped, so exactly one gate answers to the name.
1355 async fn replace_admission_gate(&self, gate: Arc<dyn crate::admission::AdmissionGate>) {
1356 let name = gate.name().to_string();
1357 let mut gates = self.admission_gates.write().await;
1358 match gates.iter().position(|g| g.name() == name) {
1359 Some(i) => {
1360 gates[i] = gate;
1361 let mut idx = 0usize;
1362 gates.retain(|g| {
1363 let keep = idx == i || g.name() != name;
1364 idx += 1;
1365 keep
1366 });
1367 }
1368 None => gates.push(gate),
1369 }
1370 }
1371
1372 /// Remove all registered admission gates (primarily for tests and
1373 /// reconfiguration). After this, proposal admission reverts to the
1374 /// transactional pre-check only.
1375 pub async fn clear_admission_gates(&self) {
1376 self.admission_gates.write().await.clear();
1377 }
1378
1379 /// The number of currently-registered admission gates.
1380 pub async fn admission_gate_count(&self) -> usize {
1381 self.admission_gates.read().await.len()
1382 }
1383
1384 /// The `name()` of every registered admission gate, in registration order.
1385 ///
1386 /// Prefer this over [`Self::admission_gate_count`] when asserting that a
1387 /// *particular* gate is installed: a bare count couples the assertion to
1388 /// every other gate the runtime happens to register, so adding one breaks
1389 /// unrelated tests that only cared about their own.
1390 pub async fn admission_gate_names(&self) -> Vec<String> {
1391 self.admission_gates
1392 .read()
1393 .await
1394 .iter()
1395 .map(|g| g.name().to_string())
1396 .collect()
1397 }
1398
1399 /// Register the VIGIL intent gate (arXiv 2601.05755 — the live
1400 /// verify-before-commit call-site). Installs
1401 /// [`crate::intent_gate::IntentGate`] as an admission gate: a
1402 /// forbidden capability or a tool-stream-influenced out-of-intent
1403 /// action hard-rejects the proposal; untainted drift escalates to
1404 /// the durable approval flow (A7) by content-bound fingerprint.
1405 /// Replans are covered automatically (gates re-run on every
1406 /// replanned proposal).
1407 ///
1408 /// This is also where the runtime's [`crate::taint::TaintLedger`] is
1409 /// installed. From here on, every successful action records whether its
1410 /// result was tainted and which state keys it wrote, and the gate marks
1411 /// any incoming action that READS a tainted key as tool-stream-
1412 /// influenced. That closes the two holes the static `untrusted_tools`
1413 /// list leaves open: a trusted tool laundering attacker-controlled
1414 /// content out of state, and a replanned proposal that carries no
1415 /// dependency edge back to the poisoned action. Installing no intent
1416 /// gate installs no ledger, so nothing changes for a runtime that
1417 /// doesn't configure VIGIL.
1418 ///
1419 /// **Calling this again REPLACES the installed gate; it does not add a
1420 /// second one.** Reinstalling is a real operation — an operator editing
1421 /// `.car/intent.json` and re-running
1422 /// [`Runtime::install_intent_gate_from_project`] lands here — and each
1423 /// call builds a *fresh* ledger that becomes the only one the executor
1424 /// writes to. Appending would leave the previous gate registered while
1425 /// holding an orphaned ledger, and because gate aggregation is
1426 /// fail-closed that frozen gate's verdict would still win: every key
1427 /// tainted before the reload would stay tainted forever, no trusted
1428 /// overwrite could clear it, and legitimate out-of-intent work would
1429 /// hard-refuse with no approval path. So the runtime holds exactly one
1430 /// intent gate, and it is always the one bound to the live ledger. The
1431 /// replacement keeps the previous gate's position in the admission
1432 /// order, so a reload never silently reorders the other gates.
1433 pub async fn install_intent_gate(&self, config: crate::intent_gate::IntentGateConfig) {
1434 let ledger = Arc::new(crate::taint::TaintLedger::new(
1435 config.untrusted_tools.iter().cloned().collect(),
1436 ));
1437 *self.taint_ledger.write().await = Some(Arc::clone(&ledger));
1438 self.replace_admission_gate(Arc::new(crate::intent_gate::IntentGate::with_taint(
1439 config, ledger,
1440 )))
1441 .await;
1442 }
1443
1444 /// The runtime's taint provenance ledger, when an intent gate is
1445 /// installed (see [`Runtime::install_intent_gate`]). `None` otherwise —
1446 /// the ledger is strictly opt-in, alongside VIGIL.
1447 pub async fn taint_ledger(&self) -> Option<Arc<crate::taint::TaintLedger>> {
1448 self.taint_ledger.read().await.clone()
1449 }
1450
1451 /// Load `.car/intent.json` from `car_dir` and install the intent
1452 /// gate when present. Absent file → Ok(false) (opt-in, ungated);
1453 /// malformed file → loud error, never a silently-ungated session.
1454 pub async fn install_intent_gate_from_project(
1455 &self,
1456 car_dir: impl AsRef<std::path::Path>,
1457 ) -> Result<bool, crate::intent_gate::IntentLoadError> {
1458 match crate::intent_gate::load_intent_config(car_dir)? {
1459 Some(cfg) => {
1460 self.install_intent_gate(cfg).await;
1461 Ok(true)
1462 }
1463 None => Ok(false),
1464 }
1465 }
1466
1467 /// Register the skill deployment-tier ceiling gate (EPIC A / A8).
1468 ///
1469 /// Requires a memgine to be attached (skills live there). When a
1470 /// proposal names its driving skill in `context["skill"]`, the gate
1471 /// caps the proposal's actions at that skill's persisted
1472 /// `deployment_tier`, escalating an over-ceiling action to the durable
1473 /// approval flow (A7). Returns false if no memgine is attached.
1474 pub async fn install_skill_ceiling_gate(&self) -> bool {
1475 match &self.memgine {
1476 Some(mem) => {
1477 let gate = Arc::new(crate::skill_ceiling::SkillCeilingGate::new(mem.clone()));
1478 self.register_admission_gate(gate).await;
1479 true
1480 }
1481 None => false,
1482 }
1483 }
1484
1485 /// Drain buffered chunks + status for a detached tool invocation (C2).
1486 /// `None` for an unknown or fully-consumed handle. See
1487 /// [`crate::tool_handles::ToolHandleRegistry::poll`] for the
1488 /// consume-on-terminal contract.
1489 pub async fn tool_poll(&self, handle_id: &str) -> Option<crate::tool_handles::ToolPollResult> {
1490 self.tool_handles.poll(handle_id).await
1491 }
1492
1493 /// Request cancellation of a detached tool invocation (C2): fires the
1494 /// handle's cancel token (dropping the executor's chunk receiver) and
1495 /// seals its status as `cancelled` unless already terminal. Returns
1496 /// false for an unknown handle.
1497 pub async fn tool_cancel(&self, handle_id: &str) -> bool {
1498 self.tool_handles.cancel(handle_id).await
1499 }
1500
1501 /// Subscribe to the live [`car_ir::ToolStreamEvent`] fanout for all
1502 /// detached tool invocations on this runtime (C2). The WS layer
1503 /// forwards these as `tools.stream.event` notifications.
1504 pub fn subscribe_tool_events(
1505 &self,
1506 ) -> tokio::sync::broadcast::Receiver<car_ir::ToolStreamEvent> {
1507 self.tool_handles.subscribe()
1508 }
1509
1510 /// Enable tamper-evident hash chaining on the event log (EPIC A / A9).
1511 /// Every event appended from now on is linked to its predecessor by a
1512 /// content hash, so an after-the-fact edit to a chained event — or an
1513 /// interior deletion/reordering — is detectable via
1514 /// [`Runtime::verify_event_log_chain`]. Truncation at either end of the
1515 /// log (dropping a prefix or suffix wholesale) is NOT detectable — the
1516 /// chain has no anchored head hash and no trusted tail witness; that is
1517 /// out of scope until the chain head is anchored. Opt-in: existing logs
1518 /// stay byte-identical until enabled.
1519 pub async fn enable_event_log_hash_chaining(&self) {
1520 self.log.lock().await.enable_hash_chaining();
1521 }
1522
1523 /// Verify the event log's tamper-evidence chain (EPIC A / A9). Returns
1524 /// `Ok(n)` for `n` verified chained events, or `Err(index)` naming the
1525 /// first event whose hash/linkage doesn't match — the point of an
1526 /// interior edit, deletion, or reordering. Head/tail truncation is not
1527 /// detectable (no anchored head hash; the first chained event's
1528 /// `prev_hash` is taken on trust) — see `EventLog::verify_chain`.
1529 pub async fn verify_event_log_chain(&self) -> Result<usize, usize> {
1530 self.log.lock().await.verify_chain()
1531 }
1532
1533 /// Install a durable HITL approval ledger backed by a JSONL journal
1534 /// (EPIC A / A7). Loads any existing decisions so approvals survive a
1535 /// restart, then resolves future admission-gate `NeedsApproval`
1536 /// verdicts against it. The canonical daemon path is
1537 /// `~/.car/approvals.jsonl`.
1538 pub async fn set_approval_ledger_path(
1539 &self,
1540 path: impl Into<std::path::PathBuf>,
1541 ) -> std::io::Result<()> {
1542 let ledger = ApprovalLedger::with_journal(path.into())?;
1543 // Surface journal corruption (or a torn concurrent write) instead of
1544 // silently trusting a partial ledger (review A7 — the doc on
1545 // `skipped_on_load` promises callers surface it).
1546 if ledger.skipped_on_load() > 0 {
1547 tracing::warn!(
1548 skipped = ledger.skipped_on_load(),
1549 "approval ledger journal had unparseable lines skipped on load — \
1550 the ledger may be missing decisions"
1551 );
1552 }
1553 *self.approval_ledger.write().await = Some(ledger);
1554 Ok(())
1555 }
1556
1557 /// Use an in-memory approval ledger (no persistence) — primarily for
1558 /// tests and ephemeral runtimes.
1559 pub async fn set_approval_ledger_in_memory(&self) {
1560 *self.approval_ledger.write().await = Some(ApprovalLedger::new());
1561 }
1562
1563 /// Record a human approval for an admission fingerprint (EPIC A / A7).
1564 /// A subsequently re-submitted proposal whose escalation matches this
1565 /// fingerprint is admitted without asking again.
1566 pub async fn approve_admission(
1567 &self,
1568 fingerprint: &str,
1569 reviewer: &str,
1570 reason: &str,
1571 ) -> Result<(), String> {
1572 self.record_admission_decision(fingerprint, ApprovalDecision::Approved, reviewer, reason)
1573 .await
1574 }
1575
1576 /// Record a human rejection for an admission fingerprint (EPIC A / A7).
1577 /// A proposal whose escalation matches a rejected fingerprint is
1578 /// blocked outright.
1579 pub async fn reject_admission(
1580 &self,
1581 fingerprint: &str,
1582 reviewer: &str,
1583 reason: &str,
1584 ) -> Result<(), String> {
1585 self.record_admission_decision(fingerprint, ApprovalDecision::Rejected, reviewer, reason)
1586 .await
1587 }
1588
1589 async fn record_admission_decision(
1590 &self,
1591 fingerprint: &str,
1592 decision: ApprovalDecision,
1593 reviewer: &str,
1594 reason: &str,
1595 ) -> Result<(), String> {
1596 {
1597 let mut guard = self.approval_ledger.write().await;
1598 let ledger = guard
1599 .as_mut()
1600 .ok_or_else(|| "no approval ledger installed".to_string())?;
1601 ledger
1602 .record(ApprovalRecord {
1603 fingerprint: fingerprint.to_string(),
1604 // Admission escalations aren't tier-classified; record the
1605 // most restrictive tier so the decision reads as "elevated".
1606 required_tier: PermissionTier::FullAccess,
1607 decision,
1608 reviewer: reviewer.to_string(),
1609 reason: reason.to_string(),
1610 evidence: None,
1611 decided_at: chrono::Utc::now().to_rfc3339(),
1612 })
1613 // A journal write failure means the decision is NOT durable —
1614 // surface it instead of emitting a false ApprovalRecorded
1615 // audit event (review A7).
1616 .map_err(|e| format!("failed to persist approval decision: {e}"))?;
1617 }
1618 let approval = match decision {
1619 ApprovalDecision::Approved => "approved",
1620 ApprovalDecision::Rejected => "rejected",
1621 };
1622 let mut log = self.log.lock().await;
1623 log.append(
1624 EventKind::ApprovalRecorded,
1625 None,
1626 None,
1627 [
1628 ("fingerprint".to_string(), Value::from(fingerprint)),
1629 ("approval".to_string(), Value::from(approval)),
1630 ("reviewer".to_string(), Value::from(reviewer)),
1631 ("reason".to_string(), Value::from(reason)),
1632 ]
1633 .into(),
1634 );
1635 Ok(())
1636 }
1637
1638 /// Back the idempotency cache with a durable JSONL journal (EPIC A / C3).
1639 ///
1640 /// Loads any existing entries into the in-memory cache, then persists
1641 /// future idempotent results (and rollback invalidations, as
1642 /// tombstones) to the journal. After a crash-restart, re-submitting a
1643 /// completed idempotent action returns the cached result instead of
1644 /// re-executing it — preventing duplicate external side effects. The
1645 /// canonical daemon path is `~/.car/idempotency.jsonl`. Returns the
1646 /// number of live entries loaded.
1647 pub async fn set_idempotency_cache_path(
1648 &self,
1649 path: impl Into<std::path::PathBuf>,
1650 ) -> std::io::Result<usize> {
1651 let path = path.into();
1652 if let Some(parent) = path.parent() {
1653 let _ = std::fs::create_dir_all(parent);
1654 }
1655 // Replay the journal (last entry per key wins; a tombstone removes).
1656 let mut loaded: HashMap<String, ActionResult> = HashMap::new();
1657 if path.exists() {
1658 let content = std::fs::read_to_string(&path)?;
1659 for line in content.lines() {
1660 if line.trim().is_empty() {
1661 continue;
1662 }
1663 if let Ok(entry) = serde_json::from_str::<IdempotencyEntry>(line) {
1664 match entry.result {
1665 Some(r) => {
1666 loaded.insert(entry.key, r);
1667 }
1668 None => {
1669 loaded.remove(&entry.key);
1670 }
1671 }
1672 }
1673 }
1674 }
1675 {
1676 let mut cache = self.idempotency_cache.lock().await;
1677 for (k, v) in loaded.iter() {
1678 cache.entry(k.clone()).or_insert_with(|| v.clone());
1679 }
1680 }
1681 let count = loaded.len();
1682 *self.idempotency_journal.write().await = Some(path);
1683 Ok(count)
1684 }
1685
1686 /// Append an idempotency record to the journal if one is configured.
1687 /// `result = None` writes a tombstone (invalidation).
1688 async fn journal_idempotency(&self, key: &str, result: Option<&ActionResult>) {
1689 let path = {
1690 let guard = self.idempotency_journal.read().await;
1691 guard.clone()
1692 };
1693 let Some(path) = path else {
1694 return;
1695 };
1696 let entry = IdempotencyEntry {
1697 key: key.to_string(),
1698 result: result.cloned(),
1699 };
1700 if let Ok(mut line) = serde_json::to_string(&entry) {
1701 line.push('\n');
1702 use std::io::Write;
1703 // Persistence failure must be VISIBLE (linus review): a
1704 // silently-dropped journal write means an idempotent action
1705 // re-executes its side effects after a restart while the
1706 // operator believes it's covered. Fail-open (the in-memory
1707 // cache still dedups this process) but loudly.
1708 let write = std::fs::OpenOptions::new()
1709 .create(true)
1710 .append(true)
1711 .open(&path)
1712 .and_then(|mut f| f.write_all(line.as_bytes()));
1713 if let Err(e) = write {
1714 tracing::warn!(
1715 path = %path.display(),
1716 error = %e,
1717 "idempotency journal write failed — durable dedup is NOT covering this result"
1718 );
1719 }
1720 }
1721 }
1722
1723 /// Look up the current decision for an admission fingerprint, if any.
1724 pub async fn admission_decision(&self, fingerprint: &str) -> Option<ApprovalDecision> {
1725 let guard = self.approval_ledger.read().await;
1726 guard
1727 .as_ref()
1728 .and_then(|l| l.lookup(fingerprint).map(|r| r.decision))
1729 }
1730
1731 /// Verify a model's tool-use claims against the runtime's own execution
1732 /// receipts (EPIC A / A6 — arXiv 2603.10060). The runtime owns tool
1733 /// execution and logs it, so it holds unforgeable ground truth: this
1734 /// projects [`car_eventlog::tool_receipts::ToolReceipt`]s from the event
1735 /// log and cross-checks the supplied claims, catching a fabricated tool
1736 /// reference, a misstated result count, or a false "found nothing".
1737 ///
1738 /// `proposal_id` scopes the cross-check window to a single proposal's
1739 /// events (pass the proposal whose response the claims came from) — a
1740 /// claim is never judged against another run's receipts. The check is
1741 /// retention-coherent (review A6): when the log has trimmed events and
1742 /// the window can't be proven complete (unscoped, or the proposal's
1743 /// `ProposalReceived` marker — which precedes every receipt of that
1744 /// proposal — was itself evicted), a claim without a receipt comes back
1745 /// in `ReceiptReport::ungroundable` ("window evicted") instead of being
1746 /// mis-flagged `fabricated_tool_reference`.
1747 ///
1748 /// Returns the [`car_eventlog::tool_receipts::ReceiptReport`]; when it is not grounded, a
1749 /// `ToolReceiptHallucination` event is emitted so the caller's
1750 /// verdict→action loop (reject/flag the response) is auditable.
1751 /// Deterministic, zero-inference. Claims arrive structured — CAR's
1752 /// thesis is that intent is structured IR, so a caller extracts claims
1753 /// from the model's tool_calls / IR rather than regexing prose.
1754 pub async fn verify_tool_receipts(
1755 &self,
1756 claims: &[car_eventlog::tool_receipts::ToolClaim],
1757 proposal_id: Option<&str>,
1758 ) -> car_eventlog::tool_receipts::ReceiptReport {
1759 let (receipts, window_complete) = {
1760 let log = self.log.lock().await;
1761 let complete = if log.trimmed_events() == 0 {
1762 // Nothing was ever evicted — the window is complete whether
1763 // or not it is scoped.
1764 true
1765 } else {
1766 match proposal_id {
1767 // A proposal's window is complete iff its ProposalReceived
1768 // marker survived retention: every receipt of the proposal
1769 // was appended after it, so if the marker is retained, so
1770 // are the receipts.
1771 Some(pid) => log.events().iter().any(|e| {
1772 e.kind == EventKind::ProposalReceived
1773 && e.proposal_id.as_deref() == Some(pid)
1774 }),
1775 // Unscoped check over a trimmed log: unknowable.
1776 None => false,
1777 }
1778 };
1779 (
1780 car_eventlog::tool_receipts::receipts_from_events_scoped(log.events(), proposal_id),
1781 complete,
1782 )
1783 };
1784 let report = car_eventlog::tool_receipts::verify_tool_claims_windowed(
1785 claims,
1786 &receipts,
1787 window_complete,
1788 );
1789 if !report.grounded {
1790 let mut log = self.log.lock().await;
1791 log.append(
1792 EventKind::ToolReceiptHallucination,
1793 None,
1794 proposal_id,
1795 [
1796 (
1797 "count".to_string(),
1798 Value::from(report.hallucinations.len()),
1799 ),
1800 (
1801 "hallucinations".to_string(),
1802 serde_json::to_value(&report.hallucinations).unwrap_or_default(),
1803 ),
1804 ]
1805 .into(),
1806 );
1807 }
1808 report
1809 }
1810
1811 /// Run every registered admission gate against a proposal and fold
1812 /// their verdicts into a single [`crate::admission::AdmissionDecision`].
1813 ///
1814 /// Each gate's outcome is recorded as an `AdmissionGateDecision` event
1815 /// so a denial is attributable. Aggregation is conjunctive and
1816 /// fail-closed: the proposal is admitted only if every gate allowed it.
1817 /// Returns an admit decision immediately when no gates are registered
1818 /// (zero overhead on the common path).
1819 async fn run_admission_gates(
1820 &self,
1821 proposal: &ActionProposal,
1822 session_id: Option<&str>,
1823 scope: Option<&crate::scope::RuntimeScope>,
1824 ) -> crate::admission::AdmissionDecision {
1825 use crate::admission::{AdmissionDecision, GateContext};
1826
1827 let gates = self.admission_gates.read().await;
1828 if gates.is_empty() {
1829 return AdmissionDecision::admit();
1830 }
1831
1832 // One consistent snapshot for every gate this pass.
1833 let (state, versions) = self.state.versioned_snapshot();
1834 let ctx = GateContext {
1835 session_id,
1836 scope,
1837 state: &state,
1838 versions: &versions,
1839 };
1840
1841 let mut decision = AdmissionDecision::admit();
1842 for gate in gates.iter() {
1843 let outcome = gate.check(proposal, &ctx).await;
1844 // Audit every gate decision (allow included) so the trail shows
1845 // which gates ran, not just which objected.
1846 let mut props: HashMap<String, Value> = HashMap::new();
1847 props.insert("gate".to_string(), Value::from(gate.name()));
1848 // Pre-execution proposal admission (vs the multi-agent
1849 // commit barrier, which emits the same event kind with
1850 // phase:"commit_barrier").
1851 props.insert("phase".to_string(), Value::from("admission"));
1852 props.insert("decision".to_string(), Value::from(outcome.label()));
1853 match &outcome {
1854 crate::admission::GateOutcome::Allow => {}
1855 crate::admission::GateOutcome::Reject { blocked, reason } => {
1856 props.insert("reason".to_string(), Value::from(reason.clone()));
1857 props.insert(
1858 "blocked".to_string(),
1859 serde_json::to_value(blocked).unwrap_or_default(),
1860 );
1861 }
1862 crate::admission::GateOutcome::NeedsApproval {
1863 actions,
1864 fingerprint,
1865 reason,
1866 } => {
1867 props.insert("reason".to_string(), Value::from(reason.clone()));
1868 props.insert(
1869 "blocked".to_string(),
1870 serde_json::to_value(actions).unwrap_or_default(),
1871 );
1872 props.insert("fingerprint".to_string(), Value::from(fingerprint.clone()));
1873 }
1874 }
1875 {
1876 let mut log = self.log.lock().await;
1877 log.append(
1878 EventKind::AdmissionGateDecision,
1879 None,
1880 Some(&proposal.id),
1881 props,
1882 );
1883 }
1884 decision.absorb(gate.name(), outcome);
1885 }
1886 decision
1887 }
1888
1889 /// Install a harness operating config — the live end of the Evolution
1890 /// Agent loop (survey §3.5). After the meta-agent's `HarnessConfig::apply`
1891 /// produces a governed, regression-gated config, hand it here to take
1892 /// effect: `max_retries`/`retry_backoff_ms` drive the per-action retry
1893 /// loop, and `planning_max_replans` is mapped onto the replan budget.
1894 /// Every `HarnessConfig` knob is consumed here — none is aspirational.
1895 pub async fn set_harness_config(&self, cfg: car_memgine::HarnessConfig) {
1896 self.replan_config.write().await.max_replans = cfg.planning_max_replans;
1897 *self.harness_config.write().await = Some(cfg);
1898 }
1899
1900 /// The current harness operating config, if one has been installed.
1901 pub async fn harness_config(&self) -> Option<car_memgine::HarnessConfig> {
1902 self.harness_config.read().await.clone()
1903 }
1904
1905 /// Atomically read-modify-write the harness operating config under ONE
1906 /// write lock (installing the default first when none is set). The
1907 /// get→mutate→set alternative is a lost-update race when two requests
1908 /// mutate concurrently — the daemon's `evolution.run` harness-apply path
1909 /// uses this instead (kernel review S3). Keeps the same replan-budget
1910 /// propagation as [`Self::set_harness_config`].
1911 pub async fn update_harness_config<R>(
1912 &self,
1913 f: impl FnOnce(&mut car_memgine::HarnessConfig) -> R,
1914 ) -> R {
1915 let mut guard = self.harness_config.write().await;
1916 let cfg = guard.get_or_insert_with(car_memgine::HarnessConfig::default);
1917 let out = f(cfg);
1918 let max_replans = cfg.planning_max_replans;
1919 drop(guard);
1920 self.replan_config.write().await.max_replans = max_replans;
1921 out
1922 }
1923
1924 /// Run the pre-execution transactional check against the current
1925 /// versioned shared state. Returns the conflicting action ids when the
1926 /// mode is `Strict` and conflicts exist (so the caller can reject those
1927 /// actions); always emits `TransactionConflict` telemetry for each
1928 /// conflict found. Empty/`None` means "proceed".
1929 async fn transaction_precheck(
1930 &self,
1931 proposal: &ActionProposal,
1932 ) -> std::collections::HashSet<String> {
1933 let mode = *self.transaction_check.read().await;
1934 if mode == TransactionCheckMode::Off {
1935 return std::collections::HashSet::new();
1936 }
1937 let (state, versions) = self.state.versioned_snapshot();
1938 let report = car_verify::check_transaction(proposal, &versions, Some(&state));
1939 if report.consistent {
1940 return std::collections::HashSet::new();
1941 }
1942 let mut blocked = std::collections::HashSet::new();
1943 let mut log = self.log.lock().await;
1944 for c in &report.conflicts {
1945 for aid in &c.actions {
1946 blocked.insert(aid.clone());
1947 }
1948 log.append(
1949 EventKind::TransactionConflict,
1950 c.actions.first().map(|s| s.as_str()),
1951 Some(&proposal.id),
1952 [
1953 // Use the serde representation (snake_case: write_write
1954 // / read_write / stale_assumption) the .d.ts/.pyi/doc
1955 // contract is written against — NOT Debug, which would
1956 // emit "writewrite" (neo review #4).
1957 (
1958 "kind".to_string(),
1959 serde_json::to_value(c.kind).unwrap_or_default(),
1960 ),
1961 ("key".to_string(), Value::from(c.key.clone())),
1962 (
1963 "actions".to_string(),
1964 serde_json::to_value(&c.actions).unwrap_or_default(),
1965 ),
1966 (
1967 "explanation".to_string(),
1968 Value::from(c.explanation.clone()),
1969 ),
1970 ("resolution".to_string(), Value::from(c.resolution.clone())),
1971 ]
1972 .into(),
1973 );
1974 }
1975 drop(log);
1976 // Only Strict blocks execution; Warn records and proceeds.
1977 match mode {
1978 TransactionCheckMode::Strict => blocked,
1979 _ => std::collections::HashSet::new(),
1980 }
1981 }
1982
1983 /// Register a tool with just a name (backward compatible).
1984 pub async fn register_tool(&self, name: &str) {
1985 let schema = ToolSchema {
1986 name: name.to_string(),
1987 source: car_ir::ToolSourceKind::UserDefined,
1988 description: String::new(),
1989 parameters: serde_json::Value::Object(Default::default()),
1990 returns: None,
1991 idempotent: false,
1992 cache_ttl_secs: None,
1993 rate_limit: None,
1994 };
1995 self.register_tool_schema(schema).await;
1996 }
1997
1998 /// Register a tool with full schema.
1999 pub async fn register_tool_schema(&self, schema: ToolSchema) {
2000 // Auto-configure cache if schema specifies it
2001 if let Some(ttl) = schema.cache_ttl_secs {
2002 self.result_cache.enable_caching(&schema.name, ttl).await;
2003 }
2004 // Auto-configure rate limit if schema specifies it
2005 if let Some(ref rl) = schema.rate_limit {
2006 self.rate_limiter
2007 .set_limit(
2008 &schema.name,
2009 RateLimit {
2010 max_calls: rl.max_calls,
2011 interval_secs: rl.interval_secs,
2012 },
2013 )
2014 .await;
2015 }
2016 self.tools.write().await.insert(schema.name.clone(), schema);
2017 }
2018
2019 /// Register a tool via the canonical registry.
2020 /// This is the preferred way to register tools — it updates both the
2021 /// registry and the legacy tools HashMap for backward compatibility.
2022 pub async fn register_tool_entry(&self, mut entry: crate::registry::ToolEntry) {
2023 // The registry's source is authoritative for both event provenance and
2024 // the public tools.list/schema view, including literal-built entries.
2025 entry.schema.source = entry.source.kind();
2026 let schema = entry.schema.clone();
2027 self.registry.register(entry).await;
2028 self.register_tool_schema(schema).await;
2029 }
2030
2031 /// Remove a tool from both the canonical registry and the legacy
2032 /// `tools` schema map, so the model no longer sees it and the
2033 /// validator no longer accepts it. Used when a remote MCP connector
2034 /// tool is disabled or its connector is removed. Returns true if the
2035 /// tool was present in either store.
2036 pub async fn unregister_tool(&self, name: &str) -> bool {
2037 let removed_entry = self.registry.remove(name).await.is_some();
2038 let removed_schema = self.tools.write().await.remove(name).is_some();
2039 removed_entry || removed_schema
2040 }
2041
2042 /// Register CAR's built-in agent utility stdlib.
2043 ///
2044 /// This is an opt-in convenience layer for common local-file and text tools.
2045 /// Existing runtimes remain unchanged until this is called.
2046 pub async fn register_agent_basics(&self) {
2047 for entry in crate::agent_basics::entries() {
2048 self.register_tool_entry(entry).await;
2049 }
2050 }
2051
2052 /// Get all registered tool schemas (for model prompt generation).
2053 pub async fn tool_schemas(&self) -> Vec<ToolSchema> {
2054 self.tools.read().await.values().cloned().collect()
2055 }
2056
2057 /// Set a cost budget that limits proposal execution.
2058 pub async fn set_cost_budget(&self, budget: CostBudget) {
2059 *self.cost_budget.write().await = Some(budget);
2060 }
2061
2062 /// Set per-agent capability permissions that restrict tools, state keys, and action count.
2063 pub async fn set_capabilities(&self, caps: CapabilitySet) {
2064 *self.capabilities.write().await = Some(caps);
2065 }
2066
2067 /// Set a per-tool rate limit (token bucket).
2068 ///
2069 /// `max_calls` tokens are available per `interval_secs` window.
2070 /// When the bucket is empty, `dispatch()` applies backpressure by
2071 /// waiting until a token refills.
2072 pub async fn set_rate_limit(&self, tool: &str, max_calls: u32, interval_secs: f64) {
2073 self.rate_limiter
2074 .set_limit(
2075 tool,
2076 RateLimit {
2077 max_calls,
2078 interval_secs,
2079 },
2080 )
2081 .await;
2082 }
2083
2084 /// Enable cross-proposal result caching for a tool with a TTL in seconds.
2085 pub async fn enable_tool_cache(&self, tool: &str, ttl_secs: u64) {
2086 self.result_cache.enable_caching(tool, ttl_secs).await;
2087 }
2088
2089 /// Execute a proposal with automatic replanning on failure.
2090 ///
2091 /// If a `ReplanCallback` is registered and `max_replans > 0`, the runtime
2092 /// will catch abort failures, roll back state, ask the model for an
2093 /// alternative proposal via the callback, and re-execute. This transforms
2094 /// "execute-and-hope" into "execute-and-recover."
2095 ///
2096 /// If no callback is registered or `max_replans == 0`, behaves identically
2097 /// to a single `execute_inner()` call (zero overhead, fully backward compatible).
2098 #[instrument(
2099 name = "proposal.execute",
2100 skip_all,
2101 fields(
2102 proposal_id = %proposal.id,
2103 action_count = proposal.actions.len(),
2104 )
2105 )]
2106 pub async fn execute(&self, proposal: &ActionProposal) -> ProposalResult {
2107 // Forward to the cancel-aware variant with a never-cancelled
2108 // token. Existing callers see no behaviour change.
2109 let token = tokio_util::sync::CancellationToken::new();
2110 self.execute_with_cancel(proposal, &token).await
2111 }
2112
2113 /// Execute a proposal scoped to a specific session id.
2114 ///
2115 /// Validation walks the global policy registry plus the session's
2116 /// own registry — both must pass for an action to run. Session
2117 /// policies can deny what global allows; they cannot allow what
2118 /// global denies (validation is conjunctive).
2119 ///
2120 /// Returns the same [`ProposalResult`] shape as [`Self::execute`].
2121 /// Errors with an action-level rejection if the session id is
2122 /// unknown — callers should check via [`Self::session_exists`] or
2123 /// trust an id they minted via [`Self::open_session`].
2124 pub async fn execute_with_session(
2125 &self,
2126 proposal: &ActionProposal,
2127 session_id: &str,
2128 ) -> ProposalResult {
2129 let token = tokio_util::sync::CancellationToken::new();
2130 self.execute_with_session_and_cancel(proposal, session_id, &token)
2131 .await
2132 }
2133
2134 /// Combined session-scoped + cancellable execute. The session id
2135 /// is passed verbatim to the per-action policy check; the cancel
2136 /// token behaves identically to [`Self::execute_with_cancel`].
2137 pub async fn execute_with_session_and_cancel(
2138 &self,
2139 proposal: &ActionProposal,
2140 session_id: &str,
2141 cancel: &tokio_util::sync::CancellationToken,
2142 ) -> ProposalResult {
2143 self.execute_with_optional_session(proposal, Some(session_id), None, None, cancel)
2144 .await
2145 }
2146
2147 /// Execute an active-run proposal while requiring every accepted replan
2148 /// to retain the authenticated proposal id already claimed by the server.
2149 pub async fn execute_with_session_and_stable_replan_id(
2150 &self,
2151 proposal: &ActionProposal,
2152 session_id: &str,
2153 ) -> ProposalResult {
2154 let token = tokio_util::sync::CancellationToken::new();
2155 self.execute_with_optional_session(
2156 proposal,
2157 Some(session_id),
2158 None,
2159 Some(&proposal.id),
2160 &token,
2161 )
2162 .await
2163 }
2164
2165 /// Execute a proposal with cooperative cancellation.
2166 ///
2167 /// The runtime checks `token.is_cancelled()` at each DAG level
2168 /// boundary. When set, every action that hadn't yet started runs
2169 /// is reported as `Skipped` with `error = "canceled: ..."` so
2170 /// callers can distinguish "user pulled the plug" from "earlier
2171 /// abort cascaded." Actions already in flight continue to
2172 /// completion — tool calls dispatched to user-provided executors
2173 /// can't be safely interrupted from the engine.
2174 ///
2175 /// The CAR A2A bridge uses this so `tasks/cancel` produces a
2176 /// `ProposalResult` with clean partial state rather than relying
2177 /// on `JoinHandle::abort` to interrupt mid-await (which leaves
2178 /// no record of which actions actually ran).
2179 ///
2180 /// **FFI exposure:** this method is intentionally not surfaced
2181 /// through the NAPI / PyO3 / `car-server-core` JSON-RPC bindings.
2182 /// Those consumers (Node, Python, WebSocket) don't currently
2183 /// expose long-running async-task surfaces that need
2184 /// cancellation; the bridge is the lone consumer. When a binding
2185 /// gains a long-running task surface, the path is clear: add a
2186 /// per-binding token registry keyed by some caller-provided id,
2187 /// expose `cancelExecution(id)` / `cancel_execution(id)` /
2188 /// `proposal.cancel { id }`, and have the runtime call
2189 /// `execute_with_cancel` with the matching token. Skipping that
2190 /// today avoids speculative API surface that bloats bindings
2191 /// without a consumer.
2192 pub async fn execute_with_cancel(
2193 &self,
2194 proposal: &ActionProposal,
2195 cancel: &tokio_util::sync::CancellationToken,
2196 ) -> ProposalResult {
2197 self.execute_with_optional_session(proposal, None, None, None, cancel)
2198 .await
2199 }
2200
2201 pub async fn execute_with_stable_replan_id(&self, proposal: &ActionProposal) -> ProposalResult {
2202 let token = tokio_util::sync::CancellationToken::new();
2203 self.execute_with_optional_session(proposal, None, None, Some(&proposal.id), &token)
2204 .await
2205 }
2206
2207 /// Execute a proposal with an attached [`RuntimeScope`](crate::scope::RuntimeScope)
2208 /// (Parslee-ai/car#187 phase 3).
2209 ///
2210 /// Same contract as [`Self::execute`] plus a per-execution
2211 /// identity surface — typically built by the car-a2a dispatcher
2212 /// from the verified `Identity` and cooperative `a2a_caller`
2213 /// metadata on the inbound `ActionProposal`. The scope is
2214 /// recorded on the event log so downstream audit / log analysis
2215 /// can see which caller / tenant issued each action.
2216 ///
2217 /// **What this enforces today**: scope is captured + logged.
2218 /// Memgine queries and state-store ops still hit global
2219 /// namespaces — those follow-ups are tracked under #187.
2220 /// Tool / policy code that needs per-tenant behaviour right now
2221 /// should keep reading `proposal.context["a2a_caller_verified"]`
2222 /// directly (the phase 1 / 2 surface).
2223 pub async fn execute_scoped(
2224 &self,
2225 proposal: &ActionProposal,
2226 scope: &crate::scope::RuntimeScope,
2227 ) -> ProposalResult {
2228 let token = tokio_util::sync::CancellationToken::new();
2229 self.execute_scoped_with_cancel(proposal, scope, &token)
2230 .await
2231 }
2232
2233 /// Combined scoped + cancellable execute. Mirrors the shape of
2234 /// [`Self::execute_with_session_and_cancel`] for symmetry — both
2235 /// add a side-channel (session id / scope) on top of the
2236 /// cancellable form.
2237 pub async fn execute_scoped_with_cancel(
2238 &self,
2239 proposal: &ActionProposal,
2240 scope: &crate::scope::RuntimeScope,
2241 cancel: &tokio_util::sync::CancellationToken,
2242 ) -> ProposalResult {
2243 self.execute_with_optional_session(proposal, None, Some(scope), None, cancel)
2244 .await
2245 }
2246
2247 pub async fn execute_scoped_with_stable_replan_id(
2248 &self,
2249 proposal: &ActionProposal,
2250 scope: &crate::scope::RuntimeScope,
2251 ) -> ProposalResult {
2252 let token = tokio_util::sync::CancellationToken::new();
2253 self.execute_with_optional_session(proposal, None, Some(scope), Some(&proposal.id), &token)
2254 .await
2255 }
2256
2257 /// Internal entry point that backs both
2258 /// [`Self::execute_with_cancel`] (no session) and
2259 /// [`Self::execute_with_session_and_cancel`]. Holds the replan
2260 /// loop and threads `session_id` into the per-action validation
2261 /// path so session-scoped policies stack on top of global ones.
2262 async fn execute_with_optional_session(
2263 &self,
2264 proposal: &ActionProposal,
2265 session_id: Option<&str>,
2266 scope: Option<&crate::scope::RuntimeScope>,
2267 required_replan_proposal_id: Option<&str>,
2268 cancel: &tokio_util::sync::CancellationToken,
2269 ) -> ProposalResult {
2270 // The shared StateStore has one proposal transaction at a time. The
2271 // guard spans validation, replanning, and rollback so distinct Runtime
2272 // facades over the same store cannot interleave state snapshots.
2273 let _proposal_execution = self.state.lock_proposal_execution().await;
2274 self.execute_with_optional_session_already_guarded(
2275 proposal,
2276 session_id,
2277 scope,
2278 required_replan_proposal_id,
2279 cancel,
2280 )
2281 .await
2282 }
2283
2284 /// Execute while the caller retains this StateStore's proposal-execution
2285 /// guard. `plan_and_execute` uses this path so one guard covers its initial
2286 /// snapshot, ranking, every candidate, and the final commit/rollback.
2287 async fn execute_with_optional_session_already_guarded(
2288 &self,
2289 proposal: &ActionProposal,
2290 session_id: Option<&str>,
2291 scope: Option<&crate::scope::RuntimeScope>,
2292 required_replan_proposal_id: Option<&str>,
2293 cancel: &tokio_util::sync::CancellationToken,
2294 ) -> ProposalResult {
2295 // Establish canonical identity before any other rejection so an
2296 // undigested lineage entry can only mean the proposal was genuinely
2297 // impossible to represent as JCS/I-JSON. No journal row is emitted.
2298 let original_proposal_digest = match proposal_digest(proposal) {
2299 Ok(digest) => digest,
2300 Err(error) => {
2301 self.log.lock().await.append(
2302 EventKind::StateRollback,
2303 None,
2304 Some(&proposal.id),
2305 proposal_rejection_boundary_data(proposal, None, &error),
2306 );
2307 let result = ProposalResult::for_proposal(
2308 proposal,
2309 proposal
2310 .actions
2311 .iter()
2312 .map(|action| rejected_result(&action.id, error.clone()))
2313 .collect(),
2314 CostSummary::default(),
2315 );
2316 return finalize_proposal_result(
2317 result,
2318 &proposal.id,
2319 &[ProposalLineageEntry {
2320 generation: 0,
2321 proposal_id: proposal.id.clone(),
2322 proposal_digest: None,
2323 status: ProposalLineageStatus::Rejected,
2324 rejection_reason: Some(error),
2325 }],
2326 &[],
2327 );
2328 }
2329 };
2330
2331 // Proposal-local action ids are the join key for DAG scheduling,
2332 // result rows, receipts, and StateTransition attribution. Reject a
2333 // duplicate before scope/admission/transaction logging or any state
2334 // mutation. Retried attempts remain valid reuse of one admitted id.
2335 if let Err(error) = validate_proposal_action_ids(proposal) {
2336 self.log.lock().await.append(
2337 EventKind::StateRollback,
2338 None,
2339 Some(&proposal.id),
2340 proposal_rejection_boundary_data(proposal, Some(&original_proposal_digest), &error),
2341 );
2342 let result = ProposalResult::for_proposal(
2343 proposal,
2344 proposal
2345 .actions
2346 .iter()
2347 .map(|action| rejected_result(&action.id, error.clone()))
2348 .collect(),
2349 CostSummary::default(),
2350 );
2351 return finalize_proposal_result(
2352 result,
2353 &proposal.id,
2354 &[proposal_lineage_entry(
2355 proposal,
2356 0,
2357 ProposalLineageStatus::Rejected,
2358 Some(error),
2359 )],
2360 &[],
2361 );
2362 }
2363 if let Err(error) = validate_proposal_retry_limits(proposal) {
2364 self.log.lock().await.append(
2365 EventKind::StateRollback,
2366 None,
2367 Some(&proposal.id),
2368 proposal_rejection_boundary_data(proposal, Some(&original_proposal_digest), &error),
2369 );
2370 let result = ProposalResult::for_proposal(
2371 proposal,
2372 proposal
2373 .actions
2374 .iter()
2375 .map(|action| rejected_result(&action.id, error.clone()))
2376 .collect(),
2377 CostSummary::default(),
2378 );
2379 return finalize_proposal_result(
2380 result,
2381 &proposal.id,
2382 &[proposal_lineage_entry(
2383 proposal,
2384 0,
2385 ProposalLineageStatus::Rejected,
2386 Some(error),
2387 )],
2388 &[],
2389 );
2390 }
2391 let config = self.replan_config.read().await.clone();
2392 let mut current_proposal = proposal.clone();
2393 let mut attempt: u32 = 0;
2394 let mut lineage = vec![proposal_lineage_entry(
2395 proposal,
2396 0,
2397 ProposalLineageStatus::Accepted,
2398 None,
2399 )];
2400
2401 // Phase 3 foundation (Parslee-ai/car#187): record the scope
2402 // on the event log so audit / log analysis can correlate
2403 // actions to the caller / tenant that triggered them. Only
2404 // logged when at least one identity field is set — keeps
2405 // the existing in-process call sites free of noise.
2406 if let Some(s) = scope {
2407 if !s.is_unscoped() {
2408 let mut props: HashMap<String, Value> = HashMap::new();
2409 if let Some(cid) = &s.caller_id {
2410 props.insert("caller_id".to_string(), Value::from(cid.as_str()));
2411 }
2412 if let Some(tid) = &s.tenant_id {
2413 props.insert("tenant_id".to_string(), Value::from(tid.as_str()));
2414 }
2415 if !s.claims.is_empty() {
2416 if let Ok(claims_json) = serde_json::to_value(&s.claims) {
2417 props.insert("claims".to_string(), claims_json);
2418 }
2419 }
2420 let mut log = self.log.lock().await;
2421 log.append(EventKind::SessionScope, None, Some(&proposal.id), props);
2422 }
2423 }
2424
2425 // Pre-execution transactional conflict check (survey §4.3/§5.2.4).
2426 // Off by default; Warn records conflicts; Strict rejects the whole
2427 // proposal before any action runs, since a transactional conflict is
2428 // a property of the action *set* against current state, not an
2429 // isolated action. This is an advisory planning-time gate, not a
2430 // substitute for per-action execution-time validation. Replanned
2431 // proposals are re-checked inside the loop (at the replan quality
2432 // gate below). A pre-execution rejection here deliberately produces
2433 // no execution trajectory — nothing ran; the emitted
2434 // `TransactionConflict` events are the audit record.
2435 let blocked = self.transaction_precheck(¤t_proposal).await;
2436 if !blocked.is_empty() {
2437 let reason = "transactional conflict with current shared state (strict mode); \
2438 see TransactionConflict events for details and resolution"
2439 .to_string();
2440 let results = current_proposal
2441 .actions
2442 .iter()
2443 .map(|a| rejected_result(&a.id, reason.clone()))
2444 .collect();
2445 self.log.lock().await.append(
2446 EventKind::StateRollback,
2447 None,
2448 Some(¤t_proposal.id),
2449 proposal_rejection_boundary_data(
2450 ¤t_proposal,
2451 Some(&original_proposal_digest),
2452 &reason,
2453 ),
2454 );
2455 lineage[0].status = ProposalLineageStatus::Rejected;
2456 lineage[0].rejection_reason = Some(reason);
2457 return finalize_proposal_result(
2458 ProposalResult::for_proposal(proposal, results, Default::default()),
2459 &proposal.id,
2460 &lineage,
2461 &[],
2462 );
2463 }
2464
2465 // Pre-execution admission gates (EPIC A / task A1 — the safety
2466 // seam). Runs every registered AdmissionGate against the proposal
2467 // before any action executes. Aggregation is conjunctive and
2468 // fail-closed: a proposal any gate blocks (or escalates to
2469 // approval) does not run. Like the transactional pre-check above, a
2470 // rejection here produces no execution trajectory — the emitted
2471 // AdmissionGateDecision events are the audit record. No gates
2472 // registered → zero overhead, identical behavior.
2473 let admission = self
2474 .run_admission_gates(¤t_proposal, session_id, scope)
2475 .await;
2476 if !admission.admitted {
2477 let gate = admission.deciding_gate.as_deref().unwrap_or("admission");
2478 let base_reason = admission
2479 .reason
2480 .clone()
2481 .unwrap_or_else(|| "blocked by admission gate".to_string());
2482 // Resolve approval escalations against the durable ledger (A7).
2483 // A hard `Reject` from ANY gate is never overridable — the
2484 // ledger is not consulted at all in that case (an old approval
2485 // for one gate's escalation must not steamroll another gate's
2486 // deny). Otherwise EVERY escalation must resolve to Approved,
2487 // each by its own fingerprint: one Rejected fingerprint blocks,
2488 // one unseen fingerprint stays pending (fail-closed).
2489 let mut approved = false;
2490 let reason = if admission.hard_rejected {
2491 format!("{base_reason} (gate: {gate})")
2492 } else if admission.needs_approval() {
2493 let mut blocking_reason = None;
2494 for esc in &admission.escalations {
2495 match self.admission_decision(&esc.fingerprint).await {
2496 Some(ApprovalDecision::Approved) => continue,
2497 Some(ApprovalDecision::Rejected) => {
2498 blocking_reason = Some(format!(
2499 "rejected by operator (gate: {}; fingerprint: {})",
2500 esc.gate, esc.fingerprint
2501 ));
2502 break;
2503 }
2504 None => {
2505 blocking_reason = Some(format!(
2506 "requires human approval (gate: {}; {}); \
2507 approve fingerprint '{}' then re-run",
2508 esc.gate, esc.reason, esc.fingerprint
2509 ));
2510 break;
2511 }
2512 }
2513 }
2514 match blocking_reason {
2515 None => {
2516 approved = true;
2517 // Audit every durable approval taking effect.
2518 let mut log = self.log.lock().await;
2519 for esc in &admission.escalations {
2520 log.append(
2521 EventKind::ApprovalRecorded,
2522 None,
2523 Some(&proposal.id),
2524 [
2525 (
2526 "fingerprint".to_string(),
2527 Value::from(esc.fingerprint.as_str()),
2528 ),
2529 ("gate".to_string(), Value::from(esc.gate.as_str())),
2530 ("approval".to_string(), Value::from("approved")),
2531 ("applied".to_string(), Value::from(true)),
2532 ]
2533 .into(),
2534 );
2535 }
2536 String::new()
2537 }
2538 Some(r) => r,
2539 }
2540 } else {
2541 format!("{base_reason} (gate: {gate})")
2542 };
2543 if approved {
2544 // Escalation cleared by a durable approval — proceed to
2545 // execution as if admitted.
2546 } else {
2547 let results = current_proposal
2548 .actions
2549 .iter()
2550 .map(|a| {
2551 // Name the specific reason on the offending actions; a
2552 // generic note on the rest (the whole proposal is held,
2553 // since a safety hazard is a property of the set).
2554 if admission.blocked.is_empty() || admission.blocked.contains(&a.id) {
2555 rejected_result(&a.id, reason.clone())
2556 } else {
2557 rejected_result(
2558 &a.id,
2559 format!("proposal blocked by admission gate: {gate}"),
2560 )
2561 }
2562 })
2563 .collect();
2564 self.log.lock().await.append(
2565 EventKind::StateRollback,
2566 None,
2567 Some(¤t_proposal.id),
2568 proposal_rejection_boundary_data(
2569 ¤t_proposal,
2570 Some(&original_proposal_digest),
2571 &reason,
2572 ),
2573 );
2574 lineage[0].status = ProposalLineageStatus::Rejected;
2575 lineage[0].rejection_reason = Some(reason.clone());
2576 let result = finalize_proposal_result(
2577 ProposalResult::for_proposal(proposal, results, Default::default()),
2578 &proposal.id,
2579 &lineage,
2580 &[],
2581 );
2582 // Record the trajectory even though nothing dispatched. This
2583 // return is *before* the execution loop's persist calls, so
2584 // without this an admission rejection is invisible to the
2585 // trajectory store — and the per-tool success rates
2586 // `verify.monte_carlo` derives from it would silently skew
2587 // optimistic, counting only calls that got far enough to run.
2588 // The state map is empty because no action mutated anything.
2589 if let Some(err) = self.persist_trajectory(
2590 proposal,
2591 ¤t_proposal,
2592 &result,
2593 car_memgine::TrajectoryOutcome::Failed,
2594 0,
2595 &HashMap::new(),
2596 ) {
2597 tracing::warn!(
2598 error = %err,
2599 proposal_id = %proposal.id,
2600 "failed to persist trajectory for admission-rejected proposal"
2601 );
2602 }
2603 return finalize_proposal_result(result, &proposal.id, &lineage, &[]);
2604 }
2605 }
2606
2607 let mut accepted_proposal_preimages = vec![AcceptedProposalPreimage {
2608 generation: 0,
2609 proposal_digest: original_proposal_digest,
2610 proposal: proposal.clone(),
2611 }];
2612
2613 loop {
2614 let (result, state_before_map) = self
2615 .execute_inner_with_cancel(¤t_proposal, Some(cancel), session_id, scope)
2616 .await;
2617
2618 // Statuses that trigger rollback + replan: always runtime Failed,
2619 // and (opt-in) validator/policy/capability Rejected.
2620 let replan_triggers = |s: &ActionStatus| {
2621 *s == ActionStatus::Failed
2622 || (config.replan_on_rejected && *s == ActionStatus::Rejected)
2623 };
2624
2625 // A terminal tool failure ends this engine execution. It may not
2626 // enter the retry loop above or the proposal-replan loop here;
2627 // daemon-owned session halting is layered on top by the caller.
2628 let terminal_failure = result.results.iter().any(|result| result.terminal);
2629 // Check if we aborted
2630 let aborted = result.results.iter().any(|r| replan_triggers(&r.status));
2631 let rollback_durability_failed = result.results.iter().any(|result| {
2632 result.error.as_deref().is_some_and(|error| {
2633 error.contains(ROLLBACK_DURABILITY_ERROR)
2634 || error.contains(ROLLBACK_DURABILITY_UNKNOWN)
2635 })
2636 });
2637 if !aborted
2638 || terminal_failure
2639 || attempt >= config.max_replans
2640 || rollback_durability_failed
2641 {
2642 if aborted && !terminal_failure && attempt > 0 && !rollback_durability_failed {
2643 // Exhausted all replan attempts
2644 let mut log = self.log.lock().await;
2645 log.append(
2646 EventKind::ReplanExhausted,
2647 None,
2648 Some(&proposal.id),
2649 [("attempts".to_string(), Value::from(attempt))].into(),
2650 );
2651 }
2652
2653 // Persist trajectory
2654 let outcome = if !aborted {
2655 if attempt > 0 {
2656 car_memgine::TrajectoryOutcome::ReplanSuccess
2657 } else {
2658 car_memgine::TrajectoryOutcome::Success
2659 }
2660 } else if attempt > 0 && !terminal_failure {
2661 car_memgine::TrajectoryOutcome::ReplanExhausted
2662 } else {
2663 car_memgine::TrajectoryOutcome::Failed
2664 };
2665 if let Some(err) = self.persist_trajectory(
2666 proposal,
2667 ¤t_proposal,
2668 &result,
2669 outcome,
2670 attempt,
2671 &state_before_map,
2672 ) {
2673 let mut log = self.log.lock().await;
2674 log.append(
2675 EventKind::ActionFailed,
2676 None,
2677 Some(&proposal.id),
2678 [(
2679 "trajectory_persist_error".to_string(),
2680 Value::from(err.as_str()),
2681 )]
2682 .into(),
2683 );
2684 }
2685
2686 return finalize_proposal_result(
2687 result,
2688 &proposal.id,
2689 &lineage,
2690 &accepted_proposal_preimages,
2691 );
2692 }
2693
2694 // Get replan callback (clone Arc, drop lock immediately)
2695 let callback = {
2696 let guard = self.replan_callback.lock().await;
2697 guard.clone()
2698 };
2699 let Some(callback) = callback else {
2700 // No callback registered — persist trajectory and return
2701 if let Some(err) = self.persist_trajectory(
2702 proposal,
2703 ¤t_proposal,
2704 &result,
2705 car_memgine::TrajectoryOutcome::Failed,
2706 attempt,
2707 &state_before_map,
2708 ) {
2709 let mut log = self.log.lock().await;
2710 log.append(
2711 EventKind::ActionFailed,
2712 None,
2713 Some(&proposal.id),
2714 [(
2715 "trajectory_persist_error".to_string(),
2716 Value::from(err.as_str()),
2717 )]
2718 .into(),
2719 );
2720 }
2721 return finalize_proposal_result(
2722 result,
2723 &proposal.id,
2724 &lineage,
2725 &accepted_proposal_preimages,
2726 );
2727 };
2728
2729 // Build the stable failure context once. Candidate rejection is a
2730 // planning-only loop below: it consumes a replan generation but
2731 // never returns to `execute_inner_with_cancel`, so a failed
2732 // proposal with external effects cannot be dispatched twice.
2733 let failed_actions: Vec<FailedActionSummary> = result
2734 .results
2735 .iter()
2736 .filter(|r| replan_triggers(&r.status) && !r.rolled_back)
2737 .map(|r| {
2738 let action = current_proposal
2739 .actions
2740 .iter()
2741 .find(|a| a.id == r.action_id);
2742 FailedActionSummary {
2743 action_id: r.action_id.clone(),
2744 tool: action.and_then(|a| a.tool.clone()),
2745 error: r.error.clone().unwrap_or_default(),
2746 parameters: action.map(|a| a.parameters.clone()).unwrap_or_default(),
2747 }
2748 })
2749 .collect();
2750
2751 let completed_action_ids: Vec<String> = result
2752 .results
2753 .iter()
2754 .filter(|r| r.rolled_back)
2755 .map(|r| r.action_id.clone())
2756 .collect();
2757
2758 'replan_candidates: loop {
2759 if attempt >= config.max_replans {
2760 let mut log = self.log.lock().await;
2761 log.append(
2762 EventKind::ReplanExhausted,
2763 None,
2764 Some(&proposal.id),
2765 [("attempts".to_string(), Value::from(attempt))].into(),
2766 );
2767 drop(log);
2768 if let Some(err) = self.persist_trajectory(
2769 proposal,
2770 ¤t_proposal,
2771 &result,
2772 car_memgine::TrajectoryOutcome::ReplanExhausted,
2773 attempt,
2774 &state_before_map,
2775 ) {
2776 self.log.lock().await.append(
2777 EventKind::ActionFailed,
2778 None,
2779 Some(&proposal.id),
2780 [("trajectory_persist_error".to_string(), Value::from(err))].into(),
2781 );
2782 }
2783 return finalize_proposal_result(
2784 result,
2785 &proposal.id,
2786 &lineage,
2787 &accepted_proposal_preimages,
2788 );
2789 }
2790
2791 let generation = attempt + 1;
2792 let ctx = ReplanContext {
2793 proposal_id: proposal.id.clone(),
2794 attempt: generation,
2795 failed_actions: failed_actions.clone(),
2796 completed_action_ids: completed_action_ids.clone(),
2797 state_snapshot: self.state.snapshot(),
2798 replans_remaining: config.max_replans.saturating_sub(generation),
2799 original_source: proposal.source.clone(),
2800 original_action_count: proposal.actions.len(),
2801 original_context: proposal.context.clone(),
2802 };
2803
2804 // Backoff delay between replan attempts
2805 if config.delay_ms > 0 {
2806 tokio::time::sleep(Duration::from_millis(config.delay_ms)).await;
2807 }
2808
2809 // Log replan attempt
2810 {
2811 let mut log = self.log.lock().await;
2812 log.append(
2813 EventKind::ReplanAttempted,
2814 None,
2815 Some(&proposal.id),
2816 [
2817 ("attempt".to_string(), Value::from(generation)),
2818 (
2819 "failed_count".to_string(),
2820 Value::from(ctx.failed_actions.len()),
2821 ),
2822 ]
2823 .into(),
2824 );
2825 // Deep-telemetry breadcrumbs (§3.5.1): record the fork the
2826 // harness took (replan vs accept vs abandon) and the
2827 // approach it discarded, so failure-mode diagnosis can see
2828 // the path *not* taken, not just the path taken.
2829 let rejected_ids: Vec<String> = ctx
2830 .failed_actions
2831 .iter()
2832 .map(|f| f.action_id.clone())
2833 .collect();
2834 log.append(
2835 EventKind::BranchDecision,
2836 None,
2837 Some(&proposal.id),
2838 [
2839 ("branch".to_string(), Value::from("replan")),
2840 (
2841 "reason".to_string(),
2842 Value::from("actions failed; requesting a revised plan"),
2843 ),
2844 ("attempt".to_string(), Value::from(generation)),
2845 ]
2846 .into(),
2847 );
2848 log.append(
2849 EventKind::AlternativeRejected,
2850 None,
2851 Some(&proposal.id),
2852 [
2853 (
2854 "alternative".to_string(),
2855 serde_json::to_value(&rejected_ids).unwrap_or_default(),
2856 ),
2857 (
2858 "reason".to_string(),
2859 Value::from("plan superseded by replan after action failure"),
2860 ),
2861 ]
2862 .into(),
2863 );
2864 }
2865
2866 // Call the model for a new plan
2867 match callback.replan(&ctx).await {
2868 Ok(new_proposal) => {
2869 attempt = generation;
2870 let (new_proposal_preimage, new_proposal_digest) =
2871 match proposal_journal_identity(&new_proposal) {
2872 Ok(identity) => identity,
2873 Err(error) => {
2874 lineage.push(ProposalLineageEntry {
2875 generation,
2876 proposal_id: new_proposal.id.clone(),
2877 proposal_digest: None,
2878 status: ProposalLineageStatus::Rejected,
2879 rejection_reason: Some(error.clone()),
2880 });
2881 self.log.lock().await.append(
2882 EventKind::ReplanRejected,
2883 None,
2884 Some(&proposal.id),
2885 [
2886 ("errors".to_string(), Value::from(error)),
2887 ("attempt".to_string(), Value::from(generation)),
2888 ]
2889 .into(),
2890 );
2891 continue 'replan_candidates;
2892 }
2893 };
2894 if required_replan_proposal_id
2895 .is_some_and(|required| new_proposal.id != required)
2896 {
2897 let reason = format!(
2898 "replan proposal id `{}` does not retain authenticated proposal id `{}`",
2899 new_proposal.id,
2900 required_replan_proposal_id.expect("checked above")
2901 );
2902 lineage.push(ProposalLineageEntry {
2903 generation,
2904 proposal_id: new_proposal.id.clone(),
2905 proposal_digest: Some(new_proposal_digest),
2906 status: ProposalLineageStatus::Rejected,
2907 rejection_reason: Some(reason.clone()),
2908 });
2909 self.log.lock().await.append(
2910 EventKind::ReplanRejected,
2911 None,
2912 Some(&proposal.id),
2913 [
2914 ("errors".to_string(), Value::from(reason)),
2915 ("attempt".to_string(), Value::from(generation)),
2916 ]
2917 .into(),
2918 );
2919 continue 'replan_candidates;
2920 }
2921 if let Err(error) = validate_proposal_action_ids(&new_proposal) {
2922 lineage.push(ProposalLineageEntry {
2923 generation,
2924 proposal_id: new_proposal.id.clone(),
2925 proposal_digest: Some(new_proposal_digest),
2926 status: ProposalLineageStatus::Rejected,
2927 rejection_reason: Some(error.clone()),
2928 });
2929 let mut log = self.log.lock().await;
2930 log.append(
2931 EventKind::ReplanRejected,
2932 None,
2933 Some(&proposal.id),
2934 [
2935 (
2936 "errors".to_string(),
2937 Value::from(format!("invalid proposal: {error}")),
2938 ),
2939 ("attempt".to_string(), Value::from(generation)),
2940 ]
2941 .into(),
2942 );
2943 continue 'replan_candidates;
2944 }
2945
2946 if let Err(error) = validate_proposal_retry_limits(&new_proposal) {
2947 lineage.push(ProposalLineageEntry {
2948 generation,
2949 proposal_id: new_proposal.id.clone(),
2950 proposal_digest: Some(new_proposal_digest),
2951 status: ProposalLineageStatus::Rejected,
2952 rejection_reason: Some(error.clone()),
2953 });
2954 self.log.lock().await.append(
2955 EventKind::ReplanRejected,
2956 None,
2957 Some(&proposal.id),
2958 [
2959 ("errors".to_string(), Value::from(error)),
2960 ("attempt".to_string(), Value::from(generation)),
2961 ]
2962 .into(),
2963 );
2964 continue 'replan_candidates;
2965 }
2966
2967 // Quality gate: verify replan proposal before executing
2968 if config.verify_before_execute {
2969 let current_state = self.state.snapshot();
2970 // Verify against the full registered schemas
2971 // (not just names) so a replan with a bad
2972 // parameter type / missing required field is
2973 // rejected here, per register_tool_schema's
2974 // contract (car-releases#56). The read guard is
2975 // held across the synchronous verify call.
2976 // Same helper the admission gate uses, so the two
2977 // verification points cannot disagree about what is
2978 // fatal. They previously did: this path blocked on the
2979 // loop heuristic and on state-dependent findings that
2980 // the gate treats as advisory, and it passed the tool
2981 // map through unconditionally — so an embedder with no
2982 // registered schemas burned its whole replan budget on
2983 // "unregistered tool" for every call.
2984 let blocking = {
2985 let tools_guard = self.tools.read().await;
2986 crate::verify_gate::blocking_errors(
2987 &new_proposal,
2988 Some(¤t_state),
2989 &tools_guard,
2990 100,
2991 )
2992 };
2993 if !blocking.is_empty() {
2994 let error_msgs: Vec<String> =
2995 blocking.iter().map(|i| i.message.clone()).collect();
2996 let reason = error_msgs.join("; ");
2997 lineage.push(ProposalLineageEntry {
2998 generation,
2999 proposal_id: new_proposal.id.clone(),
3000 proposal_digest: Some(new_proposal_digest),
3001 status: ProposalLineageStatus::Rejected,
3002 rejection_reason: Some(reason.clone()),
3003 });
3004 let mut log = self.log.lock().await;
3005 log.append(
3006 EventKind::ReplanRejected,
3007 None,
3008 Some(&proposal.id),
3009 [
3010 ("errors".to_string(), Value::from(reason)),
3011 ("attempt".to_string(), Value::from(generation)),
3012 ]
3013 .into(),
3014 );
3015 // Don't execute a broken replan — count as failed attempt
3016 continue 'replan_candidates;
3017 }
3018 }
3019
3020 // Transactional re-check of the replan against the now-
3021 // mutated shared state (neo review #1): a replan runs
3022 // after earlier actions changed state, so it's exactly
3023 // where a fresh stale-assumption/write conflict appears.
3024 // A Strict conflict rejects this replan attempt via the
3025 // same bounded ReplanRejected path (capped by
3026 // max_replans), never an infinite loop.
3027 let replan_blocked = self.transaction_precheck(&new_proposal).await;
3028 if !replan_blocked.is_empty() {
3029 let reason = "transactional conflict with current state".to_string();
3030 lineage.push(ProposalLineageEntry {
3031 generation,
3032 proposal_id: new_proposal.id.clone(),
3033 proposal_digest: Some(new_proposal_digest),
3034 status: ProposalLineageStatus::Rejected,
3035 rejection_reason: Some(reason.clone()),
3036 });
3037 let mut log = self.log.lock().await;
3038 log.append(
3039 EventKind::ReplanRejected,
3040 None,
3041 Some(&proposal.id),
3042 [
3043 ("errors".to_string(), Value::from(reason)),
3044 ("attempt".to_string(), Value::from(generation)),
3045 ]
3046 .into(),
3047 );
3048 drop(log);
3049 continue 'replan_candidates;
3050 }
3051
3052 // Admission gates re-run on EVERY replanned proposal
3053 // (linus review C-2): the replan callback is exactly
3054 // where injected/adversarial content reshapes a plan,
3055 // so a proposal that was clean at first admission must
3056 // not smuggle a hazardous replan past the gates. Same
3057 // fail-closed contract as first admission — a hard
3058 // reject or any unapproved escalation rejects this
3059 // replan attempt (bounded by max_replans).
3060 let replan_admission = self
3061 .run_admission_gates(&new_proposal, session_id, scope)
3062 .await;
3063 if !replan_admission.admitted {
3064 let mut cleared = !replan_admission.hard_rejected
3065 && replan_admission.needs_approval();
3066 if cleared {
3067 for esc in &replan_admission.escalations {
3068 if self.admission_decision(&esc.fingerprint).await
3069 != Some(ApprovalDecision::Approved)
3070 {
3071 cleared = false;
3072 break;
3073 }
3074 }
3075 }
3076 if !cleared {
3077 let gate = replan_admission
3078 .deciding_gate
3079 .as_deref()
3080 .unwrap_or("admission");
3081 let why = replan_admission
3082 .reason
3083 .clone()
3084 .unwrap_or_else(|| "blocked by admission gate".to_string());
3085 let reason =
3086 format!("admission gate blocked replan (gate: {gate}; {why})");
3087 lineage.push(ProposalLineageEntry {
3088 generation,
3089 proposal_id: new_proposal.id.clone(),
3090 proposal_digest: Some(new_proposal_digest),
3091 status: ProposalLineageStatus::Rejected,
3092 rejection_reason: Some(reason.clone()),
3093 });
3094 let mut log = self.log.lock().await;
3095 log.append(
3096 EventKind::ReplanRejected,
3097 None,
3098 Some(&proposal.id),
3099 [
3100 ("errors".to_string(), Value::from(reason)),
3101 ("attempt".to_string(), Value::from(generation)),
3102 ]
3103 .into(),
3104 );
3105 drop(log);
3106 continue 'replan_candidates;
3107 }
3108 }
3109
3110 lineage.push(ProposalLineageEntry {
3111 generation,
3112 proposal_id: new_proposal.id.clone(),
3113 proposal_digest: Some(new_proposal_digest.clone()),
3114 status: ProposalLineageStatus::Accepted,
3115 rejection_reason: None,
3116 });
3117 accepted_proposal_preimages.push(AcceptedProposalPreimage {
3118 generation,
3119 proposal_digest: new_proposal_digest.clone(),
3120 proposal: new_proposal.clone(),
3121 });
3122
3123 // Log accepted proposal
3124 {
3125 let mut log = self.log.lock().await;
3126 log.append(
3127 EventKind::ReplanProposalReceived,
3128 None,
3129 Some(&proposal.id),
3130 [
3131 ("attempt".to_string(), Value::from(generation)),
3132 (
3133 "proposal_id".to_string(),
3134 Value::from(new_proposal.id.as_str()),
3135 ),
3136 (
3137 "proposal_digest".to_string(),
3138 Value::from(new_proposal_digest),
3139 ),
3140 ("proposal".to_string(), new_proposal_preimage),
3141 (
3142 "new_action_count".to_string(),
3143 Value::from(new_proposal.actions.len()),
3144 ),
3145 ]
3146 .into(),
3147 );
3148 }
3149 current_proposal = new_proposal;
3150 break 'replan_candidates;
3151 }
3152 Err(e) => {
3153 // Replan callback itself failed — log and return original failure
3154 let mut log = self.log.lock().await;
3155 log.append(
3156 EventKind::ReplanExhausted,
3157 None,
3158 Some(&proposal.id),
3159 [
3160 ("reason".to_string(), Value::from("callback_error")),
3161 ("error".to_string(), Value::from(e.as_str())),
3162 ("attempt".to_string(), Value::from(generation)),
3163 ]
3164 .into(),
3165 );
3166 if let Some(err) = self.persist_trajectory(
3167 proposal,
3168 ¤t_proposal,
3169 &result,
3170 car_memgine::TrajectoryOutcome::Failed,
3171 attempt,
3172 &state_before_map,
3173 ) {
3174 log.append(
3175 EventKind::ActionFailed,
3176 None,
3177 Some(&proposal.id),
3178 [(
3179 "trajectory_persist_error".to_string(),
3180 Value::from(err.as_str()),
3181 )]
3182 .into(),
3183 );
3184 }
3185 return finalize_proposal_result(
3186 result,
3187 &proposal.id,
3188 &lineage,
3189 &accepted_proposal_preimages,
3190 );
3191 }
3192 }
3193 }
3194 }
3195 }
3196
3197 /// Persist a trajectory to the store if configured.
3198 fn persist_trajectory(
3199 &self,
3200 proposal: &ActionProposal,
3201 current_proposal: &ActionProposal,
3202 result: &ProposalResult,
3203 outcome: car_memgine::TrajectoryOutcome,
3204 attempt: u32,
3205 state_before_map: &HashMap<String, HashMap<String, Value>>,
3206 ) -> Option<String> {
3207 let store = self.trajectory_store.as_ref()?;
3208
3209 let trace_events: Vec<car_memgine::TraceEvent> = result
3210 .results
3211 .iter()
3212 .map(|r| {
3213 let kind = match r.status {
3214 ActionStatus::Succeeded => "action_succeeded",
3215 ActionStatus::Failed => "action_failed",
3216 ActionStatus::Rejected => "action_rejected",
3217 ActionStatus::Skipped => "action_skipped",
3218 _ => "unknown",
3219 };
3220 let tool = current_proposal
3221 .actions
3222 .iter()
3223 .find(|a| a.id == r.action_id)
3224 .and_then(|a| a.tool.clone());
3225 let reward = match r.status {
3226 ActionStatus::Succeeded => Some(1.0),
3227 ActionStatus::Failed => Some(0.0),
3228 ActionStatus::Rejected => Some(0.0),
3229 ActionStatus::Skipped => None,
3230 _ => None,
3231 };
3232 car_memgine::TraceEvent {
3233 kind: kind.to_string(),
3234 action_id: Some(r.action_id.clone()),
3235 tool,
3236 data: r
3237 .error
3238 .as_ref()
3239 .map(|e| serde_json::json!({"error": e}))
3240 .unwrap_or(serde_json::json!({})),
3241 duration_ms: r.duration_ms,
3242 state_before: state_before_map.get(&r.action_id).cloned(),
3243 state_after: if !r.state_changes.is_empty() {
3244 Some(r.state_changes.clone())
3245 } else {
3246 None
3247 },
3248 reward,
3249 }
3250 })
3251 .collect();
3252
3253 let trajectory = car_memgine::Trajectory {
3254 proposal_id: proposal.id.clone(),
3255 source: proposal.source.clone(),
3256 action_count: current_proposal.actions.len(),
3257 events: trace_events,
3258 outcome,
3259 timestamp: chrono::Utc::now(),
3260 duration_ms: result.cost.total_duration_ms,
3261 replan_attempts: attempt,
3262 };
3263
3264 match store.append(&trajectory) {
3265 Ok(()) => None,
3266 Err(e) => Some(e.to_string()),
3267 }
3268 }
3269
3270 async fn rollback_failed_plan_candidate(
3271 &self,
3272 proposal: &ActionProposal,
3273 result: &mut ProposalResult,
3274 pre_plan_snapshot: &HashMap<String, Value>,
3275 pre_plan_transitions: usize,
3276 pre_plan_idempotency_keys: &std::collections::HashSet<String>,
3277 ) -> Result<(), String> {
3278 let executed_proposal = result.final_proposal.as_ref().unwrap_or(proposal);
3279 let executed_proposal_digest = result
3280 .accepted_proposal_preimages
3281 .last()
3282 .map(|accepted| accepted.proposal_digest.clone())
3283 .unwrap_or_else(|| {
3284 proposal_digest(executed_proposal)
3285 .expect("an executed planning candidate already passed proposal identity")
3286 });
3287 let observed_changes_by_action: std::collections::BTreeMap<String, HashMap<String, Value>> =
3288 result
3289 .results
3290 .iter()
3291 .filter(|action_result| {
3292 action_result.status == ActionStatus::Succeeded
3293 && !action_result.state_changes.is_empty()
3294 })
3295 .map(|action_result| {
3296 (
3297 action_result.action_id.clone(),
3298 action_result.state_changes.clone(),
3299 )
3300 })
3301 .collect();
3302 let affected_actions: Vec<String> = observed_changes_by_action.keys().cloned().collect();
3303
3304 // The planning transaction owns the StateStore proposal guard, so this
3305 // exact snapshot/count restore cannot erase another proposal's commit.
3306 let durability = match self
3307 .state
3308 .restore(pre_plan_snapshot.clone(), pre_plan_transitions)
3309 {
3310 Ok(durability) => durability,
3311 Err(error) => {
3312 let detail = record_rollback_durability_error(
3313 &mut result.results,
3314 ROLLBACK_DURABILITY_ERROR,
3315 &error.to_string(),
3316 );
3317 self.log.lock().await.append(
3318 EventKind::ActionFailed,
3319 None,
3320 Some(&proposal.id),
3321 [
3322 ("error".to_string(), Value::from(detail.clone())),
3323 ("attempted".to_string(), Value::from(true)),
3324 ("publication_succeeded".to_string(), Value::from(false)),
3325 ("rollback_succeeded".to_string(), Value::from(false)),
3326 ("stage".to_string(), Value::from("plan_fallback_rollback")),
3327 ]
3328 .into(),
3329 );
3330 return Err(detail);
3331 }
3332 };
3333 let durability_error = match &durability {
3334 RestoreDurability::Durable => None,
3335 RestoreDurability::DurabilityUnknown { error } => Some(error.clone()),
3336 };
3337
3338 {
3339 let mut log = self.log.lock().await;
3340 log.append(
3341 EventKind::StateSnapshot,
3342 None,
3343 Some(&proposal.id),
3344 [(
3345 "state".to_string(),
3346 serde_json::to_value(pre_plan_snapshot).unwrap_or_default(),
3347 )]
3348 .into(),
3349 );
3350 log.append(
3351 EventKind::StateRollback,
3352 None,
3353 Some(&proposal.id),
3354 [
3355 (
3356 "rolled_back_to".to_string(),
3357 Value::from("pre-plan snapshot"),
3358 ),
3359 (
3360 "affected_actions".to_string(),
3361 serde_json::to_value(&affected_actions).unwrap_or_default(),
3362 ),
3363 (
3364 "rolled_back_changes".to_string(),
3365 serde_json::to_value(&observed_changes_by_action).unwrap_or_default(),
3366 ),
3367 (
3368 "changes_semantics".to_string(),
3369 Value::from("rolled_back_failed_planning_candidate"),
3370 ),
3371 (
3372 "proposal_digest".to_string(),
3373 Value::from(executed_proposal_digest),
3374 ),
3375 ("attempted".to_string(), Value::from(true)),
3376 (
3377 "durability_unknown".to_string(),
3378 Value::from(durability_error.is_some()),
3379 ),
3380 ("publication_succeeded".to_string(), Value::from(true)),
3381 ("rollback_succeeded".to_string(), Value::from(true)),
3382 ("stage".to_string(), Value::from("plan_fallback_rollback")),
3383 ]
3384 .into(),
3385 );
3386 }
3387
3388 // Only invalidate entries introduced by this planning transaction.
3389 // Pre-existing keys may have been read by a candidate, but their
3390 // committed result predates the pre-plan snapshot and must survive.
3391 let mut invalidated = std::collections::BTreeSet::new();
3392 {
3393 let mut cache = self.idempotency_cache.lock().await;
3394 for action_result in &result.results {
3395 if action_result.status != ActionStatus::Succeeded {
3396 continue;
3397 }
3398 let Some(action) = executed_proposal
3399 .actions
3400 .iter()
3401 .find(|action| action.id == action_result.action_id)
3402 else {
3403 continue;
3404 };
3405 if !action.idempotent || action.invocation_mode.is_detached() {
3406 continue;
3407 }
3408 let key = idempotency_key(action, None);
3409 if !pre_plan_idempotency_keys.contains(&key) && cache.remove(&key).is_some() {
3410 invalidated.insert(key);
3411 }
3412 }
3413 }
3414 for key in invalidated {
3415 self.journal_idempotency(&key, None).await;
3416 }
3417
3418 // Same reasoning as the abort path above: a rejected planning
3419 // candidate loses its state effects, not the record of what ran. This
3420 // result is observable — `plan_and_execute` returns it directly on a
3421 // rollback durability error, and otherwise keeps it as
3422 // `first_failure` (car#1157). `rolled_back` is the machine-readable
3423 // fact; the warning remains descriptive and must never drive logic.
3424 for action_result in &mut result.results {
3425 if action_result.status == ActionStatus::Succeeded {
3426 action_result.rolled_back = true;
3427 action_result.error = Some(PLAN_FALLBACK_ROLLBACK_WARNING.to_string());
3428 action_result.state_changes.clear();
3429 }
3430 }
3431 if let Some(error) = durability_error {
3432 return Err(record_rollback_durability_error(
3433 &mut result.results,
3434 ROLLBACK_DURABILITY_UNKNOWN,
3435 &error,
3436 ));
3437 }
3438 Ok(())
3439 }
3440
3441 /// Score N candidate proposals, execute the best valid one, fall back to
3442 /// next-best on failure. Combines car-planner scoring with engine execution.
3443 ///
3444 /// Returns the result from whichever proposal was executed (best or fallback).
3445 /// If all candidates fail verification, returns an error result for the first.
3446 pub async fn plan_and_execute(
3447 &self,
3448 candidates: &[ActionProposal],
3449 planner_config: Option<car_planner::PlannerConfig>,
3450 feedback: Option<&car_planner::ToolFeedback>,
3451 ) -> ProposalResult {
3452 if candidates.is_empty() {
3453 return ProposalResult::new("empty", vec![], car_ir::CostSummary::default());
3454 }
3455
3456 // One transaction guard covers ranking's state snapshot, every
3457 // candidate attempt and rollback, and the accepted candidate's commit.
3458 // Candidate execution calls the already-guarded path to avoid recursive
3459 // acquisition of the non-reentrant mutex.
3460 let _proposal_execution = self.state.lock_proposal_execution().await;
3461
3462 // Score all candidates
3463 let planner = car_planner::Planner::new(planner_config.unwrap_or_default());
3464 let tools_guard = self.tools.read().await;
3465 let tool_names: std::collections::HashSet<String> = tools_guard.keys().cloned().collect();
3466 drop(tools_guard);
3467
3468 let pre_plan_snapshot = self.state.snapshot();
3469 let pre_plan_transitions = self.state.transition_count();
3470 let pre_plan_idempotency_keys: std::collections::HashSet<String> = self
3471 .idempotency_cache
3472 .lock()
3473 .await
3474 .keys()
3475 .cloned()
3476 .collect();
3477 let ranked = planner.rank_with_feedback(
3478 candidates,
3479 Some(&pre_plan_snapshot),
3480 Some(&tool_names),
3481 feedback,
3482 );
3483
3484 // Try each valid candidate in score order
3485 let mut first_failure: Option<ProposalResult> = None;
3486 for scored in &ranked {
3487 if !scored.valid {
3488 continue;
3489 }
3490
3491 let proposal = &candidates[scored.index];
3492 let cancel = tokio_util::sync::CancellationToken::new();
3493 let mut result = self
3494 .execute_with_optional_session_already_guarded(proposal, None, None, None, &cancel)
3495 .await;
3496
3497 if result.all_succeeded() {
3498 return result;
3499 }
3500
3501 if self
3502 .rollback_failed_plan_candidate(
3503 proposal,
3504 &mut result,
3505 &pre_plan_snapshot,
3506 pre_plan_transitions,
3507 &pre_plan_idempotency_keys,
3508 )
3509 .await
3510 .is_err()
3511 {
3512 return result;
3513 }
3514
3515 tracing::info!(
3516 proposal_id = %proposal.id,
3517 score = scored.score,
3518 "plan_and_execute: proposal failed, trying next candidate"
3519 );
3520
3521 if first_failure.is_none() {
3522 first_failure = Some(result);
3523 }
3524 }
3525
3526 // Return the first failure result (don't re-execute — avoids duplicate side effects)
3527 first_failure.unwrap_or_else(|| {
3528 ProposalResult::for_proposal(&candidates[0], vec![], car_ir::CostSummary::default())
3529 })
3530 }
3531
3532 /// Execute a single proposal through the runtime loop (no replanning).
3533 /// Returns (result, state_before_map) where state_before_map has per-action snapshots.
3534 ///
3535 /// `session_id`, when `Some`, scopes per-action policy validation
3536 /// to the named session in addition to global policies. Both
3537 /// layers must pass for an action to run; the session layer cannot
3538 /// loosen what global denies.
3539 async fn execute_inner_with_cancel(
3540 &self,
3541 proposal: &ActionProposal,
3542 cancel: Option<&tokio_util::sync::CancellationToken>,
3543 session_id: Option<&str>,
3544 scope: Option<&crate::scope::RuntimeScope>,
3545 ) -> (ProposalResult, HashMap<String, HashMap<String, Value>>) {
3546 // Identity is established before CAR admits the proposal or emits any
3547 // action transition. In particular, an I-JSON-incompatible number may
3548 // not produce a journal row whose mandatory JCS digest is missing.
3549 let (proposal_preimage, proposal_digest) = match proposal_journal_identity(proposal) {
3550 Ok(identity) => identity,
3551 Err(error) => {
3552 let results = proposal
3553 .actions
3554 .iter()
3555 .map(|action| rejected_result(&action.id, error.clone()))
3556 .collect();
3557 self.log.lock().await.append(
3558 EventKind::StateRollback,
3559 None,
3560 Some(&proposal.id),
3561 proposal_rejection_boundary_data(proposal, None, &error),
3562 );
3563 return (
3564 ProposalResult::for_proposal(proposal, results, CostSummary::default()),
3565 HashMap::new(),
3566 );
3567 }
3568 };
3569
3570 // Whether validator/policy/capability rejections should count toward
3571 // the abort-and-replan path (default false). Read once up front so the
3572 // per-action loop doesn't take the replan_config lock repeatedly (and
3573 // never inside join_all). Independent lock from log/tools/policies/
3574 // capabilities, so no lock-ordering conflict.
3575 let replan_on_rejected = self.replan_config.read().await.replan_on_rejected;
3576
3577 // Generate trace_id for this proposal execution
3578 let trace_id = Uuid::new_v4().to_string();
3579
3580 // Begin root span for proposal execution
3581 let root_span_id = {
3582 let mut log = self.log.lock().await;
3583 log.begin_span(
3584 "proposal.execute",
3585 &trace_id,
3586 None,
3587 [("proposal_id".to_string(), Value::from(proposal.id.as_str()))].into(),
3588 )
3589 };
3590
3591 // Log proposal received
3592 {
3593 let mut log = self.log.lock().await;
3594 log.append(
3595 EventKind::ProposalReceived,
3596 None,
3597 Some(&proposal.id),
3598 [
3599 ("source".to_string(), Value::from(proposal.source.as_str())),
3600 (
3601 "action_count".to_string(),
3602 Value::from(proposal.actions.len()),
3603 ),
3604 ("proposal".to_string(), proposal_preimage.clone()),
3605 (
3606 "proposal_digest".to_string(),
3607 Value::from(proposal_digest.clone()),
3608 ),
3609 ]
3610 .into(),
3611 );
3612 }
3613
3614 // Capability check: max_actions budget for entire proposal
3615 {
3616 let caps = self.capabilities.read().await;
3617 if let Some(ref cap) = *caps {
3618 if !cap.actions_within_budget(proposal.actions.len() as u32) {
3619 let reason = format!(
3620 "capability denied: proposal has {} actions, max allowed is {:?}",
3621 proposal.actions.len(),
3622 cap.max_actions
3623 );
3624 let mut action_results = Vec::new();
3625 let mut log = self.log.lock().await;
3626 for action in &proposal.actions {
3627 log.append(
3628 EventKind::ActionRejected,
3629 Some(&action.id),
3630 Some(&proposal.id),
3631 action_outcome_data("proposal_capability", &reason, false, None),
3632 );
3633 action_results.push(rejected_result(&action.id, reason.clone()));
3634 }
3635 log.append(
3636 EventKind::StateRollback,
3637 None,
3638 Some(&proposal.id),
3639 proposal_rejection_boundary_data(proposal, Some(&proposal_digest), &reason),
3640 );
3641 drop(log);
3642 return (
3643 ProposalResult::for_proposal(
3644 proposal,
3645 action_results,
3646 CostSummary::default(),
3647 ),
3648 HashMap::new(),
3649 );
3650 }
3651 }
3652 }
3653
3654 // Snapshot for rollback. When the proposal carries a tenant scope,
3655 // snapshot only that tenant's namespace so a rollback can't clobber
3656 // concurrent tenants' state (EPIC E / E2). No tenant → unchanged
3657 // full snapshot.
3658 let rollback_tenant: Option<&str> = scope.and_then(|s| s.tenant_id.as_deref());
3659 let snapshot = match rollback_tenant {
3660 Some(_) => self.state.snapshot_scoped(rollback_tenant),
3661 None => self.state.snapshot(),
3662 };
3663 let transition_count = self.state.transition_count();
3664
3665 let mut results: Vec<ActionResult> = Vec::new();
3666 // Per-action state snapshots captured before execution (for TraceEvent.state_before).
3667 let mut state_before_map: HashMap<String, HashMap<String, Value>> = HashMap::new();
3668 let mut aborted = false;
3669 let mut budget_exceeded = false;
3670 let mut total_retries: u32 = 0;
3671
3672 // Running cost counters for budget enforcement
3673 let mut running_tool_calls: u32 = 0;
3674 let mut running_actions: u32 = 0;
3675 let mut running_duration_ms: f64 = 0.0;
3676
3677 // Snapshot the budget once
3678 let budget = self.cost_budget.read().await.clone();
3679
3680 // Build DAG
3681 let levels = build_dag(&proposal.actions);
3682
3683 let mut canceled = false;
3684 for level in &levels {
3685 // Cooperative cancellation check at the level boundary.
3686 // Actions already in flight aren't interrupted (we can't
3687 // safely cancel a tool call dispatched to a user-provided
3688 // executor), but every action that hadn't started runs
3689 // is recorded as canceled with a clear reason.
3690 if !canceled {
3691 if let Some(token) = cancel {
3692 if token.is_cancelled() {
3693 canceled = true;
3694 }
3695 }
3696 }
3697 if canceled {
3698 for &idx in level {
3699 let action = &proposal.actions[idx];
3700 let reason = format!("{CANCELED_PREFIX}cancellation requested by caller");
3701 self.log.lock().await.append(
3702 EventKind::ActionSkipped,
3703 Some(&action.id),
3704 Some(&proposal.id),
3705 action_outcome_data("cancellation", &reason, false, None),
3706 );
3707 results.push(canceled_result(
3708 &action.id,
3709 "cancellation requested by caller",
3710 ));
3711 }
3712 continue;
3713 }
3714 if aborted || budget_exceeded {
3715 let skip_reason = if budget_exceeded {
3716 "cost budget exceeded"
3717 } else {
3718 "skipped due to earlier abort"
3719 };
3720 for &idx in level {
3721 let action = &proposal.actions[idx];
3722 let stage = if budget_exceeded {
3723 "cost_budget"
3724 } else {
3725 "dependency_abort"
3726 };
3727 self.log.lock().await.append(
3728 EventKind::ActionSkipped,
3729 Some(&action.id),
3730 Some(&proposal.id),
3731 action_outcome_data(stage, skip_reason, false, None),
3732 );
3733 results.push(skipped_result(&action.id, skip_reason));
3734 }
3735 continue;
3736 }
3737
3738 // Check if any action in this level has ABORT behavior
3739 let has_abort = level
3740 .iter()
3741 .any(|&i| proposal.actions[i].failure_behavior == FailureBehavior::Abort);
3742
3743 if level.len() == 1 || has_abort {
3744 // Sequential execution
3745 for &idx in level {
3746 if aborted || budget_exceeded {
3747 let skip_reason = if budget_exceeded {
3748 "cost budget exceeded"
3749 } else {
3750 "skipped due to abort"
3751 };
3752 let action = &proposal.actions[idx];
3753 let stage = if budget_exceeded {
3754 "cost_budget"
3755 } else {
3756 "dependency_abort"
3757 };
3758 self.log.lock().await.append(
3759 EventKind::ActionSkipped,
3760 Some(&action.id),
3761 Some(&proposal.id),
3762 action_outcome_data(stage, skip_reason, false, None),
3763 );
3764 results.push(skipped_result(&action.id, skip_reason));
3765 continue;
3766 }
3767
3768 // Budget check before execution
3769 if let Some(ref b) = budget {
3770 if let Some(max) = b.max_actions {
3771 if running_actions >= max {
3772 budget_exceeded = true;
3773 self.log.lock().await.append(
3774 EventKind::ActionSkipped,
3775 Some(&proposal.actions[idx].id),
3776 Some(&proposal.id),
3777 action_outcome_data(
3778 "cost_budget",
3779 "cost budget exceeded",
3780 false,
3781 None,
3782 ),
3783 );
3784 results.push(skipped_result(
3785 &proposal.actions[idx].id,
3786 "cost budget exceeded",
3787 ));
3788 continue;
3789 }
3790 }
3791 if let Some(max) = b.max_tool_calls {
3792 if proposal.actions[idx].action_type == ActionType::ToolCall
3793 && running_tool_calls >= max
3794 {
3795 budget_exceeded = true;
3796 self.log.lock().await.append(
3797 EventKind::ActionSkipped,
3798 Some(&proposal.actions[idx].id),
3799 Some(&proposal.id),
3800 action_outcome_data(
3801 "cost_budget",
3802 "cost budget exceeded",
3803 false,
3804 None,
3805 ),
3806 );
3807 results.push(skipped_result(
3808 &proposal.actions[idx].id,
3809 "cost budget exceeded",
3810 ));
3811 continue;
3812 }
3813 }
3814 if let Some(max) = b.max_duration_ms {
3815 if running_duration_ms >= max {
3816 budget_exceeded = true;
3817 self.log.lock().await.append(
3818 EventKind::ActionSkipped,
3819 Some(&proposal.actions[idx].id),
3820 Some(&proposal.id),
3821 action_outcome_data(
3822 "cost_budget",
3823 "cost budget exceeded",
3824 false,
3825 None,
3826 ),
3827 );
3828 results.push(skipped_result(
3829 &proposal.actions[idx].id,
3830 "cost budget exceeded",
3831 ));
3832 continue;
3833 }
3834 }
3835 }
3836
3837 state_before_map.insert(
3838 proposal.actions[idx].id.clone(),
3839 snapshot_relevant_keys(&self.state, &proposal.actions[idx]),
3840 );
3841 let (ar, action_retries) = self
3842 .process_action(
3843 &proposal.actions[idx],
3844 &proposal.id,
3845 &trace_id,
3846 &root_span_id,
3847 session_id,
3848 scope,
3849 )
3850 .await;
3851 total_retries += action_retries;
3852
3853 // Update running counters
3854 if ar.status == ActionStatus::Succeeded
3855 && proposal.actions[idx].action_type == ActionType::ToolCall
3856 {
3857 running_tool_calls += 1;
3858 }
3859 if ar.status != ActionStatus::Skipped {
3860 running_actions += 1;
3861 }
3862 if let Some(d) = ar.duration_ms {
3863 running_duration_ms += d;
3864 }
3865
3866 if ar.terminal
3867 || requires_integrity_rollback(&ar)
3868 || ((ar.status == ActionStatus::Failed
3869 || (replan_on_rejected && ar.status == ActionStatus::Rejected))
3870 && proposal.actions[idx].failure_behavior == FailureBehavior::Abort)
3871 {
3872 aborted = true;
3873 }
3874 results.push(ar);
3875 }
3876 } else {
3877 // Concurrent execution via futures::join_all
3878 // Snapshot only relevant keys per action (all see same pre-level state)
3879 for &idx in level {
3880 state_before_map.insert(
3881 proposal.actions[idx].id.clone(),
3882 snapshot_relevant_keys(&self.state, &proposal.actions[idx]),
3883 );
3884 }
3885 let futs: Vec<_> = level
3886 .iter()
3887 .map(|&idx| {
3888 self.process_action(
3889 &proposal.actions[idx],
3890 &proposal.id,
3891 &trace_id,
3892 &root_span_id,
3893 session_id,
3894 scope,
3895 )
3896 })
3897 .collect();
3898 let level_results = futures::future::join_all(futs).await;
3899
3900 for (i, (ar, action_retries)) in level_results.into_iter().enumerate() {
3901 let idx = level[i];
3902 total_retries += action_retries;
3903 if ar.status == ActionStatus::Succeeded
3904 && proposal.actions[idx].action_type == ActionType::ToolCall
3905 {
3906 running_tool_calls += 1;
3907 }
3908 if ar.status != ActionStatus::Skipped {
3909 running_actions += 1;
3910 }
3911 if let Some(d) = ar.duration_ms {
3912 running_duration_ms += d;
3913 }
3914 if ar.terminal || requires_integrity_rollback(&ar) {
3915 aborted = true;
3916 }
3917 results.push(ar);
3918 }
3919 }
3920 }
3921
3922 let observed_changes_by_action: std::collections::BTreeMap<String, HashMap<String, Value>> =
3923 proposal
3924 .actions
3925 .iter()
3926 .filter_map(|action| {
3927 results
3928 .iter()
3929 .find(|result| {
3930 result.action_id == action.id
3931 && result.status == ActionStatus::Succeeded
3932 && !result.state_changes.is_empty()
3933 })
3934 .map(|result| (action.id.clone(), result.state_changes.clone()))
3935 })
3936 .collect();
3937 let affected_actions: Vec<String> = observed_changes_by_action.keys().cloned().collect();
3938
3939 // Handle rollback. Per-action StateChanged rows are provisional until
3940 // this proposal-level transaction boundary is durably recorded.
3941 if aborted {
3942 // Tenant-scoped rollback when the proposal is tenant-scoped
3943 // (EPIC E / E2), else the full restore.
3944 let rollback = match rollback_tenant {
3945 Some(_) => {
3946 self.state
3947 .restore_scoped(rollback_tenant, snapshot.clone(), transition_count)
3948 }
3949 None => self.state.restore(snapshot.clone(), transition_count),
3950 };
3951
3952 let rollback_durability = match rollback {
3953 Ok(durability) => Some(durability),
3954 Err(error) => {
3955 let detail = record_rollback_durability_error(
3956 &mut results,
3957 ROLLBACK_DURABILITY_ERROR,
3958 &error.to_string(),
3959 );
3960 self.log.lock().await.append(
3961 EventKind::ActionFailed,
3962 None,
3963 Some(&proposal.id),
3964 [
3965 ("error".to_string(), Value::from(detail)),
3966 ("attempted".to_string(), Value::from(true)),
3967 ("publication_succeeded".to_string(), Value::from(false)),
3968 ("in_memory_state_preserved".to_string(), Value::from(true)),
3969 (
3970 "idempotency_entries_preserved".to_string(),
3971 Value::from(true),
3972 ),
3973 ("rollback_succeeded".to_string(), Value::from(false)),
3974 ("stage".to_string(), Value::from("proposal_rollback")),
3975 ]
3976 .into(),
3977 );
3978 None
3979 }
3980 };
3981
3982 if let Some(durability) = rollback_durability {
3983 let durability_error = match &durability {
3984 RestoreDurability::Durable => None,
3985 RestoreDurability::DurabilityUnknown { error } => Some(error.clone()),
3986 };
3987 if let Some(error) = &durability_error {
3988 record_rollback_durability_error(
3989 &mut results,
3990 ROLLBACK_DURABILITY_UNKNOWN,
3991 error,
3992 );
3993 }
3994 let mut log = self.log.lock().await;
3995 log.append(
3996 EventKind::StateSnapshot,
3997 None,
3998 Some(&proposal.id),
3999 [(
4000 "state".to_string(),
4001 serde_json::to_value(&snapshot).unwrap_or_default(),
4002 )]
4003 .into(),
4004 );
4005 log.append(
4006 EventKind::StateRollback,
4007 None,
4008 Some(&proposal.id),
4009 [
4010 (
4011 "rolled_back_to".to_string(),
4012 Value::from("pre-proposal snapshot"),
4013 ),
4014 (
4015 "affected_actions".to_string(),
4016 serde_json::to_value(&affected_actions).unwrap_or_default(),
4017 ),
4018 (
4019 "rolled_back_changes".to_string(),
4020 serde_json::to_value(&observed_changes_by_action).unwrap_or_default(),
4021 ),
4022 (
4023 "changes_semantics".to_string(),
4024 Value::from("rolled_back_provisional_state_mutations"),
4025 ),
4026 (
4027 "proposal_digest".to_string(),
4028 Value::from(proposal_digest.clone()),
4029 ),
4030 ("attempted".to_string(), Value::from(true)),
4031 (
4032 "durability_unknown".to_string(),
4033 Value::from(durability_error.is_some()),
4034 ),
4035 ("publication_succeeded".to_string(), Value::from(true)),
4036 ("rollback_succeeded".to_string(), Value::from(true)),
4037 ("stage".to_string(), Value::from("proposal_rollback")),
4038 ]
4039 .into(),
4040 );
4041 drop(log);
4042
4043 // Clear idempotency cache for rolled-back actions only after
4044 // the state journal has durably accepted the rollback.
4045 let mut invalidated: Vec<String> = Vec::new();
4046 {
4047 let mut cache = self.idempotency_cache.lock().await;
4048 for r in &results {
4049 if r.status == ActionStatus::Succeeded {
4050 for action in &proposal.actions {
4051 if action.id == r.action_id && action.idempotent {
4052 let key = idempotency_key(action, scope);
4053 cache.remove(&key);
4054 invalidated.push(key);
4055 }
4056 }
4057 }
4058 }
4059 }
4060 // Record tombstones so the durable journal (C3) doesn't
4061 // resurrect a rolled-back result on reload.
4062 for key in &invalidated {
4063 self.journal_idempotency(key, None).await;
4064 }
4065 // The proposal did not commit, so no action's state changes
4066 // survive. Each action's STATUS still records whether that
4067 // action executed; `rolled_back` independently records whether
4068 // that successful execution preceded this rollback. Folding
4069 // the second fact into ActionStatus erased which action
4070 // aborted the proposal and marked `failed` precisely the
4071 // actions whose external effects DID happen and may remain.
4072 // The warning is human-readable context, never control flow
4073 // (car#1157).
4074 for result in &mut results {
4075 if result.status == ActionStatus::Succeeded {
4076 result.rolled_back = true;
4077 result.error = Some(ROLLBACK_WARNING.to_string());
4078 result.state_changes.clear();
4079 }
4080 }
4081 }
4082 } else {
4083 self.log.lock().await.append(
4084 EventKind::StateCommitted,
4085 None,
4086 Some(&proposal.id),
4087 [
4088 (
4089 "affected_actions".to_string(),
4090 serde_json::to_value(&affected_actions).unwrap_or_default(),
4091 ),
4092 (
4093 "committed_changes".to_string(),
4094 serde_json::to_value(&observed_changes_by_action).unwrap_or_default(),
4095 ),
4096 (
4097 "changes_semantics".to_string(),
4098 Value::from("committed_provisional_state_mutations"),
4099 ),
4100 ("proposal_digest".to_string(), Value::from(proposal_digest)),
4101 ("attempted".to_string(), Value::from(true)),
4102 ("stage".to_string(), Value::from("proposal_commit")),
4103 ]
4104 .into(),
4105 );
4106 }
4107
4108 // Sort results to match original action order
4109 let action_order: HashMap<String, usize> = proposal
4110 .actions
4111 .iter()
4112 .enumerate()
4113 .map(|(i, a)| (a.id.clone(), i))
4114 .collect();
4115 results.sort_by_key(|r| {
4116 action_order
4117 .get(&r.action_id)
4118 .copied()
4119 .unwrap_or(usize::MAX)
4120 });
4121
4122 // Compute cost summary from results
4123 let mut cost = CostSummary::default();
4124 for r in &results {
4125 let action = action_order
4126 .get(&r.action_id)
4127 .and_then(|&i| proposal.actions.get(i));
4128 match r.status {
4129 ActionStatus::Succeeded => {
4130 cost.actions_executed += 1;
4131 if let Some(a) = action {
4132 if a.action_type == ActionType::ToolCall {
4133 cost.tool_calls += 1;
4134 }
4135 }
4136 }
4137 // A failed action ran — the tool was invoked and errored — so it
4138 // belongs in `actions_executed`. A rejected one was blocked by
4139 // the validator or a policy and never started; counting it as
4140 // executed reported work that never happened (car#624).
4141 ActionStatus::Failed => {
4142 cost.actions_executed += 1;
4143 }
4144 ActionStatus::Rejected => {
4145 cost.actions_rejected += 1;
4146 }
4147 ActionStatus::Skipped => {
4148 cost.actions_skipped += 1;
4149 }
4150 _ => {}
4151 }
4152 if let Some(d) = r.duration_ms {
4153 cost.total_duration_ms += d;
4154 }
4155 }
4156
4157 // Set retries from inline counter
4158 cost.retries = total_retries;
4159
4160 // End root span — Ok if no abort, Error if aborted
4161 {
4162 let span_status = if aborted {
4163 SpanStatus::Error
4164 } else {
4165 SpanStatus::Ok
4166 };
4167 let mut log = self.log.lock().await;
4168 log.end_span(&root_span_id, span_status);
4169 }
4170
4171 let proposal_result = ProposalResult::for_proposal(proposal, results, cost);
4172
4173 // Post-execution: auto-distill skills from this execution trace
4174 if self.auto_distill {
4175 if let Some(ref memgine) = self.memgine {
4176 // Convert results to TraceEvents for distillation
4177 let trace_events: Vec<car_memgine::TraceEvent> = proposal_result
4178 .results
4179 .iter()
4180 .map(|r| {
4181 let kind = match r.status {
4182 ActionStatus::Succeeded => "action_succeeded",
4183 ActionStatus::Failed => "action_failed",
4184 ActionStatus::Rejected => "action_rejected",
4185 ActionStatus::Skipped => "action_skipped",
4186 _ => "unknown",
4187 };
4188 // Find the matching action to get the tool name
4189 let tool = proposal
4190 .actions
4191 .iter()
4192 .find(|a| a.id == r.action_id)
4193 .and_then(|a| a.tool.clone());
4194 let mut data = serde_json::Map::new();
4195 if let Some(ref e) = r.error {
4196 data.insert("error".into(), Value::from(e.as_str()));
4197 }
4198 if let Some(ref o) = r.output {
4199 data.insert("output".into(), o.clone());
4200 }
4201 car_memgine::TraceEvent {
4202 kind: kind.to_string(),
4203 action_id: Some(r.action_id.clone()),
4204 tool,
4205 data: Value::Object(data),
4206 duration_ms: r.duration_ms,
4207 reward: match r.status {
4208 ActionStatus::Succeeded => Some(1.0),
4209 ActionStatus::Failed | ActionStatus::Rejected => Some(0.0),
4210 _ => None,
4211 },
4212 ..Default::default()
4213 }
4214 })
4215 .collect();
4216
4217 let mut engine = memgine.lock().await;
4218 let skills = engine.distill_skills(&trace_events).await;
4219 if !skills.is_empty() {
4220 let count = skills.len();
4221 // Validation-gated ingest (SkillOpt-inspired). Distilled
4222 // skills are NOT trusted straight into the active pool; each
4223 // enters as a PROVISIONAL candidate on trial and must prove
4224 // itself (or beat its incumbent) before the promotion gate in
4225 // consolidate() makes it Active. Tenant is threaded through so
4226 // candidates are stamped for the calling tenant's namespace.
4227 let tenant = scope.and_then(|s| s.tenant_id.as_deref());
4228 let provisional = engine.ingest_provisional_candidates(&skills, tenant);
4229
4230 // Log the distillation event
4231 let mut log = self.log.lock().await;
4232 log.append(
4233 EventKind::SkillDistilled,
4234 None,
4235 Some(&proposal_result.proposal_id),
4236 [
4237 ("skills_count".to_string(), Value::from(count)),
4238 ("provisional_ingested".to_string(), Value::from(provisional)),
4239 (
4240 "skill_names".to_string(),
4241 Value::from(
4242 skills.iter().map(|s| s.name.as_str()).collect::<Vec<_>>(),
4243 ),
4244 ),
4245 ]
4246 .into(),
4247 );
4248
4249 // Check if any domains need evolution
4250 let threshold = engine.evolution_threshold();
4251 let domains = engine.domains_needing_evolution(threshold);
4252 for domain in &domains {
4253 // Collect failed events for this domain
4254 let failed: Vec<car_memgine::TraceEvent> = trace_events
4255 .iter()
4256 .filter(|e| {
4257 matches!(e.kind.as_str(), "action_failed" | "action_rejected")
4258 })
4259 .cloned()
4260 .collect();
4261 if !failed.is_empty() {
4262 let evolved = engine.evolve_skills(&failed, domain).await;
4263 if !evolved.is_empty() {
4264 log.append(
4265 EventKind::EvolutionTriggered,
4266 None,
4267 Some(&proposal_result.proposal_id),
4268 [
4269 ("domain".to_string(), Value::from(domain.as_str())),
4270 ("new_skills".to_string(), Value::from(evolved.len())),
4271 ]
4272 .into(),
4273 );
4274 }
4275 }
4276 }
4277 }
4278 }
4279 }
4280
4281 (proposal_result, state_before_map)
4282 }
4283
4284 /// Process a single action: capability → validate → policy → idempotency → execute.
4285 /// (Idempotency dedup runs AFTER the deny checks — review C3: a durable
4286 /// cached result must never outlive a tool's revocation.)
4287 /// Returns (ActionResult, retries_count).
4288 async fn process_action(
4289 &self,
4290 action: &Action,
4291 proposal_id: &str,
4292 trace_id: &str,
4293 parent_span_id: &str,
4294 session_id: Option<&str>,
4295 scope: Option<&crate::scope::RuntimeScope>,
4296 ) -> (ActionResult, u32) {
4297 // Derive action type name for span naming
4298 let action_type_name = serde_json::to_string(&action.action_type)
4299 .unwrap_or_default()
4300 .trim_matches('"')
4301 .to_string();
4302 let span_name = format!("action.{}", action_type_name);
4303
4304 // Begin child span for this action
4305 let action_span_id = {
4306 let mut attrs: HashMap<String, Value> = HashMap::new();
4307 attrs.insert("action_id".to_string(), Value::from(action.id.as_str()));
4308 if let Some(ref tool) = action.tool {
4309 attrs.insert("tool".to_string(), Value::from(tool.as_str()));
4310 }
4311 let mut log = self.log.lock().await;
4312 log.begin_span(&span_name, trace_id, Some(parent_span_id), attrs)
4313 };
4314
4315 // Execute the action pipeline and capture result
4316 let (result, retries) = self
4317 .process_action_inner(action, proposal_id, session_id, scope)
4318 .await;
4319
4320 // End action span based on result status
4321 let span_status = match result.status {
4322 ActionStatus::Succeeded => SpanStatus::Ok,
4323 ActionStatus::Failed | ActionStatus::Rejected => SpanStatus::Error,
4324 _ => SpanStatus::Unset,
4325 };
4326 {
4327 let mut log = self.log.lock().await;
4328 log.end_span(&action_span_id, span_status);
4329 }
4330
4331 (result, retries)
4332 }
4333
4334 /// Inner action processing: idempotency -> validate -> policy -> execute.
4335 /// Returns (ActionResult, retries_count).
4336 #[instrument(
4337 name = "action.process",
4338 skip_all,
4339 fields(
4340 action_id = %action.id,
4341 action_type = ?action.action_type,
4342 tool = action.tool.as_deref().unwrap_or("none"),
4343 )
4344 )]
4345 async fn process_action_inner(
4346 &self,
4347 action: &Action,
4348 proposal_id: &str,
4349 session_id: Option<&str>,
4350 scope: Option<&crate::scope::RuntimeScope>,
4351 ) -> (ActionResult, u32) {
4352 // Capability check
4353 {
4354 let caps = self.capabilities.read().await;
4355 if let Some(ref cap) = *caps {
4356 // Check tool capability for ToolCall actions
4357 if action.action_type == ActionType::ToolCall {
4358 if let Some(ref tool_name) = action.tool {
4359 if !cap.tool_allowed(tool_name) {
4360 let reason =
4361 format!("capability denied: tool '{}' not allowed", tool_name);
4362 let mut log = self.log.lock().await;
4363 log.append(
4364 EventKind::ActionRejected,
4365 Some(&action.id),
4366 Some(proposal_id),
4367 action_outcome_data("capability", &reason, false, None),
4368 );
4369 return (rejected_result(&action.id, reason), 0);
4370 }
4371 }
4372 }
4373
4374 // Check state key capability for StateWrite/StateRead actions
4375 if action.action_type == ActionType::StateWrite
4376 || action.action_type == ActionType::StateRead
4377 {
4378 if let Some(key) = action.parameters.get("key").and_then(|v| v.as_str()) {
4379 if !cap.state_key_allowed(key) {
4380 let reason =
4381 format!("capability denied: state key '{}' not allowed", key);
4382 let mut log = self.log.lock().await;
4383 log.append(
4384 EventKind::ActionRejected,
4385 Some(&action.id),
4386 Some(proposal_id),
4387 action_outcome_data("capability", &reason, false, None),
4388 );
4389 return (rejected_result(&action.id, reason), 0);
4390 }
4391 }
4392 }
4393 }
4394 }
4395
4396 // Validate
4397 let tools = self.tools.read().await;
4398 let validation = validate_action(action, &self.state, &tools);
4399 drop(tools);
4400
4401 if !validation.valid() {
4402 let error = validation
4403 .errors
4404 .iter()
4405 .map(|e| e.reason.as_str())
4406 .collect::<Vec<_>>()
4407 .join("; ");
4408 let mut log = self.log.lock().await;
4409 log.append(
4410 EventKind::ActionRejected,
4411 Some(&action.id),
4412 Some(proposal_id),
4413 action_outcome_data("validation", &error, false, None),
4414 );
4415 return (rejected_result(&action.id, error), 0);
4416 }
4417
4418 // Policy check — global registry plus, when the proposal is
4419 // executed under a session, that session's registry. Both
4420 // layers must pass for the action to proceed; session
4421 // policies are additive deny rules — they can deny what
4422 // global allows but cannot allow what global denies.
4423 {
4424 let mut violations = {
4425 let policies = self.policies.read().await;
4426 policies.check(action, &self.state)
4427 };
4428 if let Some(sid) = session_id {
4429 // Snapshot the per-session engine handle out from under
4430 // the outer registry lock so the inner check holds
4431 // only the engine's own RwLock — preserves the
4432 // documented lock-ordering discipline.
4433 let session_engine = {
4434 let sessions = self.session_policies.read().await;
4435 sessions.get(sid).cloned()
4436 };
4437 if let Some(engine) = session_engine {
4438 let engine = engine.read().await;
4439 violations.extend(engine.check(action, &self.state));
4440 } else {
4441 let error = format!(
4442 "unknown session id '{sid}' — open one via Runtime::open_session before executing under a session"
4443 );
4444 // Unknown session id — refuse the action rather
4445 // than silently fall back to global-only. A
4446 // proposal submitted under a closed session
4447 // shouldn't run with looser rules than the caller
4448 // intended.
4449 let mut log = self.log.lock().await;
4450 log.append(
4451 EventKind::PolicyViolation,
4452 Some(&action.id),
4453 Some(proposal_id),
4454 action_outcome_data("policy", &error, false, None),
4455 );
4456 return (rejected_result(&action.id, error), 0);
4457 }
4458 }
4459 if !violations.is_empty() {
4460 let error = violations
4461 .iter()
4462 .map(|v| format!("policy '{}': {}", v.policy_name, v.reason))
4463 .collect::<Vec<_>>()
4464 .join("; ");
4465 let mut log = self.log.lock().await;
4466 log.append(
4467 EventKind::PolicyViolation,
4468 Some(&action.id),
4469 Some(proposal_id),
4470 action_outcome_data("policy", &error, false, None),
4471 );
4472 return (rejected_result(&action.id, error), 0);
4473 }
4474 }
4475
4476 // Idempotency check — deliberately AFTER capability + policy
4477 // (linus review C3): with the durable journal, a cached result
4478 // consulted first would let a tool that is denied TODAY serve
4479 // yesterday's cached result after every restart, forever. Deny
4480 // rules must win over dedup.
4481 if action.idempotent && !action.invocation_mode.is_detached() {
4482 let key = idempotency_key(action, scope);
4483 let cache = self.idempotency_cache.lock().await;
4484 if let Some(cached) = cache.get(&key) {
4485 let mut log = self.log.lock().await;
4486 log.append(
4487 EventKind::ActionDeduplicated,
4488 Some(&action.id),
4489 Some(proposal_id),
4490 [
4491 (
4492 "cached_action_id".to_string(),
4493 Value::from(cached.action_id.as_str()),
4494 ),
4495 ("attempted".to_string(), Value::from(false)),
4496 ("stage".to_string(), Value::from("idempotency_cache")),
4497 ]
4498 .into(),
4499 );
4500 return (
4501 ActionResult {
4502 action_id: action.id.clone(),
4503 status: cached.status.clone(),
4504 output: cached.output.clone(),
4505 error: cached.error.clone(),
4506 terminal: cached.terminal,
4507 // The cached action committed these mutations in an
4508 // earlier proposal. This proposal did not observe or
4509 // apply them, so its actual-change evidence is empty.
4510 state_changes: HashMap::new(),
4511 rolled_back: false,
4512 duration_ms: Some(0.0),
4513 timestamp: chrono::Utc::now(),
4514 },
4515 0,
4516 );
4517 }
4518 }
4519
4520 // Validated
4521 {
4522 let mut log = self.log.lock().await;
4523 log.append(
4524 EventKind::ActionValidated,
4525 Some(&action.id),
4526 Some(proposal_id),
4527 HashMap::new(),
4528 );
4529 }
4530
4531 // Execute with retry
4532 let (result, retries) = self
4533 .execute_with_retry(action, proposal_id, session_id, scope)
4534 .await;
4535
4536 // Cache idempotent results. Detached invocations are exempt
4537 // (linus review D4): their Succeeded output is a live tool
4538 // HANDLE, exactly as uncacheable here as in the result cache —
4539 // deduping would return a stale handle instead of starting the
4540 // tool, and journaling would replay a handle id that doesn't
4541 // exist in a fresh registry after restart.
4542 if action.idempotent
4543 && !action.invocation_mode.is_detached()
4544 && result.status == ActionStatus::Succeeded
4545 {
4546 let key = idempotency_key(action, scope);
4547 {
4548 let mut cache = self.idempotency_cache.lock().await;
4549 cache.insert(key.clone(), result.clone());
4550 }
4551 // Persist to the durable journal (C3) so the result survives a
4552 // restart and the action isn't re-executed.
4553 self.journal_idempotency(&key, Some(&result)).await;
4554 }
4555
4556 tracing::info!(
4557 status = ?result.status,
4558 duration_ms = result.duration_ms,
4559 "action completed"
4560 );
4561
4562 (result, retries)
4563 }
4564
4565 /// Execute with retry logic and timeout.
4566 /// Returns (ActionResult, retries_count).
4567 async fn execute_with_retry(
4568 &self,
4569 action: &Action,
4570 proposal_id: &str,
4571 session_id: Option<&str>,
4572 scope: Option<&crate::scope::RuntimeScope>,
4573 ) -> (ActionResult, u32) {
4574 // The harness config (Evolution Agent, §3.5), when installed, caps
4575 // the per-action retry budget and sets the inter-attempt backoff —
4576 // this is where an applied retry-config mutation actually takes
4577 // effect. `None` (the default) preserves prior behavior exactly.
4578 let hc = self.harness_config.read().await.clone();
4579 let retry_cap = hc.as_ref().map(|c| c.max_retries).unwrap_or(u32::MAX);
4580 let backoff_override = hc.as_ref().map(|c| c.retry_backoff_ms).filter(|&v| v > 0);
4581 let max_attempts = if action.failure_behavior == FailureBehavior::Retry {
4582 match action.max_retries.min(retry_cap).checked_add(1) {
4583 Some(max_attempts) => max_attempts,
4584 None => {
4585 let reason =
4586 format!("action '{}' retry attempt count overflows u32", action.id);
4587 self.log.lock().await.append(
4588 EventKind::ActionRejected,
4589 Some(&action.id),
4590 Some(proposal_id),
4591 action_outcome_data("retry_admission", &reason, false, None),
4592 );
4593 return (rejected_result(&action.id, reason), 0);
4594 }
4595 }
4596 } else {
4597 1
4598 };
4599
4600 let mut last_error: Option<String> = None;
4601 let mut retries: u32 = 0;
4602 let tool_source = self.tool_source_for_action(action).await;
4603 let params_digest = action_params_digest(action);
4604
4605 for attempt in 0..max_attempts {
4606 if attempt > 0 {
4607 retries += 1;
4608 // Base delay from the harness config when installed (an
4609 // applied retry-config mutation), else the built-in default.
4610 let base = backoff_override.unwrap_or(RETRY_BASE_DELAY_MS);
4611 let delay = base * RETRY_BACKOFF_FACTOR.pow(attempt - 1);
4612 tokio::time::sleep(Duration::from_millis(delay)).await;
4613 let mut log = self.log.lock().await;
4614 log.append(
4615 EventKind::ActionRetrying,
4616 Some(&action.id),
4617 Some(proposal_id),
4618 [("attempt".to_string(), Value::from(attempt + 1))].into(),
4619 );
4620 }
4621
4622 {
4623 let mut data: HashMap<String, Value> =
4624 [("attempt".to_string(), Value::from(attempt + 1))].into();
4625 insert_tool_event_provenance(&mut data, action, tool_source);
4626 let mut log = self.log.lock().await;
4627 log.append(
4628 EventKind::ActionExecuting,
4629 Some(&action.id),
4630 Some(proposal_id),
4631 data,
4632 );
4633 }
4634
4635 let start = std::time::Instant::now();
4636 let transitions_before = self.state.transition_count();
4637
4638 // Execute with optional timeout. Keep the timeout bit typed until
4639 // event classification rather than trying to recover it from the
4640 // human-readable error later.
4641 let (exec_result, engine_timeout) = if let Some(timeout_ms) = action.timeout_ms {
4642 match timeout(
4643 Duration::from_millis(timeout_ms),
4644 self.dispatch(action, session_id, scope, attempt + 1),
4645 )
4646 .await
4647 {
4648 Ok(result) => (result, false),
4649 Err(_) => (
4650 Err(ToolFailure::ordinary(format!(
4651 "action timed out after {}ms",
4652 timeout_ms
4653 ))),
4654 true,
4655 ),
4656 }
4657 } else {
4658 (
4659 self.dispatch(action, session_id, scope, attempt + 1).await,
4660 false,
4661 )
4662 };
4663
4664 let duration_ms = start.elapsed().as_secs_f64() * 1000.0;
4665
4666 match exec_result {
4667 Ok(output) => {
4668 if let Err(error) = car_inference::catalog_identity::canonical_json(&output) {
4669 let reason = format!(
4670 "tool output failed JCS/I-JSON validation after dispatch; external effect may have occurred and was not undone: {error}"
4671 );
4672 let mut data = action_outcome_data(
4673 "output_validation",
4674 &reason,
4675 true,
4676 Some(attempt + 1),
4677 );
4678 data.insert(
4679 "external_effect_status".to_string(),
4680 Value::from("may_have_occurred_not_undone"),
4681 );
4682 insert_tool_event_provenance(&mut data, action, tool_source);
4683 insert_action_outcome_signal(
4684 &mut data,
4685 action,
4686 ¶ms_digest,
4687 Some("validation"),
4688 );
4689 if action.tool.is_some() {
4690 data.insert("ok".to_string(), Value::from(false));
4691 }
4692 self.log.lock().await.append(
4693 EventKind::ActionFailed,
4694 Some(&action.id),
4695 Some(proposal_id),
4696 data,
4697 );
4698 return (
4699 ActionResult {
4700 action_id: action.id.clone(),
4701 status: ActionStatus::Failed,
4702 output: None,
4703 error: Some(reason),
4704 terminal: false,
4705 state_changes: HashMap::new(),
4706 rolled_back: false,
4707 duration_ms: Some(duration_ms),
4708 timestamp: chrono::Utc::now(),
4709 },
4710 retries,
4711 );
4712 }
4713
4714 // Snapshot the changes made by dispatch before applying
4715 // any result accounting. Deletion and set-to-null are
4716 // intentionally tagged differently on the authenticated
4717 // wire; a bare JSON null cannot represent both.
4718 let runtime_state_changes: HashMap<String, Value> = self
4719 .state
4720 .transitions_since(transitions_before)
4721 .into_iter()
4722 // Actions at the same DAG level execute concurrently,
4723 // so the shared transition tail can contain a sibling's
4724 // writes. StateTransition.action_id is assigned at the
4725 // mutation site and is the authoritative attribution
4726 // boundary for this per-action journal field.
4727 .filter(|transition| transition.action_id == action.id)
4728 .map(|transition| {
4729 (
4730 transition.key,
4731 StateMutation::from_new_value(transition.new_value).encode(),
4732 )
4733 })
4734 .collect();
4735
4736 // Declared expected effects are plan assertions only.
4737 // They must never be applied as if the runtime observed
4738 // them, nor merged into committed result evidence.
4739 let tenant_for_effects = scope.and_then(|s| s.tenant_id.as_deref());
4740 let state_changes = runtime_state_changes.clone();
4741
4742 // Record this result's provenance for the VIGIL intent
4743 // gate (`crate::taint`) — the only point in the system
4744 // that knows which SPECIFIC results were tainted. Only
4745 // on success: a failed action commits no effects, so it
4746 // changes no taint. No intent gate ⇒ no ledger ⇒ one
4747 // `Option` read and nothing else.
4748 let ledger = self.taint_ledger.read().await.clone();
4749 if let Some(ledger) = ledger {
4750 ledger
4751 .record_result(
4752 tenant_for_effects,
4753 action,
4754 state_changes.keys().cloned(),
4755 )
4756 .await;
4757 }
4758
4759 let mut log = self.log.lock().await;
4760 // Record the tool name + result count so the tool-receipt
4761 // verifier (A6) can project ground-truth receipts from the
4762 // log and catch tool-use hallucinations.
4763 let mut succ_data: HashMap<String, Value> = HashMap::new();
4764 succ_data.insert("attempt".to_string(), Value::from(attempt + 1));
4765 succ_data.insert("attempted".to_string(), Value::from(true));
4766 succ_data.insert("stage".to_string(), Value::from("dispatch"));
4767 insert_tool_event_provenance(&mut succ_data, action, tool_source);
4768 insert_action_outcome_signal(&mut succ_data, action, ¶ms_digest, None);
4769 if action.tool.is_some() {
4770 succ_data.insert("ok".to_string(), Value::from(true));
4771 if let Value::Array(arr) = &output {
4772 succ_data
4773 .insert("result_count".to_string(), Value::from(arr.len() as u64));
4774 }
4775 }
4776 // Record duration through the standardized metric path
4777 // (§3.5.1) so it feeds `EventLog::metrics_totals` via the
4778 // same contract as inference token metrics — one path,
4779 // not a coincidentally-matching `"duration_ms"` string.
4780 log.append_metered(
4781 EventKind::ActionSucceeded,
4782 Some(&action.id),
4783 Some(proposal_id),
4784 succ_data,
4785 car_eventlog::Metrics::latency(duration_ms),
4786 );
4787
4788 if !action.expected_effects.is_empty() || !runtime_state_changes.is_empty() {
4789 log.append(
4790 EventKind::StateChanged,
4791 Some(&action.id),
4792 Some(proposal_id),
4793 [
4794 (
4795 "declared_expected_effects".to_string(),
4796 serde_json::to_value(&action.expected_effects)
4797 .unwrap_or_default(),
4798 ),
4799 (
4800 "runtime_state_mutations".to_string(),
4801 serde_json::to_value(&runtime_state_changes)
4802 .unwrap_or_default(),
4803 ),
4804 (
4805 "changes".to_string(),
4806 serde_json::to_value(&state_changes).unwrap_or_default(),
4807 ),
4808 (
4809 "changes_semantics".to_string(),
4810 Value::from("provisional_state_mutations"),
4811 ),
4812 ("attempt".to_string(), Value::from(attempt + 1)),
4813 ("attempted".to_string(), Value::from(true)),
4814 ("stage".to_string(), Value::from("state_effect")),
4815 ]
4816 .into(),
4817 );
4818 }
4819
4820 return (
4821 ActionResult {
4822 action_id: action.id.clone(),
4823 status: ActionStatus::Succeeded,
4824 output: Some(output),
4825 error: None,
4826 terminal: false,
4827 state_changes,
4828 rolled_back: false,
4829 duration_ms: Some(duration_ms),
4830 timestamp: chrono::Utc::now(),
4831 },
4832 retries,
4833 );
4834 }
4835 Err(failure) => {
4836 let terminal = match failure.classification {
4837 ToolFailureClassification::Ordinary => false,
4838 ToolFailureClassification::Terminal => true,
4839 };
4840 let error = failure.message;
4841 last_error = Some(error.clone());
4842 let mut log = self.log.lock().await;
4843 let mut fail_data: HashMap<String, Value> = [
4844 ("error".to_string(), Value::from(error.as_str())),
4845 ("attempt".to_string(), Value::from(attempt + 1)),
4846 ("attempted".to_string(), Value::from(true)),
4847 ("stage".to_string(), Value::from("dispatch")),
4848 ]
4849 .into();
4850 // Tool name + ok=false so the receipt verifier (A6) records
4851 // that the tool *executed* (a failed call still ran).
4852 insert_tool_event_provenance(&mut fail_data, action, tool_source);
4853 insert_action_outcome_signal(
4854 &mut fail_data,
4855 action,
4856 ¶ms_digest,
4857 Some(action_error_class(action, &error, engine_timeout)),
4858 );
4859 if action.tool.is_some() {
4860 fail_data.insert("ok".to_string(), Value::from(false));
4861 }
4862 if terminal {
4863 fail_data.insert("terminal".to_string(), Value::from(true));
4864 }
4865 log.append(
4866 EventKind::ActionFailed,
4867 Some(&action.id),
4868 Some(proposal_id),
4869 fail_data,
4870 );
4871 drop(log);
4872
4873 if terminal {
4874 return (
4875 ActionResult {
4876 action_id: action.id.clone(),
4877 status: ActionStatus::Failed,
4878 output: None,
4879 error: Some(error),
4880 terminal: true,
4881 rolled_back: false,
4882 state_changes: HashMap::new(),
4883 duration_ms: Some(duration_ms),
4884 timestamp: chrono::Utc::now(),
4885 },
4886 retries,
4887 );
4888 }
4889 }
4890 }
4891 }
4892
4893 // All attempts exhausted
4894 if action.failure_behavior == FailureBehavior::Skip {
4895 let reason = last_error.as_deref().unwrap_or("all attempts exhausted");
4896 self.log.lock().await.append(
4897 EventKind::ActionSkipped,
4898 Some(&action.id),
4899 Some(proposal_id),
4900 action_outcome_data("failure_behavior", reason, true, Some(max_attempts)),
4901 );
4902 return (skipped_result(&action.id, reason), retries);
4903 }
4904
4905 (
4906 ActionResult {
4907 action_id: action.id.clone(),
4908 status: ActionStatus::Failed,
4909 output: None,
4910 error: last_error,
4911 terminal: false,
4912 state_changes: HashMap::new(),
4913 rolled_back: false,
4914 duration_ms: None,
4915 timestamp: chrono::Utc::now(),
4916 },
4917 retries,
4918 )
4919 }
4920
4921 /// Resolve the stable source category for an action's tool. Canonical
4922 /// ToolEntry provenance wins; the ToolSchema fallback keeps legacy direct
4923 /// registrations observable, and the final default covers old callers that
4924 /// supplied a tool field on a non-tool action.
4925 async fn tool_source_for_action(&self, action: &Action) -> Option<car_ir::ToolSourceKind> {
4926 let tool = action.tool.as_deref()?;
4927 if let Some(entry) = self.registry.get(tool).await {
4928 return Some(entry.source.kind());
4929 }
4930 Some(
4931 self.tools
4932 .read()
4933 .await
4934 .get(tool)
4935 .map(|schema| schema.source)
4936 .unwrap_or(car_ir::ToolSourceKind::UserDefined),
4937 )
4938 }
4939
4940 /// Dispatch an action to the appropriate handler.
4941 ///
4942 /// `scope` is the per-execution caller / tenant surface
4943 /// (Parslee-ai/car#187 phase 3). When the scope carries a
4944 /// tenant id, state R/W operations (`StateWrite`, `StateRead`,
4945 /// `Assertion`) route through `StateStore::scoped(tenant_id)`
4946 /// so distinct tenants can't see each other's keys. Unscoped
4947 /// proposals get the legacy flat-namespace behaviour
4948 /// automatically.
4949 async fn dispatch(
4950 &self,
4951 action: &Action,
4952 session_id: Option<&str>,
4953 scope: Option<&crate::scope::RuntimeScope>,
4954 attempt: u32,
4955 ) -> Result<Value, ToolFailure> {
4956 match action.action_type {
4957 ActionType::ToolCall => {
4958 let tool_name = action.tool.as_deref().ok_or("tool_call has no tool")?;
4959 let params = Value::Object(
4960 action
4961 .parameters
4962 .iter()
4963 .map(|(k, v)| (k.clone(), v.clone()))
4964 .collect(),
4965 );
4966
4967 // Detached invocation modes (C2): start the tool via the
4968 // configured executor's streaming entry point, register a
4969 // handle, and return immediately — the DAG must not block
4970 // on a streaming/long-running tool. Deliberately BEFORE
4971 // the result cache (a handle is a live invocation, never a
4972 // cacheable value) but still behind the rate limiter.
4973 // Built-ins don't stream; a detached call requires a
4974 // configured executor that implements execute_stream.
4975 if action.invocation_mode.is_detached() {
4976 self.rate_limiter.acquire(tool_name).await;
4977 let configured = {
4978 let guard = self.tool_executor.lock().await;
4979 guard.as_ref().cloned()
4980 };
4981 let executor = configured.ok_or_else(|| {
4982 format!("tool '{tool_name}': detached invocation requires a tool executor")
4983 })?;
4984 let rx = executor
4985 .execute_stream(tool_name, ¶ms, &action.id)
4986 .await?;
4987 let (handle, cancel) = self.tool_handles.register(tool_name, &action.id).await;
4988 crate::tool_handles::spawn_drain(
4989 self.tool_handles.clone(),
4990 handle.id.clone(),
4991 rx,
4992 cancel,
4993 );
4994 return Ok(serde_json::json!({
4995 "tool_handle": handle.id,
4996 "status": "running",
4997 }));
4998 }
4999
5000 // Check cross-proposal result cache.
5001 if let Some(cached) = self.result_cache.get(tool_name, ¶ms).await {
5002 return Ok(cached);
5003 }
5004
5005 // Apply rate limit backpressure before executing.
5006 self.rate_limiter.acquire(tool_name).await;
5007
5008 // Try built-in inference tools first when the inference engine is available.
5009 if matches!(
5010 tool_name,
5011 "infer" | "infer.grounded" | "embed" | "classify" | "transcribe" | "synthesize"
5012 ) {
5013 if let Some(ref engine) = self.inference_engine {
5014 // For "infer.grounded" or "infer" with memgine available,
5015 // build context from memory and attach it to the request.
5016 let params = {
5017 let should_ground =
5018 tool_name == "infer.grounded" || tool_name == "infer";
5019 if should_ground {
5020 if let Some(ref memgine) = self.memgine {
5021 if let Some(prompt) =
5022 params.get("prompt").and_then(|v| v.as_str())
5023 {
5024 let ctx = {
5025 let mut m = memgine.lock().await;
5026 m.build_context(prompt)
5027 };
5028 if !ctx.is_empty() {
5029 let mut p = params.clone();
5030 if let Some(obj) = p.as_object_mut() {
5031 obj.insert("context".to_string(), Value::from(ctx));
5032 }
5033 p
5034 } else {
5035 params
5036 }
5037 } else {
5038 params
5039 }
5040 } else {
5041 params
5042 }
5043 } else {
5044 params
5045 }
5046 };
5047
5048 // Route "infer.grounded" to "infer" for the service layer
5049 let effective_tool = if tool_name == "infer.grounded" {
5050 "infer"
5051 } else {
5052 tool_name
5053 };
5054 let result =
5055 car_inference::service::execute_tool(engine, effective_tool, ¶ms)
5056 .await
5057 .map_err(|e| e.to_string());
5058
5059 if let Ok(ref value) = result {
5060 self.result_cache
5061 .put(tool_name, ¶ms, value.clone())
5062 .await;
5063 }
5064
5065 return result.map_err(ToolFailure::from);
5066 }
5067 }
5068
5069 // Built-in memory consolidation tool.
5070 if tool_name == "memory.consolidate" {
5071 if let Some(ref memgine) = self.memgine {
5072 let report = {
5073 let mut m = memgine.lock().await;
5074 m.consolidate().await
5075 };
5076 // Log the consolidation event
5077 {
5078 let mut log = self.log.lock().await;
5079 log.append(
5080 EventKind::Consolidated,
5081 None,
5082 None,
5083 [
5084 (
5085 "expired_pruned".to_string(),
5086 Value::from(report.expired_pruned),
5087 ),
5088 (
5089 "superseded_gc".to_string(),
5090 Value::from(report.superseded_gc),
5091 ),
5092 (
5093 "stale_embeddings_removed".to_string(),
5094 Value::from(report.stale_embeddings_removed),
5095 ),
5096 (
5097 "nodes_embedded".to_string(),
5098 Value::from(report.nodes_embedded),
5099 ),
5100 (
5101 "domains_evolved".to_string(),
5102 Value::from(report.domains_evolved.clone()),
5103 ),
5104 ("total_nodes".to_string(), Value::from(report.total_nodes)),
5105 ("total_edges".to_string(), Value::from(report.total_edges)),
5106 ]
5107 .into(),
5108 );
5109 // Per-candidate gate telemetry (SkillOpt-inspired):
5110 // one event per promotion/rejection so the skill
5111 // bank's evolution is auditable.
5112 for key in &report.candidates_promoted {
5113 log.append(
5114 EventKind::CandidatePromoted,
5115 None,
5116 None,
5117 [("candidate".to_string(), Value::from(key.as_str()))].into(),
5118 );
5119 }
5120 for key in &report.candidates_rejected {
5121 log.append(
5122 EventKind::CandidateRejected,
5123 None,
5124 None,
5125 [("candidate".to_string(), Value::from(key.as_str()))].into(),
5126 );
5127 }
5128 }
5129 return Ok(serde_json::to_value(&report).unwrap_or(Value::Null));
5130 } else {
5131 return Err(
5132 "memory.consolidate requires memgine (attach with with_learning)"
5133 .into(),
5134 );
5135 }
5136 }
5137
5138 // Built-in outbound human messaging. Sits here, alongside the
5139 // other built-ins — downstream of `validate_action` and the
5140 // policy check in the caller, and downstream of the result
5141 // cache read and `rate_limiter.acquire` above — so a message
5142 // to a human traverses exactly the same governed chain
5143 // (validator → policy → rate limit → eventlog) as any other
5144 // side effect, instead of the hand-rolled transport each agent
5145 // used to carry.
5146 if tool_name == "messaging.send" {
5147 // No fall-through to the host executor when unconfigured.
5148 // Falling through would let an ungoverned host transport
5149 // answer the call — the exact path this built-in exists to
5150 // close — so an unconfigured runtime refuses instead.
5151 let Some(ref sink) = self.message_sink else {
5152 return Err("messaging.send: messaging is not configured on this \
5153 runtime — attach a message sink via \
5154 Runtime::with_message_sink"
5155 .into());
5156 };
5157 let msg = crate::messaging::OutboundMessage::from_tool_params(¶ms)?;
5158 let receipt = sink.send(&msg).await?;
5159 // Not cached: the schema declares no cache TTL, and a
5160 // cached send would swallow a second, genuinely wanted
5161 // message with identical text.
5162 return serde_json::to_value(&receipt).map_err(|error| {
5163 ToolFailure::ordinary(format!("messaging.send: serialize receipt: {error}"))
5164 });
5165 }
5166
5167 // Prefer a configured tool_executor for any tool it claims to handle.
5168 // Fall through to agent_basics only when the configured executor is absent
5169 // or explicitly returns "unknown tool" — this prevents agent_basics' built-in
5170 // read_file/write_file (which resolve paths via std::env::current_dir) from
5171 // silently overriding an executor that carries its own working_dir.
5172 let configured = {
5173 let guard = self.tool_executor.lock().await;
5174 guard.as_ref().cloned()
5175 };
5176
5177 if let Some(ref executor) = configured {
5178 let return_schema = self
5179 .tools
5180 .read()
5181 .await
5182 .get(tool_name)
5183 .and_then(|schema| schema.returns.clone());
5184 let result = executor
5185 .execute_classified(
5186 tool_name,
5187 ¶ms,
5188 &action.id,
5189 action.timeout_ms,
5190 session_id,
5191 attempt,
5192 &action.expected_effects,
5193 return_schema.as_ref(),
5194 )
5195 .await;
5196 let fall_through = matches!(
5197 &result,
5198 Err(error) if error.classification == ToolFailureClassification::Ordinary
5199 && error.message.starts_with("unknown tool")
5200 );
5201 if !fall_through {
5202 match result {
5203 Ok(execution) => {
5204 if !execution.state_changes.is_empty() {
5205 let expected_keys: std::collections::BTreeSet<&str> = action
5206 .expected_effects
5207 .keys()
5208 .map(String::as_str)
5209 .collect();
5210 let actual_keys: std::collections::BTreeSet<&str> = execution
5211 .state_changes
5212 .keys()
5213 .map(String::as_str)
5214 .collect();
5215 if expected_keys != actual_keys {
5216 return Err(format!(
5217 "tool '{}' callback state_changes keys do not match action '{}': expected {:?}, got {:?}",
5218 tool_name, action.id, expected_keys, actual_keys
5219 )
5220 .into());
5221 }
5222 car_inference::catalog_identity::canonical_json(&execution.output)
5223 .map_err(|error| {
5224 format!(
5225 "tool output failed JCS/I-JSON validation after dispatch; external effect may have occurred and was not undone: {error}"
5226 )
5227 })?;
5228 let changes_value = serde_json::to_value(&execution.state_changes)
5229 .map_err(|error| {
5230 format!(
5231 "tool '{}' callback state_changes serialization failed: {error}",
5232 tool_name
5233 )
5234 })?;
5235 car_inference::catalog_identity::canonical_json(&changes_value)
5236 .map_err(|error| {
5237 format!(
5238 "tool '{}' callback state_changes failed JCS/I-JSON validation: {error}",
5239 tool_name
5240 )
5241 })?;
5242
5243 // Every rejection above happens before the
5244 // first write. Apply in deterministic key order,
5245 // attributed to the active action, through ONE
5246 // batch mutation boundary: a concurrent reader,
5247 // journal replay, and restart recovery observe
5248 // the complete old state or the complete new
5249 // state — never a prefix of this callback's
5250 // keys (Parslee-ai/car#1140).
5251 let mut changes: Vec<(String, Value)> = execution
5252 .state_changes
5253 .iter()
5254 .map(|(key, value)| (key.clone(), value.clone()))
5255 .collect();
5256 changes.sort_by(|(left, _), (right, _)| left.cmp(right));
5257 let tenant = scope.and_then(|scope| scope.tenant_id.as_deref());
5258 self.state.scoped(tenant).set_batch(changes, &action.id);
5259 }
5260 self.result_cache
5261 .put(tool_name, ¶ms, execution.output.clone())
5262 .await;
5263 return Ok(execution.output);
5264 }
5265 Err(error) => return Err(error),
5266 }
5267 }
5268 }
5269
5270 // Built-in commodity tools resolve against the bound substrate
5271 // (default: LocalSubstrate → historic host behavior). `calculate`
5272 // stays pure inside agent_basics and ignores the substrate.
5273 let substrate = self.substrate.read().await.clone();
5274 let read_ledger = self.read_ledgers.ledger_for(session_id);
5275 if let Some(result) = crate::agent_basics::execute_with_ledger(
5276 &substrate,
5277 &read_ledger,
5278 tool_name,
5279 ¶ms,
5280 )
5281 .await
5282 {
5283 if let Ok(ref value) = result {
5284 self.result_cache
5285 .put(tool_name, ¶ms, value.clone())
5286 .await;
5287 }
5288 return result.map_err(ToolFailure::from);
5289 }
5290
5291 Err(format!("no handler for tool '{}'", tool_name).into())
5292 }
5293 ActionType::StateWrite => {
5294 let key = action
5295 .parameters
5296 .get("key")
5297 .and_then(|v| v.as_str())
5298 .ok_or("state_write requires 'key' parameter")?;
5299 let value = action
5300 .parameters
5301 .get("value")
5302 .cloned()
5303 .unwrap_or(Value::Null);
5304 let tenant = scope.and_then(|s| s.tenant_id.as_deref());
5305 self.state.scoped(tenant).set(key, value, &action.id);
5306 Ok(Value::from(format!("written: {}", key)))
5307 }
5308 ActionType::StateRead => {
5309 let key = action
5310 .parameters
5311 .get("key")
5312 .and_then(|v| v.as_str())
5313 .ok_or("state_read requires 'key' parameter")?;
5314 let tenant = scope.and_then(|s| s.tenant_id.as_deref());
5315 Ok(self.state.scoped(tenant).get(key).unwrap_or(Value::Null))
5316 }
5317 ActionType::Assertion => {
5318 let key = action
5319 .parameters
5320 .get("key")
5321 .and_then(|v| v.as_str())
5322 .ok_or("assertion requires 'key' parameter")?;
5323 let expected = action
5324 .parameters
5325 .get("expected")
5326 .cloned()
5327 .unwrap_or(Value::Null);
5328 let tenant = scope.and_then(|s| s.tenant_id.as_deref());
5329 let actual = self.state.scoped(tenant).get(key).unwrap_or(Value::Null);
5330 if actual != expected {
5331 Err(format!(
5332 "assertion failed: state['{}'] = {:?}, expected {:?}",
5333 key, actual, expected
5334 )
5335 .into())
5336 } else {
5337 Ok(serde_json::json!({"asserted": key, "value": actual}))
5338 }
5339 }
5340 }
5341 }
5342
5343 // --- Checkpoint and resume ---
5344
5345 /// Save a checkpoint of the current runtime state.
5346 pub async fn save_checkpoint(&self) -> Checkpoint {
5347 let state = self.state.snapshot();
5348 let tools: Vec<String> = self.tools.read().await.keys().cloned().collect();
5349 let log = self.log.lock().await;
5350 let events: Vec<Value> = log
5351 .events()
5352 .iter()
5353 .map(|e| serde_json::to_value(e).unwrap_or_default())
5354 .collect();
5355
5356 Checkpoint {
5357 checkpoint_id: Uuid::new_v4().to_string(),
5358 created_at: chrono::Utc::now(),
5359 state,
5360 events,
5361 tools,
5362 metadata: HashMap::new(),
5363 }
5364 }
5365
5366 /// Save checkpoint to a JSON file.
5367 pub async fn save_checkpoint_to_file(&self, path: &str) -> Result<(), String> {
5368 let checkpoint = self.save_checkpoint().await;
5369 let json = serde_json::to_string_pretty(&checkpoint)
5370 .map_err(|e| format!("serialize error: {}", e))?;
5371 tokio::fs::write(path, json)
5372 .await
5373 .map_err(|e| format!("write error: {}", e))?;
5374 Ok(())
5375 }
5376
5377 /// Load a checkpoint from a JSON file and restore state.
5378 pub async fn load_checkpoint_from_file(&self, path: &str) -> Result<Checkpoint, String> {
5379 let json = tokio::fs::read_to_string(path)
5380 .await
5381 .map_err(|e| format!("read error: {}", e))?;
5382 let checkpoint: Checkpoint =
5383 serde_json::from_str(&json).map_err(|e| format!("deserialize error: {}", e))?;
5384 self.restore_checkpoint(&checkpoint).await;
5385 Ok(checkpoint)
5386 }
5387
5388 /// Restore runtime state from a checkpoint.
5389 pub async fn restore_checkpoint(&self, checkpoint: &Checkpoint) {
5390 // Replace state completely — don't merge, don't create synthetic transitions
5391 self.state.replace_all(checkpoint.state.clone());
5392 // Clear idempotency cache — stale results from pre-checkpoint execution
5393 // must not bypass validation/policy on the restored state
5394 self.idempotency_cache.lock().await.clear();
5395 // Restore tools (as name-only schemas; full schemas are not persisted in checkpoint)
5396 let mut tools = self.tools.write().await;
5397 tools.clear();
5398 for tool_name in &checkpoint.tools {
5399 let schema = ToolSchema {
5400 name: tool_name.clone(),
5401 source: car_ir::ToolSourceKind::UserDefined,
5402 description: String::new(),
5403 parameters: serde_json::Value::Object(Default::default()),
5404 returns: None,
5405 idempotent: false,
5406 cache_ttl_secs: None,
5407 rate_limit: None,
5408 };
5409 tools.insert(tool_name.clone(), schema);
5410 }
5411 }
5412
5413 /// Register a subprocess tool and set up the subprocess executor.
5414 /// If no executor exists, creates a new SubprocessToolExecutor.
5415 /// If one already exists, creates a new SubprocessToolExecutor with the
5416 /// existing executor as fallback.
5417 pub async fn register_subprocess_tool(
5418 &self,
5419 name: &str,
5420 tool: crate::subprocess::SubprocessTool,
5421 ) {
5422 use crate::subprocess::SubprocessToolExecutor;
5423
5424 let schema = ToolSchema {
5425 name: name.to_string(),
5426 source: car_ir::ToolSourceKind::Subprocess,
5427 description: format!("Subprocess tool: {}", tool.command),
5428 parameters: serde_json::Value::Object(Default::default()),
5429 returns: None,
5430 idempotent: false,
5431 cache_ttl_secs: None,
5432 rate_limit: None,
5433 };
5434 self.register_tool_entry(
5435 crate::registry::ToolEntry::new(schema)
5436 .with_source(crate::registry::ToolSource::Subprocess),
5437 )
5438 .await;
5439
5440 let mut guard = self.tool_executor.lock().await;
5441 let mut executor = match guard.take() {
5442 Some(existing) => {
5443 let mut sub = SubprocessToolExecutor::new();
5444 sub = sub.with_fallback(existing);
5445 sub
5446 }
5447 None => SubprocessToolExecutor::new(),
5448 };
5449 executor.register(name, tool);
5450 *guard = Some(std::sync::Arc::new(executor));
5451 }
5452}
5453
5454impl Default for Runtime {
5455 fn default() -> Self {
5456 Self::new()
5457 }
5458}
5459
5460#[cfg(test)]
5461mod timeout_integration_tests {
5462 //! Integration coverage for the #259/#262 timeout coordination the unit
5463 //! tests in `car-server-core` / `car-ffi-common` only exercise as pure
5464 //! selection helpers (#266 item 4): that the executor's per-action deadline
5465 //! reaps a slow dispatch first, and that `action.timeout_ms` flows
5466 //! end-to-end onto `execute_with_action`.
5467 use super::*;
5468 use car_ir::ActionProposal;
5469 use std::sync::atomic::{AtomicU64, Ordering};
5470
5471 /// A tool executor that (a) records the `timeout_ms` it was handed and
5472 /// (b) sleeps `delay_ms` before returning, so a test can prove the
5473 /// executor's own `timeout(action.timeout_ms, dispatch)` reaps a call that
5474 /// outlives its budget.
5475 struct RecordingExecutor {
5476 seen_timeout_ms: Arc<AtomicU64>,
5477 delay_ms: u64,
5478 }
5479
5480 #[async_trait::async_trait]
5481 impl ToolExecutor for RecordingExecutor {
5482 async fn execute(&self, _tool: &str, _params: &Value) -> Result<Value, String> {
5483 Ok(Value::Null)
5484 }
5485 async fn execute_with_action(
5486 &self,
5487 _tool: &str,
5488 _params: &Value,
5489 _action_id: &str,
5490 timeout_ms: Option<u64>,
5491 ) -> Result<Value, String> {
5492 // `u64::MAX` sentinel = "None was passed".
5493 self.seen_timeout_ms
5494 .store(timeout_ms.unwrap_or(u64::MAX), Ordering::SeqCst);
5495 tokio::time::sleep(Duration::from_millis(self.delay_ms)).await;
5496 Ok(serde_json::json!({ "ok": true }))
5497 }
5498 }
5499
5500 fn one_tool_proposal(timeout_ms: Option<u64>) -> ActionProposal {
5501 let mut action = serde_json::json!({
5502 "id": "a0",
5503 "type": "tool_call",
5504 "tool": "slow",
5505 "parameters": {},
5506 "dependencies": [],
5507 });
5508 if let Some(ms) = timeout_ms {
5509 action["timeout_ms"] = serde_json::json!(ms);
5510 }
5511 serde_json::from_value(serde_json::json!({
5512 "source": "test",
5513 "actions": [action],
5514 }))
5515 .expect("proposal deserializes")
5516 }
5517
5518 #[tokio::test(start_paused = true)]
5519 async fn action_timeout_ms_reaches_executor_and_reaps_slow_dispatch() {
5520 let seen = Arc::new(AtomicU64::new(0));
5521 let rt = Runtime::new();
5522 rt.register_tool("slow").await;
5523 rt.set_executor(Arc::new(RecordingExecutor {
5524 seen_timeout_ms: seen.clone(),
5525 delay_ms: 2_000, // outlives the 100ms budget below
5526 }))
5527 .await;
5528
5529 // 100ms budget against a 2s dispatch: the executor's own
5530 // `timeout(action.timeout_ms, dispatch)` must reap it (#262 — the
5531 // executor is the authority), and the budget must have reached
5532 // `execute_with_action` (the #262 harness→action propagation, proven
5533 // end-to-end here rather than only via the pure helper).
5534 let result = rt.execute(&one_tool_proposal(Some(100))).await;
5535 assert_eq!(
5536 seen.load(Ordering::SeqCst),
5537 100,
5538 "budget must reach the executor"
5539 );
5540 let action_result = &result.results[0];
5541 assert!(
5542 action_result
5543 .error
5544 .as_deref()
5545 .unwrap_or("")
5546 .contains("timed out"),
5547 "slow dispatch must be reaped by the action deadline: {:?}",
5548 action_result.error
5549 );
5550 }
5551
5552 /// An executor that records every `attempt` it is handed and fails until
5553 /// the Nth, so a test can watch the counter advance across real retries.
5554 struct AttemptRecordingExecutor {
5555 seen: Arc<tokio::sync::Mutex<Vec<u32>>>,
5556 succeed_on: u32,
5557 }
5558
5559 #[async_trait::async_trait]
5560 impl ToolExecutor for AttemptRecordingExecutor {
5561 async fn execute(&self, _tool: &str, _params: &Value) -> Result<Value, String> {
5562 Ok(Value::Null)
5563 }
5564 async fn execute_with_action_in_session(
5565 &self,
5566 _tool: &str,
5567 _params: &Value,
5568 _action_id: &str,
5569 _timeout_ms: Option<u64>,
5570 _session_id: Option<&str>,
5571 attempt: u32,
5572 ) -> Result<Value, String> {
5573 self.seen.lock().await.push(attempt);
5574 if attempt >= self.succeed_on {
5575 Ok(serde_json::json!({ "ok": true }))
5576 } else {
5577 Err("transient".to_string())
5578 }
5579 }
5580 }
5581
5582 /// Parslee-ai/car#928 — the retry counter the executor receives must be the
5583 /// engine's real one, and it must advance.
5584 ///
5585 /// The WS executor hardcoded `attempt: 1` onto the `tools.execute` payload,
5586 /// so the field a host would build a retry-disambiguating join on never
5587 /// varied. Asserting on the sequence the executor actually observes is the
5588 /// property that matters: a frame test can only prove the payload carries
5589 /// whatever it was handed, not that the engine hands it the truth.
5590 #[tokio::test]
5591 async fn the_executor_sees_the_engines_real_attempt_sequence() {
5592 let seen = Arc::new(tokio::sync::Mutex::new(Vec::new()));
5593 let rt = Runtime::new();
5594 rt.register_tool("flaky").await;
5595 rt.set_executor(Arc::new(AttemptRecordingExecutor {
5596 seen: seen.clone(),
5597 succeed_on: 3,
5598 }))
5599 .await;
5600
5601 let proposal: ActionProposal = serde_json::from_value(serde_json::json!({
5602 "source": "test",
5603 "actions": [{
5604 "id": "a0",
5605 "type": "tool_call",
5606 "tool": "flaky",
5607 "parameters": {},
5608 "dependencies": [],
5609 "failure_behavior": "retry",
5610 "max_retries": 3,
5611 }],
5612 }))
5613 .expect("proposal deserializes");
5614
5615 let result = rt.execute(&proposal).await;
5616 assert_eq!(
5617 result.results[0].status,
5618 ActionStatus::Succeeded,
5619 "third attempt succeeds: {:?}",
5620 result.results[0].error
5621 );
5622 assert_eq!(
5623 *seen.lock().await,
5624 vec![1, 2, 3],
5625 "1-based and advancing — a constant here is the #928 bug"
5626 );
5627 }
5628
5629 #[tokio::test(start_paused = true)]
5630 async fn no_budget_applies_no_executor_deadline() {
5631 // The `None` path: the executor applies NO deadline, so a dispatch that
5632 // outlives any per-attempt budget still completes (the WS callback wait
5633 // is the sole bound on that path, exercised in car-server-core). Tokio's
5634 // paused clock advances the 300ms dispatch without a wall-clock wait.
5635 let seen = Arc::new(AtomicU64::new(0));
5636 let rt = Runtime::new();
5637 rt.register_tool("slow").await;
5638 rt.set_executor(Arc::new(RecordingExecutor {
5639 seen_timeout_ms: seen.clone(),
5640 delay_ms: 300,
5641 }))
5642 .await;
5643
5644 let result = rt.execute(&one_tool_proposal(None)).await;
5645 assert_eq!(
5646 seen.load(Ordering::SeqCst),
5647 u64::MAX,
5648 "None budget must be forwarded as None"
5649 );
5650 assert_eq!(result.results[0].status, ActionStatus::Succeeded);
5651 }
5652}