car_engine/executor.rs
1//! Core execution engine — propose → validate → execute → commit.
2
3use crate::cache::ResultCache;
4use crate::capabilities::CapabilitySet;
5use crate::checkpoint::Checkpoint;
6use crate::rate_limit::{RateLimit, RateLimiter};
7use car_eventlog::{EventKind, EventLog, SpanStatus};
8use car_ir::{
9 build_dag, AcceptedProposalPreimage, Action, ActionProposal, ActionResult, ActionStatus,
10 ActionType, CostSummary, FailureBehavior, ProposalLineageEntry, ProposalLineageStatus,
11 ProposalResult, StateMutation, ToolFailure, ToolFailureClassification, ToolSchema,
12};
13use car_policy::{ApprovalDecision, ApprovalLedger, ApprovalRecord, PermissionTier, PolicyEngine};
14use car_state::{RestoreDurability, StateStore};
15use car_validator::validate_action;
16use serde_json::Value;
17use sha2::{Digest, Sha256};
18use std::collections::HashMap;
19use std::sync::Arc;
20use std::time::Duration;
21use tokio::sync::{Mutex as TokioMutex, RwLock as TokioRwLock};
22use tokio::time::timeout;
23use tracing::instrument;
24use uuid::Uuid;
25
26/// Retry backoff constants.
27const RETRY_BASE_DELAY_MS: u64 = 100;
28const RETRY_BACKOFF_FACTOR: u64 = 2;
29const ROLLBACK_WARNING: &str =
30 "proposal aborted; state effects were rolled back; external effects may remain and were not undone";
31const PLAN_FALLBACK_ROLLBACK_WARNING: &str =
32 "planning candidate rejected; state effects were rolled back; external effects may remain and were not undone";
33const ROLLBACK_DURABILITY_ERROR: &str =
34 "durable state rollback failed before publication; in-memory state and idempotency entries were preserved";
35const ROLLBACK_DURABILITY_UNKNOWN: &str =
36 "state rollback was published but parent-directory durability is unknown; idempotency entries were invalidated for exact retry";
37
38// ---------------------------------------------------------------------------
39// Replan types — failure recovery via model callback
40// ---------------------------------------------------------------------------
41
42/// Callback trait for replanning failed proposals.
43/// Implement this to let the runtime ask the model for an alternative plan
44/// when a proposal aborts.
45#[async_trait::async_trait]
46pub trait ReplanCallback: Send + Sync {
47 async fn replan(&self, ctx: &ReplanContext) -> Result<ActionProposal, String>;
48}
49
50/// Context provided to the replan callback so the model can generate an alternative.
51#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
52pub struct ReplanContext {
53 /// Original proposal ID.
54 pub proposal_id: String,
55 /// Which replan attempt this is (1-indexed; attempt 1 = first replan after initial failure).
56 pub attempt: u32,
57 /// Actions that failed and caused the abort.
58 pub failed_actions: Vec<FailedActionSummary>,
59 /// Action IDs that succeeded before the abort (now rolled back).
60 pub completed_action_ids: Vec<String>,
61 /// State snapshot after rollback.
62 pub state_snapshot: HashMap<String, Value>,
63 /// How many replans remain.
64 pub replans_remaining: u32,
65 /// Original proposal source (model name, agent, etc.).
66 pub original_source: String,
67 /// Total actions in the original proposal.
68 pub original_action_count: usize,
69 /// The original goal/context from the proposal (for generating alternatives).
70 pub original_context: HashMap<String, Value>,
71}
72
73/// Summary of a failed action, included in ReplanContext.
74#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
75pub struct FailedActionSummary {
76 pub action_id: String,
77 pub tool: Option<String>,
78 pub error: String,
79 pub parameters: HashMap<String, Value>,
80}
81
82/// Configuration for the replan loop.
83#[derive(Debug, Clone)]
84pub struct ReplanConfig {
85 /// Maximum number of replan attempts. 0 = disabled (default).
86 pub max_replans: u32,
87 /// Delay in milliseconds between replan attempts. Prevents burning through
88 /// attempts instantly if the model returns garbage fast. 0 = no delay.
89 pub delay_ms: u64,
90 /// If true, replan proposals are scored via car-planner's verify() before
91 /// execution. Proposals with errors are rejected without executing.
92 /// Prevents the engine from running a worse plan than the one that failed.
93 pub verify_before_execute: bool,
94 /// When true, validator/policy/capability rejections (ActionStatus::Rejected)
95 /// also trigger rollback + replan, not just runtime Failed actions.
96 /// Default false (conservative: preserves prior abort-only-on-Failed behavior).
97 pub replan_on_rejected: bool,
98}
99
100/// How the runtime treats a proposal that conflicts with the current
101/// shared state on a pre-execution transactional check (survey §4.3/§5.2.4
102/// — `car_verify::check_transaction` against the versioned `StateStore`).
103#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
104pub enum TransactionCheckMode {
105 /// Don't run the check (default — preserves prior behavior exactly).
106 #[default]
107 Off,
108 /// Run it and emit any conflicts as `TransactionConflict` telemetry, but
109 /// execute anyway.
110 Warn,
111 /// Run it; if any conflict is found, reject the proposal without
112 /// executing (the conflicting actions become `Rejected` results).
113 Strict,
114}
115
116impl Default for ReplanConfig {
117 fn default() -> Self {
118 Self {
119 max_replans: 0,
120 delay_ms: 0,
121 verify_before_execute: true,
122 replan_on_rejected: false,
123 }
124 }
125}
126
127/// Trait for tool execution. Implement this to provide tools to the runtime.
128///
129/// In-process: implement directly with function calls.
130/// Daemon mode: implement by sending JSON-RPC to the client.
131#[async_trait::async_trait]
132pub trait ToolExecutor: Send + Sync {
133 async fn execute(&self, tool: &str, params: &Value) -> Result<Value, String>;
134
135 /// Variant that also carries the originating proposal `Action.id`
136 /// and its `timeout_ms` budget.
137 ///
138 /// `action_id`: WS-based executors (`car-server-core::WsToolExecutor`)
139 /// use it so the daemon-initiated `tools.execute` request to the client
140 /// carries the same id the host's process-wide handler is keyed on
141 /// — without this round-trip the host can't disambiguate concurrent
142 /// callbacks for the same tool (Parslee-ai/car-releases#43 follow-up).
143 ///
144 /// `timeout_ms`: the action's per-call budget. WS executors MUST bound
145 /// their callback wait by this (falling back to a default when `None`)
146 /// so the daemon→host wait and the executor's own action deadline stay
147 /// coordinated — otherwise a hardcoded inner wait reaps a call the outer
148 /// deadline still permits (Parslee-ai/car#259). In-process executors
149 /// that don't need either can keep the default forward to [`execute`](crate::executor::ToolExecutor::execute).
150 async fn execute_with_action(
151 &self,
152 tool: &str,
153 params: &Value,
154 _action_id: &str,
155 _timeout_ms: Option<u64>,
156 ) -> Result<Value, String> {
157 self.execute(tool, params).await
158 }
159
160 /// Variant that also carries the Runtime execution session and the retry
161 /// attempt. Executors that retain safety-relevant state across calls (such
162 /// as a read-before-edit ledger) override this to keep that state isolated
163 /// by session. Existing executors retain their current behavior through the
164 /// default delegation.
165 ///
166 /// `attempt` is **1-based**: `1` on the first try, `2` on the first retry.
167 /// WS executors surface it on the `tools.execute` payload so a host can
168 /// tell which retry it is serving — `action_id` cannot, being neither
169 /// unique across attempts nor varying between them.
170 ///
171 /// It is threaded from `execute_with_retry`'s own counter rather than
172 /// synthesized here. It was previously hardcoded to `1` at the one place
173 /// that put it on the wire, which made the field a constant and any join
174 /// built on it silently degenerate (Parslee-ai/car#928).
175 async fn execute_with_action_in_session(
176 &self,
177 tool: &str,
178 params: &Value,
179 action_id: &str,
180 timeout_ms: Option<u64>,
181 _session_id: Option<&str>,
182 _attempt: u32,
183 ) -> Result<Value, String> {
184 self.execute_with_action(tool, params, action_id, timeout_ms)
185 .await
186 }
187
188 /// Execute one action while allowing a transport to return runtime-observed
189 /// state mutations alongside its ordinary tool output.
190 ///
191 /// The default preserves every direct/internal executor: its existing
192 /// output is wrapped with no mutations, and declared `expected_effects`
193 /// remain assertions rather than writes. A transport that overrides this
194 /// method must validate its mutation envelope against `expected_effects`
195 /// before returning. The Runtime repeats the key-set and I-JSON checks and
196 /// applies accepted changes through its own [`StateStore`].
197 async fn execute_with_action_state_in_session(
198 &self,
199 tool: &str,
200 params: &Value,
201 action_id: &str,
202 timeout_ms: Option<u64>,
203 session_id: Option<&str>,
204 attempt: u32,
205 _expected_effects: &HashMap<String, Value>,
206 _return_schema: Option<&Value>,
207 ) -> Result<ToolExecution, String> {
208 self.execute_with_action_in_session(
209 tool, params, action_id, timeout_ms, session_id, attempt,
210 )
211 .await
212 .map(ToolExecution::output_only)
213 }
214
215 /// Execute one action with typed failure evidence.
216 ///
217 /// This is a new default-delegating layer rather than a signature change
218 /// to the existing execution methods. Existing executors therefore retain
219 /// their exact behavior: every legacy string error becomes an ordinary,
220 /// non-terminal failure. Executors opt into fail-stop behavior only by
221 /// overriding this method and returning [`ToolFailure::terminal`].
222 ///
223 /// The runtime runs its built-in tool instead only for a failure that
224 /// [`ToolFailure::is_unhandled`]. The default conversion keeps the legacy
225 /// `"unknown tool"` string sentinel for in-process executors. An executor
226 /// that relays another process's errors (a WS client, an MCP server) must
227 /// override this method and return [`ToolFailure::ordinary`] for them, so a
228 /// peer's text can never pick the built-in (Parslee-ai/car#3104).
229 async fn execute_classified(
230 &self,
231 tool: &str,
232 params: &Value,
233 action_id: &str,
234 timeout_ms: Option<u64>,
235 session_id: Option<&str>,
236 attempt: u32,
237 expected_effects: &HashMap<String, Value>,
238 return_schema: Option<&Value>,
239 ) -> Result<ToolExecution, ToolFailure> {
240 self.execute_with_action_state_in_session(
241 tool,
242 params,
243 action_id,
244 timeout_ms,
245 session_id,
246 attempt,
247 expected_effects,
248 return_schema,
249 )
250 .await
251 .map_err(ToolFailure::from)
252 }
253
254 /// Streaming entry point for detached invocation modes (C2). Start
255 /// the tool and return a channel of [`car_ir::ToolStreamChunk`]s; the
256 /// runtime drains it into the per-runtime handle registry while the
257 /// DAG proceeds. End the stream with a terminal chunk (`done` /
258 /// `error`); dropping the sender without one is reported as failure.
259 /// A cooperative executor should stop work when the receiver returned
260 /// here is dropped (that's what cancellation looks like from its side).
261 ///
262 /// Default: unsupported — existing one-shot executors compile and
263 /// behave unchanged; a detached action against them is rejected with
264 /// this error.
265 async fn execute_stream(
266 &self,
267 tool: &str,
268 _params: &Value,
269 _action_id: &str,
270 ) -> Result<tokio::sync::mpsc::Receiver<car_ir::ToolStreamChunk>, String> {
271 Err(format!(
272 "tool '{tool}': this executor does not support streaming/long-running invocation"
273 ))
274 }
275}
276
277/// Result of a tool dispatch before the Runtime commits observed state.
278#[derive(Debug, Clone, PartialEq)]
279pub struct ToolExecution {
280 pub output: Value,
281 pub state_changes: HashMap<String, Value>,
282}
283
284impl ToolExecution {
285 pub fn output_only(output: Value) -> Self {
286 Self {
287 output,
288 state_changes: HashMap::new(),
289 }
290 }
291}
292
293/// Deterministic key for idempotency deduplication.
294/// Deterministic key for idempotency deduplication. Carries the tenant
295/// dimension (linus review): without it, tenant A's cached result for
296/// an identical idempotent action was served to tenant B — a
297/// cross-tenant data leak through the dedup cache. Unscoped executions
298/// keep their historical keys (empty tenant segment).
299fn idempotency_key(action: &Action, scope: Option<&crate::scope::RuntimeScope>) -> String {
300 let sorted: std::collections::BTreeMap<_, _> = action.parameters.iter().collect();
301 let params = serde_json::to_string(&sorted).unwrap_or_default();
302 let tenant = scope.and_then(|s| s.tenant_id.as_deref()).unwrap_or("");
303 format!(
304 "{}:{}:{}:{}",
305 tenant,
306 serde_json::to_string(&action.action_type).unwrap_or_default(),
307 action.tool.as_deref().unwrap_or(""),
308 params
309 )
310}
311
312/// Exact normal-serde proposal preimage plus its lowercase RFC 8785/JCS
313/// SHA-256 identity. The shared inference helper is the repository's settled
314/// canonicalizer; journal identity must not drift onto a second JSON sorter.
315fn proposal_journal_identity(proposal: &ActionProposal) -> Result<(Value, String), String> {
316 let preimage = serde_json::to_value(proposal)
317 .map_err(|error| format!("proposal serialization failed: {error}"))?;
318 let canonical = car_inference::catalog_identity::canonical_json(&preimage)?;
319 let digest = format!("{:x}", Sha256::digest(canonical.as_bytes()));
320 Ok((preimage, digest))
321}
322
323fn proposal_digest(proposal: &ActionProposal) -> Result<String, String> {
324 proposal_journal_identity(proposal).map(|(_, digest)| digest)
325}
326
327fn proposal_rejection_boundary_data(
328 proposal: &ActionProposal,
329 proposal_digest: Option<&str>,
330 reason: &str,
331) -> HashMap<String, Value> {
332 let mut data: HashMap<String, Value> = [
333 ("affected_actions".to_string(), Value::Array(Vec::new())),
334 (
335 "rolled_back_changes".to_string(),
336 Value::Object(serde_json::Map::new()),
337 ),
338 (
339 "changes_semantics".to_string(),
340 Value::from("proposal_rejected_no_state_commit"),
341 ),
342 ("attempted".to_string(), Value::from(false)),
343 ("stage".to_string(), Value::from("proposal_rejection")),
344 ("rejection_reason".to_string(), Value::from(reason)),
345 ]
346 .into();
347 if let Ok(preimage) = serde_json::to_value(proposal) {
348 data.insert("proposal".to_string(), preimage);
349 }
350 if let Some(digest) = proposal_digest {
351 data.insert("proposal_digest".to_string(), Value::from(digest));
352 }
353 data
354}
355
356fn proposal_lineage_entry(
357 proposal: &ActionProposal,
358 generation: u32,
359 status: ProposalLineageStatus,
360 rejection_reason: Option<String>,
361) -> ProposalLineageEntry {
362 ProposalLineageEntry {
363 generation,
364 proposal_id: proposal.id.clone(),
365 proposal_digest: proposal_digest(proposal).ok(),
366 status,
367 rejection_reason,
368 }
369}
370
371fn finalize_proposal_result(
372 mut result: ProposalResult,
373 original_proposal_id: &str,
374 lineage: &[ProposalLineageEntry],
375 accepted_proposal_preimages: &[AcceptedProposalPreimage],
376) -> ProposalResult {
377 result.original_proposal_id = original_proposal_id.to_string();
378 result.replan_lineage = lineage.to_vec();
379 result.accepted_proposal_preimages = accepted_proposal_preimages.to_vec();
380 result
381}
382
383/// Validate the proposal-local action identity boundary used by DAG execution,
384/// action receipts, results, and state-transition attribution. A retry reuses
385/// one admitted action id across attempts; two declared actions may not share
386/// an id because every downstream join would become ambiguous.
387pub fn validate_proposal_action_ids(proposal: &ActionProposal) -> Result<(), String> {
388 let mut seen = std::collections::HashSet::with_capacity(proposal.actions.len());
389 for action in &proposal.actions {
390 if !seen.insert(action.id.as_str()) {
391 return Err(format!("duplicate action id '{}'", action.id));
392 }
393 }
394 Ok(())
395}
396
397fn validate_proposal_retry_limits(proposal: &ActionProposal) -> Result<(), String> {
398 for action in &proposal.actions {
399 if action.failure_behavior == FailureBehavior::Retry
400 && action.max_retries.checked_add(1).is_none()
401 {
402 return Err(format!(
403 "action '{}' retry attempt count overflows u32",
404 action.id
405 ));
406 }
407 }
408 Ok(())
409}
410
411fn requires_integrity_rollback(result: &ActionResult) -> bool {
412 result.status == ActionStatus::Failed
413 && result.error.as_deref().is_some_and(|error| {
414 error.starts_with("tool output failed JCS/I-JSON validation after dispatch")
415 })
416}
417
418fn record_rollback_durability_error(
419 results: &mut [ActionResult],
420 prefix: &str,
421 error: &str,
422) -> String {
423 let detail = format!("{prefix}: {error}");
424 if let Some(result) = results
425 .iter_mut()
426 .find(|result| result.status != ActionStatus::Succeeded)
427 {
428 result.error = Some(match result.error.take() {
429 Some(existing) => format!("{existing}; {detail}"),
430 None => detail.clone(),
431 });
432 }
433 detail
434}
435
436fn rejected_result(action_id: &str, error: String) -> ActionResult {
437 ActionResult {
438 action_id: action_id.to_string(),
439 status: ActionStatus::Rejected,
440 output: None,
441 error: Some(error),
442 terminal: false,
443 state_changes: HashMap::new(),
444 rolled_back: false,
445 duration_ms: None,
446 timestamp: chrono::Utc::now(),
447 }
448}
449
450/// Whether the actions of one `build_dag` level run one after another, in level
451/// (index) order, rather than concurrently. A level runs sequentially when it
452/// holds one action or any action in it has `FailureBehavior::Abort` — the
453/// default — so an earlier action in such a level finishes before a later one
454/// is validated. The static-verification gate's cycle rule reads this too, so
455/// its notion of "finishes first" is the executor's by construction.
456pub(crate) fn level_runs_sequentially(level: &[usize], actions: &[Action]) -> bool {
457 level.len() == 1
458 || level
459 .iter()
460 .any(|&i| actions[i].failure_behavior == FailureBehavior::Abort)
461}
462
463/// The tenant namespace an action's own state keys live in when its proposal
464/// runs under `tenant` (car#3121). Dispatch routes `StateWrite`, `StateRead`
465/// and `Assertion` through `StateStore::scoped(tenant)`, while a `ToolCall`
466/// runs on the unscoped path: its handler reads and `state.set`s the shared
467/// store. So a tool call's preconditions, dependencies and assumptions name
468/// shared keys, and a state action's name its tenant's. Every pre-execution
469/// check (validator, admission gate, transactional pre-check, replan
470/// verification) resolves an action's keys through this, so it judges each
471/// action against the state that action executes against.
472pub(crate) fn action_state_tenant<'a>(action: &Action, tenant: Option<&'a str>) -> Option<&'a str> {
473 match action.action_type {
474 ActionType::ToolCall => None,
475 ActionType::StateWrite | ActionType::StateRead | ActionType::Assertion => tenant,
476 }
477}
478
479/// The store keys an action's declared writes (`expected_effects`, and a
480/// `state_write`'s key) can land on when its proposal runs under `tenant`.
481/// Each is the key in the action's own namespace ([`action_state_tenant`]);
482/// a tool call's declared effects are also its tenant's copies, because its
483/// handler can `state.set` the shared key while its callback `state_changes`
484/// (whose keys must equal `expected_effects`) land in the tenant's namespace.
485/// The static-verification gate and the transactional pre-check both read an
486/// action's writes through this, so they model the same keys (car#3133).
487/// Sorted and deduplicated.
488pub(crate) fn action_write_store_keys(action: &Action, tenant: Option<&str>) -> Vec<String> {
489 let own_tenant = action_state_tenant(action, tenant);
490 let state_write_key = (action.action_type == ActionType::StateWrite)
491 .then(|| action.parameters.get("key").and_then(|v| v.as_str()))
492 .flatten();
493 let mut keys: Vec<String> = state_write_key
494 .into_iter()
495 .chain(action.expected_effects.keys().map(String::as_str))
496 .flat_map(|key| {
497 let own = car_state::tenant_key(own_tenant, key);
498 let tenant_copy = (action.action_type == ActionType::ToolCall)
499 .then(|| car_state::tenant_key(tenant, key))
500 .filter(|k| *k != own);
501 std::iter::once(own).chain(tenant_copy)
502 })
503 .collect();
504 keys.sort_unstable();
505 keys.dedup();
506 keys
507}
508
509/// The state view an action's own keys are read through when its proposal
510/// runs under `tenant`: its tenant's namespace for a state action, and for a
511/// tool call the store under raw names restricted to the keys visible to
512/// `tenant` (shared keys and its own, never another tenant's;
513/// `StateStore::visible_to`). With no tenant, the whole store.
514fn action_state_view<'a>(
515 state: &'a StateStore,
516 action: &Action,
517 tenant: Option<&'a str>,
518) -> Box<dyn car_ir::precondition::StateView + Send + Sync + 'a> {
519 match action_state_tenant(action, tenant) {
520 Some(t) => Box::new(state.scoped(Some(t))),
521 None => Box::new(state.visible_to(tenant)),
522 }
523}
524
525/// `proposal` with every state key a tenant-namespaced action names
526/// rewritten to the store key it executes against (see
527/// [`action_state_tenant`]), so a whole-store check such as
528/// `car_verify::check_transaction` reads each action's keys in its own
529/// namespace. Borrowed unchanged when there is no tenant.
530///
531/// A tool call's keys stay shared, but its write set also gets the tenant's
532/// copies of its declared effects ([`action_write_store_keys`], car#3133).
533fn proposal_in_store_keys<'p>(
534 proposal: &'p ActionProposal,
535 tenant: Option<&str>,
536) -> std::borrow::Cow<'p, ActionProposal> {
537 if tenant.is_none_or(|t| t.is_empty()) {
538 return std::borrow::Cow::Borrowed(proposal);
539 }
540 let mut resolved = proposal.clone();
541 for action in &mut resolved.actions {
542 let Some(t) = action_state_tenant(action, tenant) else {
543 for k in action_write_store_keys(action, tenant) {
544 if !action.write_set.contains(&k) {
545 action.write_set.push(k);
546 }
547 }
548 continue;
549 };
550 let key = |k: &str| car_state::tenant_key(Some(t), k);
551 for k in action
552 .state_dependencies
553 .iter_mut()
554 .chain(&mut action.read_set)
555 .chain(&mut action.write_set)
556 {
557 *k = key(k);
558 }
559 for pre in &mut action.preconditions {
560 pre.key = key(&pre.key);
561 }
562 for assumption in &mut action.assumptions {
563 assumption.key = key(&assumption.key);
564 }
565 action.expected_effects = std::mem::take(&mut action.expected_effects)
566 .into_iter()
567 .map(|(k, v)| (key(&k), v))
568 .collect();
569 if let Some(Value::String(k)) = action.parameters.get_mut("key") {
570 *k = key(k);
571 }
572 }
573 std::borrow::Cow::Owned(resolved)
574}
575
576/// Capture only the state keys relevant to an action (state_dependencies + expected_effects).
577/// Returns an empty map if the action declares no relevant keys (e.g., tool calls
578/// that don't interact with state). `state` is the view the action executes
579/// against ([`action_state_tenant`]), so the values recorded are the ones the
580/// action saw.
581fn snapshot_relevant_keys(
582 state: &dyn car_ir::precondition::StateView,
583 action: &Action,
584) -> HashMap<String, Value> {
585 let mut keys: std::collections::HashSet<&str> = std::collections::HashSet::new();
586 for dep in &action.state_dependencies {
587 keys.insert(dep.as_str());
588 }
589 for key in action.expected_effects.keys() {
590 keys.insert(key.as_str());
591 }
592 // For state_write actions, capture the key being written
593 if action.action_type == ActionType::StateWrite {
594 if let Some(key) = action.parameters.get("key").and_then(|v| v.as_str()) {
595 keys.insert(key);
596 }
597 }
598
599 if keys.is_empty() {
600 // No declared state interaction — skip snapshot (empty map)
601 return HashMap::new();
602 }
603
604 keys.iter()
605 .filter_map(|&k| state.get_value(k).map(|v| (k.to_string(), v)))
606 .collect()
607}
608
609fn skipped_result(action_id: &str, reason: &str) -> ActionResult {
610 ActionResult {
611 action_id: action_id.to_string(),
612 status: ActionStatus::Skipped,
613 output: None,
614 error: Some(reason.to_string()),
615 terminal: false,
616 state_changes: HashMap::new(),
617 rolled_back: false,
618 duration_ms: None,
619 timestamp: chrono::Utc::now(),
620 }
621}
622
623fn insert_tool_event_provenance(
624 data: &mut HashMap<String, Value>,
625 action: &Action,
626 source: Option<car_ir::ToolSourceKind>,
627) {
628 if let Some(tool) = action.tool.as_deref() {
629 data.insert("tool".to_string(), Value::from(tool));
630 data.insert(
631 "tool_source".to_string(),
632 Value::from(
633 source
634 .unwrap_or(car_ir::ToolSourceKind::UserDefined)
635 .as_str(),
636 ),
637 );
638 }
639}
640
641/// SHA-256 over the RFC 8785/JCS rendering of only the action parameters.
642///
643/// The accepted proposal has already crossed `proposal_journal_identity`, so
644/// canonicalization cannot fail here without violating that admission
645/// invariant. Keeping only the digest on outcome events lets a later join find
646/// the raw parameters in `ProposalReceived` without copying them into every
647/// failure record.
648fn action_params_digest(action: &Action) -> String {
649 // Serialize the map directly: RFC 8785/JCS sorts object keys
650 // recursively at render time (see `canonical_json` and the
651 // `canonical_json_sorts_object_keys_recursively` test), so the map's
652 // iteration order does not affect the digest and cloning into a
653 // `Value::Object` would be pure per-action allocation overhead.
654 let canonical = car_inference::catalog_identity::canonical_json(&action.parameters)
655 .expect("accepted action parameters must remain RFC 8785/JCS canonicalizable");
656 format!("{:x}", Sha256::digest(canonical.as_bytes()))
657}
658
659fn insert_action_outcome_signal(
660 data: &mut HashMap<String, Value>,
661 action: &Action,
662 params_digest: &str,
663 error_class: Option<&str>,
664) {
665 data.insert(
666 "params_digest".to_string(),
667 Value::from(params_digest.to_string()),
668 );
669 data.insert(
670 "expected_effects".to_string(),
671 serde_json::to_value(&action.expected_effects)
672 .expect("action expected_effects must serialize as JSON"),
673 );
674 if let Some(error_class) = error_class {
675 data.insert("error_class".to_string(), Value::from(error_class));
676 }
677}
678
679/// Normalize execution failures into a stable, deliberately low-cardinality
680/// event-log vocabulary. Validation and policy checks that happen before
681/// dispatch remain `ActionRejected`; this maps failures reached after an
682/// action began executing.
683pub(crate) fn action_error_class(
684 action: &Action,
685 error: &str,
686 engine_timeout: bool,
687) -> &'static str {
688 // Classification is by substring over prose and is a published wire
689 // contract (docs/agent-ir-spec.md `error_class`). Each literal is pinned
690 // to its producer below; changing a producer's wording must update the
691 // matching literal (and `action_failure_error_class_mapping_is_stable`).
692 let error = error.to_ascii_lowercase();
693 // "callback timed out" — the engine deadline itself is typed via
694 // `engine_timeout`, so this literal only covers the detached-dispatch
695 // producer `tool '{tool}' callback timed out ({s}s)`
696 // (car-server-core/src/session.rs, `reap_detached_calls`/wait path).
697 // A tool faithfully relaying a *downstream* timeout in its own message
698 // also lands here; accepted drift, low-cardinality vocabulary.
699 if engine_timeout || error.contains("callback timed out") {
700 "timeout"
701 } else if error.starts_with("denied by policy:") || error.starts_with("rejected by policy:") {
702 // Exact producers return this prefix unwrapped from the tool's
703 // guard check: car-server-core/src/assistant/executor.rs and
704 // car-server-core/src/coder/shell_tool.rs
705 // (`format!("denied by policy: {reason}")`). An executor that
706 // *wraps* the denial (e.g. `tool 'x' failed: denied by policy: …`)
707 // deliberately falls through to `tool_error` — see the wrapped
708 // case in `action_failure_error_class_mapping_is_stable`.
709 "rejected_by_policy"
710 } else if error.contains("jcs/i-json validation")
711 || error.contains("output validation:")
712 || error.contains("invalid return json schema")
713 || error.contains("must return exact envelope {output,state_changes}")
714 || error.contains("state_changes must be an object")
715 || error.contains("state_changes keys do not match")
716 || error.contains("state_changes serialization failed")
717 {
718 "validation"
719 } else if action.tool.is_some() {
720 "tool_error"
721 } else {
722 "unknown"
723 }
724}
725
726fn action_outcome_data(
727 stage: &str,
728 reason: &str,
729 attempted: bool,
730 attempt: Option<u32>,
731) -> HashMap<String, Value> {
732 let mut data: HashMap<String, Value> = [
733 ("stage".to_string(), Value::from(stage)),
734 ("reason".to_string(), Value::from(reason)),
735 ("attempted".to_string(), Value::from(attempted)),
736 ]
737 .into();
738 if let Some(attempt) = attempt {
739 data.insert("attempt".to_string(), Value::from(attempt));
740 }
741 data
742}
743
744/// Prefix on the `error` field of an `ActionResult` that distinguishes
745/// "the user pulled the plug" from "earlier abort cascaded." The
746/// `ActionStatus` itself is `Skipped` in both cases (introducing a
747/// new variant ripples through every IR consumer + FFI binding); the
748/// prefix lets callers like the A2A bridge tell the cases apart
749/// without string-matching a magic literal.
750pub const CANCELED_PREFIX: &str = "canceled: ";
751
752/// Result for an action that didn't run because the proposal was
753/// cancelled mid-flight. Distinct from `Skipped` so callers can tell
754/// "didn't run because of an earlier abort" from "didn't run because
755/// the user pulled the plug."
756fn canceled_result(action_id: &str, reason: &str) -> ActionResult {
757 ActionResult {
758 action_id: action_id.to_string(),
759 status: ActionStatus::Skipped,
760 output: None,
761 error: Some(format!("{}{}", CANCELED_PREFIX, reason)),
762 terminal: false,
763 state_changes: HashMap::new(),
764 rolled_back: false,
765 duration_ms: None,
766 timestamp: chrono::Utc::now(),
767 }
768}
769
770/// Format a tool result for feeding back to a model.
771pub fn format_tool_result(result: &ActionResult) -> String {
772 match result.status {
773 ActionStatus::Succeeded => match &result.output {
774 Some(v) => serde_json::to_string(v).unwrap_or_else(|_| v.to_string()),
775 None => String::new(),
776 },
777 ActionStatus::Rejected => format!("[REJECTED] {}", result.error.as_deref().unwrap_or("")),
778 ActionStatus::Failed => format!("[FAILED] {}", result.error.as_deref().unwrap_or("")),
779 _ => format!(
780 "[{:?}] {}",
781 result.status,
782 result.error.as_deref().unwrap_or("")
783 ),
784 }
785}
786
787/// Names the shape [`format_tool_result_for_model`] gives a model, so a
788/// measurement can record what its model read. Change it whenever that shape
789/// changes: results measured under different formats are not comparable.
790pub const MODEL_OBSERVATION_FORMAT: &str = "text-v1";
791
792/// Fields that say how a call went; [`render_tool_output`] puts them first.
793const CONTROL_FIELDS: [&str; 6] = [
794 "truncated",
795 "error",
796 "ok",
797 "status",
798 "exit_code",
799 "timed_out",
800];
801
802/// A tool result's output as a model should read it: outcome fields first
803/// (`truncated`, `error`, `ok`, `status`, `exit_code`, `timed_out`), one line
804/// each; then other scalars as `key: value` lines; then each long or
805/// multi-line string (a file's content, a command's output) as a raw block
806/// headed `key (N bytes):`, and each array of records as `key (N records):`
807/// with one line per record. A cut at the budget loses the tail of the bulk,
808/// never the verdict.
809///
810/// Journals, receipts and events keep the JSON; only the model's copy changes.
811/// The model used to be handed the JSON too — so a file arrived as one line of
812/// `\n`, `\t` and `\"` escapes. On claude-haiku-4-5, 23% of native `edit_file`
813/// calls (42 of 182) missed with "old_text not found", against 1 of 60 for
814/// Claude Code's `Edit` on the same model and tasks; the misses carried escapes
815/// copied out of the JSON (`\"\"\"` for a docstring's `"""`). Claude Code's
816/// `Read` returns plain numbered lines; this is that shape.
817///
818/// The text reads back exactly with [`parse_rendered_output`], apart from the
819/// outcome fields (shown on one line) and a record list (read back as text):
820/// blocks carry their length, so no content line can be mistaken for a field,
821/// and a string that would read back as another JSON type (`"123"`, `"null"`)
822/// is written quoted.
823pub fn render_tool_output(v: &Value) -> String {
824 let map = match v {
825 Value::Object(map) => map,
826 Value::String(s) => return s.clone(),
827 other => return other.to_string(),
828 };
829 let mut head = Vec::new();
830 let mut lines = Vec::new();
831 let mut blocks = Vec::new();
832 // Sorted, so every build shows the same bytes: serde_json's map order
833 // depends on whether `preserve_order` is enabled anywhere in the graph.
834 let mut fields: Vec<_> = map.iter().collect();
835 fields.sort_by(|a, b| a.0.cmp(b.0));
836 for (key, value) in fields {
837 match value {
838 // Outcome fields lead, on one line each, whatever their size: a cut
839 // at the budget must never take the verdict with it.
840 _ if CONTROL_FIELDS.contains(&key.as_str()) => {
841 // Readable, not escaped: a traceback in `error` shows as one
842 // line with its breaks turned into spaces, keeping its start
843 // and its end (where the exception is) when it is long.
844 let line = match value {
845 Value::String(s) => scalar_text(&Value::String(s.replace('\n', " "))),
846 other => scalar_text(other),
847 };
848 head.push(format!(
849 "{}: {}",
850 key_text(key),
851 clip_keeping_ends(&line, CONTROL_FIELD_BYTES)
852 ));
853 }
854 Value::String(s) if s.contains('\n') || s.len() > 200 => {
855 blocks.push(format!("{} ({} bytes):\n{s}", key_text(key), s.len()))
856 }
857 Value::Array(items) if !items.is_empty() && items.iter().all(Value::is_object) => {
858 let records: Vec<String> = items.iter().map(render_record).collect();
859 blocks.push(format!(
860 "{} ({} records):\n{}",
861 key_text(key),
862 records.len(),
863 records.join("\n")
864 ));
865 }
866 other => lines.push(format!("{}: {}", key_text(key), scalar_text(other))),
867 }
868 }
869 head.extend(lines);
870 head.extend(blocks);
871 head.join("\n")
872}
873
874/// A scalar as it appears after `key: `. A string is written raw unless raw
875/// text would read back as something else — JSON of another type, a quoted
876/// string, or surrounding whitespace — in which case it is written as a JSON
877/// string. Everything else is compact JSON.
878fn scalar_text(value: &Value) -> String {
879 match value {
880 Value::String(s)
881 if s.trim() != s
882 || s.is_empty()
883 || s.contains('\n')
884 || s.ends_with("):")
885 || serde_json::from_str::<Value>(s).is_ok() =>
886 {
887 serde_json::to_string(s).unwrap_or_else(|_| s.clone())
888 }
889 Value::String(s) => s.clone(),
890 other => other.to_string(),
891 }
892}
893
894/// How much of an outcome field a model is shown on its one line.
895const CONTROL_FIELD_BYTES: usize = 500;
896
897/// A key as written before `: ` or ` (N bytes):` — raw unless raw text would
898/// not read back as this key, then as a JSON string.
899fn key_text(key: &str) -> String {
900 if key.is_empty()
901 || key.contains(char::is_whitespace)
902 || key.ends_with(':')
903 || key.contains(": ")
904 || key.contains(" (")
905 || key.contains('\n')
906 || key.starts_with('"')
907 || key.starts_with('[')
908 || key.starts_with('…')
909 {
910 serde_json::to_string(key).unwrap_or_else(|_| key.to_string())
911 } else {
912 key.to_string()
913 }
914}
915
916/// Read a key written by [`key_text`] off the front of `line`, returning it
917/// and what follows it.
918fn read_key(line: &str) -> Option<(String, &str)> {
919 if line.starts_with('"') {
920 let mut stream = serde_json::Deserializer::from_str(line).into_iter::<String>();
921 let key = stream.next()?.ok()?;
922 return Some((key, &line[stream.byte_offset()..]));
923 }
924 let end = line.find(": ").or_else(|| line.find(" ("))?;
925 let key = &line[..end];
926 // `key_text` quotes any key with whitespace or a leading `[`/`…`, so a
927 // raw one without them is a key; `[FAILED] tool 'x': …` is not.
928 (!key.is_empty()
929 && !key.contains(char::is_whitespace)
930 && !key.starts_with('[')
931 && !key.starts_with('…'))
932 .then(|| (key.to_string(), &line[end..]))
933}
934
935/// `s` in at most about `max` bytes, keeping its first third and its end,
936/// with the number of bytes dropped between them.
937fn clip_keeping_ends(s: &str, max: usize) -> String {
938 if s.len() <= max {
939 return s.to_string();
940 }
941 let mut head = max / 3;
942 while !s.is_char_boundary(head) {
943 head -= 1;
944 }
945 let mut tail = s.len() - (max - head);
946 while !s.is_char_boundary(tail) {
947 tail += 1;
948 }
949 format!("{}…[{} bytes]…{}", &s[..head], tail - head, &s[tail..])
950}
951
952/// A tool result's fields from whichever form it was stored in: the JSON a
953/// receipt holds, or the rendered text a model was shown (a transcript). Any
954/// JSON parses as itself; rendered text reads back through
955/// [`parse_rendered_output`]; `None` for anything else, such as a `[FAILED] …`
956/// line.
957pub fn parse_tool_output(text: &str) -> Option<Value> {
958 serde_json::from_str::<Value>(text)
959 .ok()
960 .or_else(|| parse_rendered_output(text))
961}
962
963/// Read [`render_tool_output`]'s text back into a JSON object — for consumers
964/// that need fields (an outcome, a receipt's structured redaction) when only
965/// the model's copy survives, such as a transcript replayed without its
966/// recorded `ok`. Exact for scalars and blocks; an outcome field reads back as
967/// the one line the model saw (breaks as spaces, long ones clipped), and a
968/// record list reads back as its text. A block cut short by a budget reads
969/// back as what is left of it, and parsing stops at the first line that is not
970/// a field, which is where a budget cut the text. `None` unless the first line
971/// is a field — a `[FAILED] …` line, a refusal or prose is not rendered output.
972pub fn parse_rendered_output(text: &str) -> Option<Value> {
973 fn block_header(line: &str) -> Option<(String, usize, bool)> {
974 let (key, rest) = read_key(line)?;
975 let count = rest.strip_prefix(" (")?.strip_suffix("):")?;
976 let (n, bytes) = if let Some(n) = count.strip_suffix(" bytes") {
977 (n, true)
978 } else {
979 (count.strip_suffix(" records")?, false)
980 };
981 Some((key, n.parse().ok()?, bytes))
982 }
983 let mut map = serde_json::Map::new();
984 let mut rest = text;
985 let mut seen_block = false;
986 while !rest.is_empty() {
987 let (line, after) = rest.split_once('\n').unwrap_or((rest, ""));
988 if let Some((key, n, bytes)) = block_header(line) {
989 let (value, remaining) = if bytes {
990 let mut end = n.min(after.len());
991 while !after.is_char_boundary(end) {
992 end -= 1;
993 }
994 (
995 &after[..end],
996 after[end..].strip_prefix('\n').unwrap_or(&after[end..]),
997 )
998 } else {
999 let mut end = 0;
1000 let mut taken = 0;
1001 while taken < n && end < after.len() {
1002 end = after[end..].find('\n').map_or(after.len(), |i| end + i + 1);
1003 taken += 1;
1004 }
1005 (after[..end].trim_end_matches('\n'), &after[end..])
1006 };
1007 // First wins: the renderer writes each key once, so a second is
1008 // text a cut spliced in, and must not replace an outcome field.
1009 map.entry(key)
1010 .or_insert_with(|| Value::String(value.to_string()));
1011 seen_block = true;
1012 rest = remaining;
1013 continue;
1014 }
1015 match read_key(line) {
1016 Some((key, raw)) if !seen_block && raw.starts_with(": ") => {
1017 let raw = &raw[2..];
1018 let value = serde_json::from_str::<Value>(raw)
1019 .unwrap_or_else(|_| Value::String(raw.to_string()));
1020 map.entry(key).or_insert(value);
1021 }
1022 // The first line decides what this is: a `[FAILED] …` line, a
1023 // refusal or prose is not rendered output and is not read as
1024 // fields. After a field, an unrecognised line is where a budget
1025 // cut the text (mid-line, or through a marker spliced into a
1026 // block): keep what came before it — above all the outcome
1027 // fields, which the renderer writes first.
1028 _ if map.is_empty() => return None,
1029 _ => break,
1030 }
1031 rest = after;
1032 }
1033 (!map.is_empty()).then_some(Value::Object(map))
1034}
1035
1036/// One record of an array field (a grep match, a directory entry) on one line,
1037/// its string values raw.
1038fn render_record(item: &Value) -> String {
1039 let Value::Object(fields) = item else {
1040 return item.to_string();
1041 };
1042 let mut fields: Vec<_> = fields.iter().collect();
1043 fields.sort_by(|a, b| a.0.cmp(b.0));
1044 fields
1045 .into_iter()
1046 .map(|(k, v)| {
1047 let k = k.replace('\n', "\\n");
1048 match v {
1049 Value::String(s) => format!("{k}={}", s.replace('\n', "\\n")),
1050 other => format!("{k}={other}"),
1051 }
1052 })
1053 .collect::<Vec<_>>()
1054 .join(" ")
1055}
1056
1057/// The phrase [`truncate_keeping_ends`] puts in its elision marker, for
1058/// readers that count cut observations.
1059pub const ELIDED_MARKER: &str = "bytes elided from the middle";
1060
1061/// Fit `s` in `max` bytes keeping its first third and its end, with a marker
1062/// saying how much of the middle was dropped. For command output: the verdict
1063/// a test run or build prints comes last, so a head-only cut loses exactly the
1064/// line the model needs.
1065pub fn truncate_keeping_ends(s: &str, max: usize) -> String {
1066 if s.len() <= max {
1067 return s.to_string();
1068 }
1069 if max < 512 {
1070 // Too small for a head, a tail and the marker: keep the end, the part
1071 // that carries the verdict.
1072 let mut start = s.len() - max;
1073 while !s.is_char_boundary(start) {
1074 start += 1;
1075 }
1076 return s[start..].to_string();
1077 }
1078 let budget = max.saturating_sub(128);
1079 let mut head = budget / 3;
1080 while !s.is_char_boundary(head) {
1081 head -= 1;
1082 }
1083 let mut tail = s.len() - (budget - head);
1084 while !s.is_char_boundary(tail) {
1085 tail += 1;
1086 }
1087 format!(
1088 "{}\n…[{} {ELIDED_MARKER} of {}; re-run with narrower output to see them]…\n{}",
1089 &s[..head],
1090 tail - head,
1091 s.len(),
1092 &s[tail..]
1093 )
1094}
1095
1096/// Whether a tool's result is command output, whose end matters most: the
1097/// tools every CAR agent loop truncates with [`truncate_keeping_ends`].
1098pub fn is_command_tool(name: &str) -> bool {
1099 matches!(name, "shell" | "run_command")
1100}
1101
1102/// [`format_tool_result`] as a model should read it: a successful result's
1103/// output goes through [`render_tool_output`] instead of being serialized as
1104/// JSON; failures keep their `[FAILED]` / `[REJECTED]` prefix.
1105pub fn format_tool_result_for_model(result: &ActionResult) -> String {
1106 match (&result.status, &result.output) {
1107 (ActionStatus::Succeeded, Some(v)) => render_tool_output(v),
1108 _ => format_tool_result(result),
1109 }
1110}
1111
1112/// Budget constraints for proposal execution.
1113#[derive(Debug, Clone)]
1114pub struct CostBudget {
1115 pub max_tool_calls: Option<u32>,
1116 pub max_duration_ms: Option<f64>,
1117 pub max_actions: Option<u32>,
1118}
1119
1120/// Common Agent Runtime — deterministic execution layer.
1121///
1122/// Lock ordering discipline (never hold multiple simultaneously, never hold sync locks across .await):
1123/// 1. capabilities (RwLock, read-only during execution)
1124/// 2. tools (RwLock, read-only during execution)
1125/// 3. policies (RwLock, read-only during execution)
1126/// 4. session_policies (RwLock to find the per-session engine, then read inner Arc)
1127/// 5. cost_budget (RwLock, read-only during execution)
1128/// 6. log (TokioMutex, acquired/released per event)
1129/// 7. tool_executor (TokioMutex, clone Arc and drop before await)
1130/// 8. idempotency_cache (TokioMutex, acquired/released per check)
1131///
1132/// StateStore uses parking_lot::Mutex (sync) — NEVER hold across .await points.
1133pub struct Runtime {
1134 pub state: Arc<StateStore>,
1135 pub tools: Arc<TokioRwLock<HashMap<String, ToolSchema>>>,
1136 pub policies: Arc<TokioRwLock<PolicyEngine>>,
1137 /// Per-session policy registries. Hosts that multiplex multiple
1138 /// concurrent agent sessions over a single `Runtime` (IDE-style
1139 /// frontends with per-project rules, multi-tenant servers) call
1140 /// [`Runtime::open_session`] to mint an id, then
1141 /// [`Runtime::register_policy_in_session`] to attach session-scoped
1142 /// rules. Validation under that session walks the global registry
1143 /// AND the session's — both must pass. Sessions can deny what
1144 /// global allows; sessions cannot allow what global denies.
1145 /// Closing a session drops its registry and any closures it holds.
1146 /// See `docs/proposals/per-session-policy-scoping.md`.
1147 pub session_policies: Arc<TokioRwLock<HashMap<String, Arc<TokioRwLock<PolicyEngine>>>>>,
1148 pub log: Arc<TokioMutex<EventLog>>,
1149 pub rate_limiter: Arc<RateLimiter>,
1150 pub result_cache: Arc<ResultCache>,
1151 /// Per-execution-session read ledgers backing the read-before-edit / staleness guard on
1152 /// the built-in file tools (H1/F4-remainder, audit 2026-07-06). Threaded into
1153 /// `agent_basics::execute_with_ledger` on the built-in fallback path so a raw
1154 /// Runtime session (no configured executor) still requires reading a file
1155 /// before editing or overwriting it. Configured executors
1156 /// (coder/assistant/bench) own their own ledger, so this one only governs the
1157 /// built-in fallback path.
1158 read_ledgers: crate::agent_basics::SessionReadLedgers,
1159 tool_executor: TokioMutex<Option<Arc<dyn ToolExecutor>>>,
1160 idempotency_cache: TokioMutex<HashMap<String, ActionResult>>,
1161 cost_budget: TokioRwLock<Option<CostBudget>>,
1162 capabilities: TokioRwLock<Option<CapabilitySet>>,
1163 inference_engine: Option<Arc<car_inference::InferenceEngine>>,
1164 /// Optional outbound transport backing the `messaging.send` built-in.
1165 /// `None` by default — a runtime with no sink refuses `messaging.send`
1166 /// outright rather than falling through to the host executor, and never
1167 /// advertises the tool. Attach one with
1168 /// [`Runtime::with_message_sink`]; see [`crate::messaging`] for why
1169 /// human-directed messaging belongs inside the governed chain at all.
1170 message_sink: Option<Arc<dyn crate::messaging::MessageSink>>,
1171 /// Optional memgine for skill learning (auto-distillation after execution).
1172 memgine: Option<Arc<TokioMutex<car_memgine::MemgineEngine>>>,
1173 /// Whether to auto-distill skills after each proposal execution.
1174 auto_distill: bool,
1175 /// Optional trajectory store for persisting execution traces.
1176 trajectory_store: Option<Arc<car_memgine::TrajectoryStore>>,
1177 /// Optional replan callback for failure recovery.
1178 replan_callback: TokioMutex<Option<Arc<dyn ReplanCallback>>>,
1179 /// Replan configuration.
1180 replan_config: TokioRwLock<ReplanConfig>,
1181 /// Pre-execution transactional conflict check mode (survey §4.3/§5.2.4).
1182 /// `Off` by default. When `Warn`/`Strict`, each proposal is checked
1183 /// against the versioned shared state before execution; `Strict`
1184 /// rejects on conflict. See [`Runtime::set_transaction_check_mode`].
1185 transaction_check: TokioRwLock<TransactionCheckMode>,
1186 /// The live harness operating config the Evolution Agent tunes (survey
1187 /// §3.5). `None` until one is installed via [`Runtime::set_harness_config`]
1188 /// — so default behavior is byte-identical to before (no cap on
1189 /// per-action retries, built-in backoff). Once installed, the runtime
1190 /// *reads* it: `max_retries` caps per-action retry budgets and
1191 /// `retry_backoff_ms` sets the inter-attempt delay; the setter also maps
1192 /// `planning_max_replans` onto the replan config. This is what makes an
1193 /// applied [`car_memgine::HarnessConfigPatch`] take effect.
1194 harness_config: TokioRwLock<Option<car_memgine::HarnessConfig>>,
1195 /// Canonical tool registry (optional — new code should use this).
1196 pub registry: Arc<crate::registry::ToolRegistry>,
1197 /// The environment the agent's side-effecting built-in tools act within.
1198 /// Defaults to [`crate::substrate::LocalSubstrate`] (host fs/process), which
1199 /// reproduces the historic `agent_basics` host behavior byte-for-byte.
1200 /// Bind a different environment (e.g. a VM via `McpSubstrate`) with
1201 /// [`Runtime::with_substrate`] / [`Runtime::set_substrate`]. `calculate`
1202 /// stays pure and never consults the substrate.
1203 substrate: TokioRwLock<Arc<dyn crate::substrate::Substrate>>,
1204 /// Refusal applied to the host paths the built-in file tools touch while
1205 /// the substrate is local. `None` by default, so a runtime that installs
1206 /// none behaves exactly as before. See [`Runtime::set_local_path_guard`].
1207 local_path_guard: TokioRwLock<Option<crate::agent_basics::LocalPathGuard>>,
1208 /// Proposal-admission gates — the pre-execution safety seam (EPIC A /
1209 /// task A1). Each registered [`crate::admission::AdmissionGate`] runs
1210 /// during admission, before any action executes; a proposal that any
1211 /// gate blocks (or escalates to approval) is refused. Empty by default,
1212 /// so a runtime that registers no gates behaves exactly as before.
1213 /// Individual gates (information-flow, concurrency, blocking-policy)
1214 /// are layered on via [`Runtime::register_admission_gate`].
1215 admission_gates: TokioRwLock<Vec<Arc<dyn crate::admission::AdmissionGate>>>,
1216 /// Runtime taint provenance for the VIGIL intent gate
1217 /// ([`crate::taint::TaintLedger`]). Installed by
1218 /// [`Runtime::install_intent_gate`] and by nothing else: with no intent
1219 /// gate there is no ledger, so a runtime that never configures VIGIL
1220 /// pays no per-action cost and behaves exactly as before. When present,
1221 /// each successful action records which state keys its result wrote and
1222 /// whether that result was tainted, and the gate reads it back at
1223 /// admission to mark actions that READ a tainted key — the cross-
1224 /// proposal (replan) and trusted-tool-laundering cases the proposal-
1225 /// local dependency DAG cannot see.
1226 taint_ledger: TokioRwLock<Option<Arc<crate::taint::TaintLedger>>>,
1227 /// Durable human-in-the-loop approval ledger (EPIC A / A7). When set,
1228 /// an admission gate's `NeedsApproval` verdict is resolved against this
1229 /// ledger by fingerprint: a prior Approved decision admits the
1230 /// proposal, a Rejected decision blocks it, an unseen one stays pending
1231 /// (fail-closed). `None` by default — escalations fail closed with an
1232 /// explanatory reason until a ledger is installed.
1233 approval_ledger: TokioRwLock<Option<ApprovalLedger>>,
1234 /// Optional JSONL journal backing the idempotency cache (EPIC A / C3).
1235 /// `None` by default — the cache is in-memory and lost on restart. When
1236 /// set via [`Runtime::set_idempotency_cache_path`], idempotent results
1237 /// are persisted and reloaded so a crash-restart doesn't re-execute a
1238 /// completed idempotent action (avoiding duplicate side effects).
1239 idempotency_journal: TokioRwLock<Option<std::path::PathBuf>>,
1240 /// Detached tool invocations (C2): a `ToolCall` with a
1241 /// `streaming`/`long_running` invocation mode is registered here and
1242 /// its handle returned as the action's output while the DAG proceeds.
1243 /// Chunks are drained via [`Runtime::tool_poll`], cancelled via
1244 /// [`Runtime::tool_cancel`], and fanned out to
1245 /// [`Runtime::subscribe_tool_events`] subscribers.
1246 pub tool_handles: Arc<crate::tool_handles::ToolHandleRegistry>,
1247}
1248
1249/// A durable idempotency-cache record (C3). `result: None` is a tombstone
1250/// recorded when a cached entry is invalidated by a rollback.
1251#[derive(serde::Serialize, serde::Deserialize)]
1252struct IdempotencyEntry {
1253 key: String,
1254 result: Option<ActionResult>,
1255}
1256
1257impl Runtime {
1258 pub fn new() -> Self {
1259 Self {
1260 state: Arc::new(StateStore::new()),
1261 tools: Arc::new(TokioRwLock::new(HashMap::new())),
1262 policies: Arc::new(TokioRwLock::new(PolicyEngine::new())),
1263 session_policies: Arc::new(TokioRwLock::new(HashMap::new())),
1264 log: Arc::new(TokioMutex::new(EventLog::new())),
1265 rate_limiter: Arc::new(RateLimiter::new()),
1266 result_cache: Arc::new(ResultCache::new()),
1267 read_ledgers: crate::agent_basics::SessionReadLedgers::new(),
1268 tool_executor: TokioMutex::new(None),
1269 idempotency_cache: TokioMutex::new(HashMap::new()),
1270 cost_budget: TokioRwLock::new(None),
1271 capabilities: TokioRwLock::new(None),
1272 inference_engine: None,
1273 message_sink: None,
1274 memgine: None,
1275 auto_distill: false,
1276 trajectory_store: None,
1277 replan_callback: TokioMutex::new(None),
1278 replan_config: TokioRwLock::new(ReplanConfig::default()),
1279 transaction_check: TokioRwLock::new(TransactionCheckMode::Off),
1280 harness_config: TokioRwLock::new(None),
1281 registry: Arc::new(crate::registry::ToolRegistry::new()),
1282 substrate: TokioRwLock::new(Arc::new(crate::substrate::LocalSubstrate::new())),
1283 local_path_guard: TokioRwLock::new(None),
1284 admission_gates: TokioRwLock::new(Vec::new()),
1285 taint_ledger: TokioRwLock::new(None),
1286 approval_ledger: TokioRwLock::new(None),
1287 idempotency_journal: TokioRwLock::new(None),
1288 tool_handles: Arc::new(crate::tool_handles::ToolHandleRegistry::new()),
1289 }
1290 }
1291
1292 /// Create a runtime with shared state, event log, and policies.
1293 /// Each runtime gets its own tool set, executor, and idempotency cache.
1294 pub fn with_shared(
1295 state: Arc<StateStore>,
1296 log: Arc<TokioMutex<EventLog>>,
1297 policies: Arc<TokioRwLock<PolicyEngine>>,
1298 ) -> Self {
1299 Self {
1300 state,
1301 tools: Arc::new(TokioRwLock::new(HashMap::new())),
1302 policies,
1303 // Session-policy registries are per-runtime — sharing them
1304 // across embedders that share global policies would defeat
1305 // the isolation point. Hosts that genuinely want shared
1306 // sessions should drive them through one shared Runtime.
1307 session_policies: Arc::new(TokioRwLock::new(HashMap::new())),
1308 log,
1309 rate_limiter: Arc::new(RateLimiter::new()),
1310 result_cache: Arc::new(ResultCache::new()),
1311 read_ledgers: crate::agent_basics::SessionReadLedgers::new(),
1312 tool_executor: TokioMutex::new(None),
1313 idempotency_cache: TokioMutex::new(HashMap::new()),
1314 cost_budget: TokioRwLock::new(None),
1315 capabilities: TokioRwLock::new(None),
1316 inference_engine: None,
1317 message_sink: None,
1318 memgine: None,
1319 auto_distill: false,
1320 trajectory_store: None,
1321 replan_callback: TokioMutex::new(None),
1322 replan_config: TokioRwLock::new(ReplanConfig::default()),
1323 transaction_check: TokioRwLock::new(TransactionCheckMode::Off),
1324 harness_config: TokioRwLock::new(None),
1325 registry: Arc::new(crate::registry::ToolRegistry::new()),
1326 substrate: TokioRwLock::new(Arc::new(crate::substrate::LocalSubstrate::new())),
1327 local_path_guard: TokioRwLock::new(None),
1328 admission_gates: TokioRwLock::new(Vec::new()),
1329 taint_ledger: TokioRwLock::new(None),
1330 approval_ledger: TokioRwLock::new(None),
1331 idempotency_journal: TokioRwLock::new(None),
1332 tool_handles: Arc::new(crate::tool_handles::ToolHandleRegistry::new()),
1333 }
1334 }
1335
1336 // ─── Session policy lifecycle ───────────────────────────────────
1337 //
1338 // Per-session policy scoping. Policies registered against a
1339 // session apply to proposals executed under that session id (via
1340 // [`Self::execute_with_session`] / [`Self::execute_with_session_and_cancel`]).
1341 // Policies registered globally always apply, on top of any
1342 // session-scoped layer. See `docs/proposals/per-session-policy-scoping.md`.
1343
1344 /// Mint a new session id and pre-register an empty policy engine
1345 /// under it. Hosts call this once per concurrent agent context
1346 /// (an IDE project window, a multi-tenant client, etc.) and pair
1347 /// it with [`Self::close_session`] when the context ends.
1348 ///
1349 /// Returns the opaque id to pass to subsequent
1350 /// [`Self::register_policy_in_session`] / [`Self::execute_with_session`]
1351 /// calls. Ids are UUIDs so collisions across concurrent calls
1352 /// don't matter.
1353 pub async fn open_session(&self) -> String {
1354 let id = Uuid::new_v4().to_string();
1355 let mut sessions = self.session_policies.write().await;
1356 sessions.insert(id.clone(), Arc::new(TokioRwLock::new(PolicyEngine::new())));
1357 self.read_ledgers.ledger_for(Some(&id));
1358 id
1359 }
1360
1361 /// Drop the session and every policy scoped to it. Returns true
1362 /// if a session by that id existed; false if it didn't (already
1363 /// closed, never opened, etc.). Idempotent in effect — closing a
1364 /// missing session is a no-op the caller is free to ignore.
1365 pub async fn close_session(&self, session_id: &str) -> bool {
1366 let mut sessions = self.session_policies.write().await;
1367 let removed = sessions.remove(session_id).is_some();
1368 if removed {
1369 self.read_ledgers.remove(session_id);
1370 }
1371 removed
1372 }
1373
1374 /// Register a policy under a specific session id. The policy
1375 /// applies only when a proposal is executed under that session;
1376 /// proposals executed without a session (the default) only see
1377 /// global policies.
1378 ///
1379 /// Returns `Err(...)` if the session is unknown — callers either
1380 /// forgot to call [`Self::open_session`] or are using a
1381 /// stale/closed id.
1382 pub async fn register_policy_in_session(
1383 &self,
1384 session_id: &str,
1385 name: &str,
1386 check: car_policy::PolicyCheck,
1387 description: &str,
1388 ) -> Result<(), String> {
1389 let engine = {
1390 let sessions = self.session_policies.read().await;
1391 sessions
1392 .get(session_id)
1393 .cloned()
1394 .ok_or_else(|| format!("unknown session id '{session_id}'"))?
1395 };
1396 let mut engine = engine.write().await;
1397 engine.register(name, check, description);
1398 Ok(())
1399 }
1400
1401 /// [`Self::register_policy_in_session`] for a check that forbids `tool`
1402 /// outright, so `PolicyEngine::blanket_denied_tools` can read it back.
1403 /// Separate method rather than an extra parameter: the existing signature
1404 /// is public and crosses the bindings, and this is purely additive.
1405 pub async fn register_tool_deny_in_session(
1406 &self,
1407 session_id: &str,
1408 name: &str,
1409 tool: &str,
1410 check: car_policy::PolicyCheck,
1411 description: &str,
1412 ) -> Result<(), String> {
1413 let engine = {
1414 let sessions = self.session_policies.read().await;
1415 sessions
1416 .get(session_id)
1417 .cloned()
1418 .ok_or_else(|| format!("unknown session id '{session_id}'"))?
1419 };
1420 let mut engine = engine.write().await;
1421 engine.register_tool_deny(name, tool, check, description);
1422 Ok(())
1423 }
1424
1425 /// Remove a policy by name. `session_id` targets a session's policy set;
1426 /// `None` targets the global one.
1427 ///
1428 /// Returns how many policies were dropped, or `Err` if `session_id` names a
1429 /// session that doesn't exist. Session-scoped policies could always be
1430 /// dropped wholesale by [`Self::close_session`], but a *global* policy had
1431 /// no removal path at all — once registered it lived until the process
1432 /// exited, so a mistyped or over-broad rule could only be cleared by
1433 /// restarting the daemon (Parslee-ai/car#623).
1434 pub async fn unregister_policy(
1435 &self,
1436 name: &str,
1437 session_id: Option<&str>,
1438 ) -> Result<usize, String> {
1439 match session_id {
1440 Some(sid) => {
1441 let engine = {
1442 let sessions = self.session_policies.read().await;
1443 sessions
1444 .get(sid)
1445 .cloned()
1446 .ok_or_else(|| format!("unknown session id '{sid}'"))?
1447 };
1448 let mut engine = engine.write().await;
1449 Ok(engine.unregister(name))
1450 }
1451 None => {
1452 let mut engine = self.policies.write().await;
1453 Ok(engine.unregister(name))
1454 }
1455 }
1456 }
1457
1458 /// Registered policies as `(name, description)`. `session_id` lists a
1459 /// session's set; `None` lists the global one. Without this a client could
1460 /// register a policy but never ask what was in force, so a rejection could
1461 /// not be explained beyond its single message (Parslee-ai/car#623).
1462 pub async fn list_policies(
1463 &self,
1464 session_id: Option<&str>,
1465 ) -> Result<Vec<(String, String)>, String> {
1466 match session_id {
1467 Some(sid) => {
1468 let engine = {
1469 let sessions = self.session_policies.read().await;
1470 sessions
1471 .get(sid)
1472 .cloned()
1473 .ok_or_else(|| format!("unknown session id '{sid}'"))?
1474 };
1475 let engine = engine.read().await;
1476 Ok(engine.policy_details())
1477 }
1478 None => {
1479 let engine = self.policies.read().await;
1480 Ok(engine.policy_details())
1481 }
1482 }
1483 }
1484
1485 /// Load declarative deny rules from a project's `.car/policies/`
1486 /// directory and register them on the global policy engine (EPIC A /
1487 /// task A2).
1488 ///
1489 /// `car_dir` is a `.car` directory; this looks for
1490 /// `car_dir/policies/*.toml`. **The caller chooses the directory and
1491 /// there is no walk-up here** — this joins the path it is given, once.
1492 /// The two production callers pick differently: the daemon passes
1493 /// `$HOME/.car`, and the assistant passes its working directory's
1494 /// `.car` (`--dir`, else cwd). So a rule file at a repository root does
1495 /// not govern a `car do` run started from a subdirectory. Say which
1496 /// directory you mean at the call site rather than assuming discovery.
1497 ///
1498 /// A missing directory is not an error (returns 0). A malformed rule
1499 /// file *is* an error — a dropped security rule must surface loudly.
1500 /// Returns the number of rules registered.
1501 ///
1502 /// These rules are additive on top of any code-registered policies, and
1503 /// every one of them is a prohibition — though `allow_tool_param` states
1504 /// its prohibition as an allowlist, denying everything about its tool
1505 /// that it does not name. Once A9 makes policy violations blocking at
1506 /// admission, a matching action refuses the proposal.
1507 pub async fn load_project_policies(
1508 &self,
1509 car_dir: impl AsRef<std::path::Path>,
1510 ) -> Result<usize, car_policy::PolicyLoadError> {
1511 let dir = car_dir.as_ref().join("policies");
1512 let rules = car_policy::load_policy_dir(&dir)?;
1513 // `PolicyRules::len` counts every kind. Summing a hand-picked subset
1514 // here is what made this under-report once already — it covered the
1515 // three kinds that existed at the time and was not revisited when
1516 // more landed, so a file full of allowlists reported zero rules
1517 // loaded while `apply` registered all of them.
1518 let count = rules.len();
1519 let mut engine = self.policies.write().await;
1520 rules.apply(&mut engine);
1521 Ok(count)
1522 }
1523
1524 /// Load information-flow tool labels from a project's `.car` directory
1525 /// and register the information-flow admission gate (EPIC A / A3+A4).
1526 ///
1527 /// Reads `car_dir/tool-labels.json` (merged over built-in defaults) and
1528 /// registers an [`crate::flow::InformationFlowGate`] so every admitted
1529 /// proposal is checked for data exfiltration (blocked) and forbidden
1530 /// tool orderings (escalated to approval). A missing labels file is
1531 /// fine — the built-in defaults still mark the network tools as sinks.
1532 /// A malformed file is a loud error.
1533 pub async fn install_information_flow_gate(
1534 &self,
1535 car_dir: impl AsRef<std::path::Path>,
1536 ) -> Result<(), crate::flow::FlowLoadError> {
1537 let config = crate::flow::load_tool_labels(car_dir)?;
1538 let gate = Arc::new(crate::flow::InformationFlowGate::new(config));
1539 self.register_admission_gate(gate).await;
1540 Ok(())
1541 }
1542
1543 /// True if a session with this id is currently open. Mostly for
1544 /// tests and FFI surface validation — production code should
1545 /// trust the id it just opened.
1546 pub async fn session_exists(&self, session_id: &str) -> bool {
1547 self.session_policies.read().await.contains_key(session_id)
1548 }
1549
1550 /// Attach a local inference engine. Registers `infer`, `embed`, `classify`
1551 /// as built-in tools with real implementations.
1552 pub fn with_inference(mut self, engine: Arc<car_inference::InferenceEngine>) -> Self {
1553 self.inference_engine = Some(engine);
1554 // Register inference tool schemas (non-async init, use try_lock).
1555 // Keep the canonical registry populated too: execution-event provenance
1556 // is read from ToolEntry.source rather than inferred from a tool name.
1557 //
1558 // Contention contract: this runs at construction time on a private
1559 // runtime, so both try_locks are expected to succeed. If either fails
1560 // the two stores would disagree (validator reads `tools`, provenance
1561 // reads the registry first) — loud in debug, logged in release.
1562 if let Ok(mut tools) = self.tools.try_write() {
1563 for schema in car_inference::service::all_schemas() {
1564 let name = schema.name.clone();
1565 let entry = crate::registry::ToolEntry::builtin(schema);
1566 tools.insert(name.clone(), entry.schema.clone());
1567 if !self.registry.try_register(entry) {
1568 tracing::warn!(
1569 tool = %name,
1570 "with_inference: canonical registry contention; \
1571 schema map registered the inference tool but the \
1572 registry did not — provenance falls back to the map"
1573 );
1574 }
1575 }
1576 } else {
1577 debug_assert!(
1578 false,
1579 "with_inference must populate both stores; schema-map lock was contended \
1580 at construction time — inference tools stay unregistered"
1581 );
1582 tracing::warn!(
1583 "with_inference: schema map lock contended at construction; \
1584 inference tools were NOT registered"
1585 );
1586 }
1587 self
1588 }
1589
1590 /// Attach an outbound message sink, making `messaging.send` a real tool on
1591 /// this runtime.
1592 ///
1593 /// Registering the schema here — and ONLY here — is deliberate. A tool the
1594 /// runtime cannot execute must not be advertised to the model: without a
1595 /// sink the dispatch arm can only return an error, and a model that keeps
1596 /// seeing "message the human" in its tool list will keep trying to use it.
1597 /// So sink and schema arrive together, and neither exists alone.
1598 ///
1599 /// The entry is `AskUser` with side effects: reaching a human is
1600 /// irreversible, so the conservative default is the right one, and hosts
1601 /// that have their own consent model can re-register with a different
1602 /// permission. `category = "messaging"` groups it for the capability
1603 /// surfaces that filter by category.
1604 ///
1605 /// Follows [`Self::with_inference`]'s non-async registration pattern
1606 /// (`try_write` during construction, while the locks are uncontended) —
1607 /// making the builders async would break every `Runtime::new().with_…()`
1608 /// chain in the workspace for no gain.
1609 pub fn with_message_sink(mut self, sink: Arc<dyn crate::messaging::MessageSink>) -> Self {
1610 self.message_sink = Some(sink);
1611
1612 let entry = crate::registry::ToolEntry::builtin(car_ir::builtins::messaging_send())
1613 .with_permission(crate::registry::ToolPermission::AskUser)
1614 .with_side_effects(true)
1615 .with_category("messaging");
1616 let schema = entry.schema.clone();
1617
1618 // Honour the schema's declared rate limit, the way the async
1619 // `register_tool_schema` path does — otherwise the backstop on
1620 // outbound human messaging would silently not exist.
1621 if let Some(ref rl) = schema.rate_limit {
1622 self.rate_limiter.try_set_limit(
1623 &schema.name,
1624 RateLimit {
1625 max_calls: rl.max_calls,
1626 interval_secs: rl.interval_secs,
1627 },
1628 );
1629 }
1630 self.registry.try_register(entry);
1631 // The legacy schema map is what `validate_action` reads, so the
1632 // validator can only check `messaging.send` parameters once it lands
1633 // here.
1634 if let Ok(mut tools) = self.tools.try_write() {
1635 tools.insert(schema.name.clone(), schema);
1636 }
1637 self
1638 }
1639
1640 /// Attach a memgine for automatic skill learning after execution.
1641 /// When `auto_distill` is true, execution traces are automatically distilled
1642 /// into skills and domains are evolved when underperforming.
1643 pub fn with_learning(
1644 mut self,
1645 memgine: Arc<TokioMutex<car_memgine::MemgineEngine>>,
1646 auto_distill: bool,
1647 ) -> Self {
1648 self.memgine = Some(memgine);
1649 self.auto_distill = auto_distill;
1650 self
1651 }
1652
1653 /// Attach a memgine with auto-distillation enabled (recommended default).
1654 pub fn with_memgine(self, memgine: Arc<TokioMutex<car_memgine::MemgineEngine>>) -> Self {
1655 self.with_learning(memgine, true)
1656 }
1657
1658 /// Attach a trajectory store for persisting execution traces.
1659 pub fn with_trajectory_store(mut self, store: Arc<car_memgine::TrajectoryStore>) -> Self {
1660 self.trajectory_store = Some(store);
1661 self
1662 }
1663
1664 /// The attached trajectory store, if any.
1665 ///
1666 /// Writing traces was always the point of the store; reading them back is
1667 /// what makes them a feedback signal rather than an audit log.
1668 pub fn trajectory_store(&self) -> Option<&Arc<car_memgine::TrajectoryStore>> {
1669 self.trajectory_store.as_ref()
1670 }
1671
1672 /// Per-tool success rates observed over the last `window_days`, or `None`
1673 /// when no trajectory store is attached.
1674 ///
1675 /// Dispatch-conditional (see
1676 /// [`ToolFeedback::dispatched_from_trajectories`](car_planner::ToolFeedback::dispatched_from_trajectories))
1677 /// — rejected and skipped actions are excluded, because the tool never ran
1678 /// and a consumer that models rejection separately would otherwise count
1679 /// it twice.
1680 ///
1681 /// The window exists because a success rate is a claim about how a tool
1682 /// behaves *now*. An API that was broken for a week six months ago and has
1683 /// been fixed since should not still be dragging its own rate down, and an
1684 /// unbounded history makes the rate progressively less responsive to
1685 /// exactly the recent change an operator is trying to see. 30 days is the
1686 /// suggested default at the call sites.
1687 ///
1688 /// Reads from disk on every call — parsing is bounded by the window (day
1689 /// files outside it are skipped by filename, never opened), and the callers
1690 /// are interactive rather than hot-path. If that stops being true, cache
1691 /// here rather than at each call site.
1692 pub fn tool_feedback(&self, window_days: u32) -> Option<car_planner::ToolFeedback> {
1693 let store = self.trajectory_store.as_ref()?;
1694 let cutoff = chrono::Utc::now() - chrono::Duration::days(i64::from(window_days));
1695 let trajectories = store.load_since(cutoff);
1696 Some(car_planner::ToolFeedback::dispatched_from_trajectories(
1697 &trajectories,
1698 ))
1699 }
1700
1701 pub fn with_executor(self, executor: Arc<dyn ToolExecutor>) -> Self {
1702 // Use try_lock for non-async init context. Safe because we just created the mutex.
1703 if let Ok(mut guard) = self.tool_executor.try_lock() {
1704 *guard = Some(executor);
1705 }
1706 self
1707 }
1708
1709 /// Set a tool executor for the next execute() call.
1710 /// Used by NAPI bindings where executor varies per call.
1711 pub async fn set_executor(&self, executor: Arc<dyn ToolExecutor>) {
1712 *self.tool_executor.lock().await = Some(executor);
1713 }
1714
1715 /// Bind the execution substrate the side-effecting built-in tools
1716 /// (`read_file`/`write_file`/`edit_file`/`list_dir`/`find_files`/
1717 /// `grep_files`) act within (builder). Defaults to
1718 /// [`crate::substrate::LocalSubstrate`]; bind e.g.
1719 /// [`crate::substrate::McpSubstrate`] to make those tools hit a VM.
1720 /// `calculate` stays pure and ignores the substrate.
1721 pub fn with_substrate(self, substrate: Arc<dyn crate::substrate::Substrate>) -> Self {
1722 // Use try_write for non-async init context. Safe because we just created the lock.
1723 if let Ok(mut guard) = self.substrate.try_write() {
1724 *guard = substrate;
1725 }
1726 self
1727 }
1728
1729 /// Set the execution substrate at runtime.
1730 pub async fn set_substrate(&self, substrate: Arc<dyn crate::substrate::Substrate>) {
1731 {
1732 *self.substrate.write().await = substrate;
1733 }
1734 self.read_ledgers.clear();
1735 }
1736
1737 /// Clone the currently bound substrate.
1738 pub async fn substrate(&self) -> Arc<dyn crate::substrate::Substrate> {
1739 self.substrate.read().await.clone()
1740 }
1741
1742 pub fn with_event_log(mut self, log: EventLog) -> Self {
1743 self.log = Arc::new(TokioMutex::new(log));
1744 self
1745 }
1746
1747 /// Bind an event log the caller already holds a handle to.
1748 ///
1749 /// [`Self::with_event_log`] takes ownership, which leaves no way for
1750 /// anything outside the runtime to read what was recorded. That was fine
1751 /// while the log was observability-only, but a model-callable events
1752 /// surface has to run inside a `ToolExecutor` — and the executor is
1753 /// constructed *before* the runtime that would own it, so the two can only
1754 /// meet through a handle created first and shared into both
1755 /// (Parslee-ai/car#815).
1756 pub fn with_shared_event_log(mut self, log: Arc<TokioMutex<EventLog>>) -> Self {
1757 self.log = log;
1758 self
1759 }
1760
1761 /// The runtime's event log handle, for callers that need to read the record
1762 /// the runtime is writing.
1763 pub fn event_log_handle(&self) -> Arc<TokioMutex<EventLog>> {
1764 Arc::clone(&self.log)
1765 }
1766
1767 /// Attach a replan callback for failure recovery (builder).
1768 pub fn with_replan(self, callback: Arc<dyn ReplanCallback>, config: ReplanConfig) -> Self {
1769 if let Ok(mut guard) = self.replan_callback.try_lock() {
1770 *guard = Some(callback);
1771 }
1772 if let Ok(mut guard) = self.replan_config.try_write() {
1773 *guard = config;
1774 }
1775 self
1776 }
1777
1778 /// Set a replan callback at runtime.
1779 pub async fn set_replan_callback(&self, callback: Arc<dyn ReplanCallback>) {
1780 *self.replan_callback.lock().await = Some(callback);
1781 }
1782
1783 /// Set replan configuration at runtime.
1784 pub async fn set_replan_config(&self, config: ReplanConfig) {
1785 *self.replan_config.write().await = config;
1786 }
1787
1788 /// Set the pre-execution transactional conflict-check mode (survey
1789 /// §4.3/§5.2.4). `Off` (default) preserves prior behavior; `Warn`
1790 /// records conflicts as telemetry; `Strict` rejects a conflicting
1791 /// proposal before executing it.
1792 pub async fn set_transaction_check_mode(&self, mode: TransactionCheckMode) {
1793 *self.transaction_check.write().await = mode;
1794 }
1795
1796 /// Register a proposal-admission gate (EPIC A / task A1).
1797 ///
1798 /// Gates run during admission, before any action executes, in the
1799 /// order they were registered. Each is a verified pre-execution safety
1800 /// check — information-flow (A4), concurrency (A5), blocking-policy
1801 /// (A9) — that can block a proposal or escalate it to human approval.
1802 /// Registering no gates leaves behavior unchanged.
1803 pub async fn register_admission_gate(&self, gate: Arc<dyn crate::admission::AdmissionGate>) {
1804 self.admission_gates.write().await.push(gate);
1805 }
1806
1807 /// Register `gate`, **replacing** any already-registered gate answering
1808 /// to the same [`crate::admission::AdmissionGate::name`] instead of
1809 /// appending a second one. For gates that are *reconfigured* rather than
1810 /// layered — the VIGIL intent gate is reloaded whenever an operator
1811 /// re-reads `.car/intent.json` — appending is a bug, not a no-op: gate
1812 /// aggregation is fail-closed, so the stale gate's verdict still wins
1813 /// while it holds state (a [`crate::taint::TaintLedger`]) nothing writes
1814 /// to any more, and two gates with the same name emit two
1815 /// `AdmissionGateDecision` events under that name.
1816 ///
1817 /// The replaced gate keeps its **position**, so a reconfigure never
1818 /// silently reorders admission relative to the other gates. Any further
1819 /// gate carrying the same name is a stale duplicate from an earlier
1820 /// registration and is dropped, so exactly one gate answers to the name.
1821 async fn replace_admission_gate(&self, gate: Arc<dyn crate::admission::AdmissionGate>) {
1822 let name = gate.name().to_string();
1823 let mut gates = self.admission_gates.write().await;
1824 match gates.iter().position(|g| g.name() == name) {
1825 Some(i) => {
1826 gates[i] = gate;
1827 let mut idx = 0usize;
1828 gates.retain(|g| {
1829 let keep = idx == i || g.name() != name;
1830 idx += 1;
1831 keep
1832 });
1833 }
1834 None => gates.push(gate),
1835 }
1836 }
1837
1838 /// Remove all registered admission gates (primarily for tests and
1839 /// reconfiguration). After this, proposal admission reverts to the
1840 /// transactional pre-check only.
1841 pub async fn clear_admission_gates(&self) {
1842 self.admission_gates.write().await.clear();
1843 }
1844
1845 /// The number of currently-registered admission gates.
1846 pub async fn admission_gate_count(&self) -> usize {
1847 self.admission_gates.read().await.len()
1848 }
1849
1850 /// The `name()` of every registered admission gate, in registration order.
1851 ///
1852 /// Prefer this over [`Self::admission_gate_count`] when asserting that a
1853 /// *particular* gate is installed: a bare count couples the assertion to
1854 /// every other gate the runtime happens to register, so adding one breaks
1855 /// unrelated tests that only cared about their own.
1856 pub async fn admission_gate_names(&self) -> Vec<String> {
1857 self.admission_gates
1858 .read()
1859 .await
1860 .iter()
1861 .map(|g| g.name().to_string())
1862 .collect()
1863 }
1864
1865 /// Register the VIGIL intent gate (arXiv 2601.05755 — the live
1866 /// verify-before-commit call-site). Installs
1867 /// [`crate::intent_gate::IntentGate`] as an admission gate: a
1868 /// forbidden capability or a tool-stream-influenced out-of-intent
1869 /// action hard-rejects the proposal; untainted drift escalates to
1870 /// the durable approval flow (A7) by content-bound fingerprint.
1871 /// Replans are covered automatically (gates re-run on every
1872 /// replanned proposal).
1873 ///
1874 /// This is also where the runtime's [`crate::taint::TaintLedger`] is
1875 /// installed. From here on, every successful action records whether its
1876 /// result was tainted and which state keys it wrote, and the gate marks
1877 /// any incoming action that READS a tainted key as tool-stream-
1878 /// influenced. That closes the two holes the static `untrusted_tools`
1879 /// list leaves open: a trusted tool laundering attacker-controlled
1880 /// content out of state, and a replanned proposal that carries no
1881 /// dependency edge back to the poisoned action. Installing no intent
1882 /// gate installs no ledger, so nothing changes for a runtime that
1883 /// doesn't configure VIGIL.
1884 ///
1885 /// **Calling this again REPLACES the installed gate; it does not add a
1886 /// second one.** Reinstalling is a real operation — an operator editing
1887 /// `.car/intent.json` and re-running
1888 /// [`Runtime::install_intent_gate_from_project`] lands here — and each
1889 /// call builds a *fresh* ledger that becomes the only one the executor
1890 /// writes to. Appending would leave the previous gate registered while
1891 /// holding an orphaned ledger, and because gate aggregation is
1892 /// fail-closed that frozen gate's verdict would still win: every key
1893 /// tainted before the reload would stay tainted forever, no trusted
1894 /// overwrite could clear it, and legitimate out-of-intent work would
1895 /// hard-refuse with no approval path. So the runtime holds exactly one
1896 /// intent gate, and it is always the one bound to the live ledger. The
1897 /// replacement keeps the previous gate's position in the admission
1898 /// order, so a reload never silently reorders the other gates.
1899 pub async fn install_intent_gate(&self, config: crate::intent_gate::IntentGateConfig) {
1900 let ledger = Arc::new(crate::taint::TaintLedger::new(
1901 config.untrusted_tools.iter().cloned().collect(),
1902 ));
1903 *self.taint_ledger.write().await = Some(Arc::clone(&ledger));
1904 self.replace_admission_gate(Arc::new(crate::intent_gate::IntentGate::with_taint(
1905 config, ledger,
1906 )))
1907 .await;
1908 }
1909
1910 /// The runtime's taint provenance ledger, when an intent gate is
1911 /// installed (see [`Runtime::install_intent_gate`]). `None` otherwise —
1912 /// the ledger is strictly opt-in, alongside VIGIL.
1913 pub async fn taint_ledger(&self) -> Option<Arc<crate::taint::TaintLedger>> {
1914 self.taint_ledger.read().await.clone()
1915 }
1916
1917 /// Load `.car/intent.json` from `car_dir` and install the intent
1918 /// gate when present. Absent file → Ok(false) (opt-in, ungated);
1919 /// malformed file → loud error, never a silently-ungated session.
1920 pub async fn install_intent_gate_from_project(
1921 &self,
1922 car_dir: impl AsRef<std::path::Path>,
1923 ) -> Result<bool, crate::intent_gate::IntentLoadError> {
1924 match crate::intent_gate::load_intent_config(car_dir)? {
1925 Some(cfg) => {
1926 self.install_intent_gate(cfg).await;
1927 Ok(true)
1928 }
1929 None => Ok(false),
1930 }
1931 }
1932
1933 /// Register the skill deployment-tier ceiling gate (EPIC A / A8).
1934 ///
1935 /// Requires a memgine to be attached (skills live there). When a
1936 /// proposal names its driving skill in `context["skill"]`, the gate
1937 /// caps the proposal's actions at that skill's persisted
1938 /// `deployment_tier`, escalating an over-ceiling action to the durable
1939 /// approval flow (A7). Returns false if no memgine is attached.
1940 pub async fn install_skill_ceiling_gate(&self) -> bool {
1941 match &self.memgine {
1942 Some(mem) => {
1943 let gate = Arc::new(crate::skill_ceiling::SkillCeilingGate::new(mem.clone()));
1944 self.register_admission_gate(gate).await;
1945 true
1946 }
1947 None => false,
1948 }
1949 }
1950
1951 /// Drain buffered chunks + status for a detached tool invocation (C2).
1952 /// `None` for an unknown or fully-consumed handle. See
1953 /// [`crate::tool_handles::ToolHandleRegistry::poll`] for the
1954 /// consume-on-terminal contract.
1955 pub async fn tool_poll(&self, handle_id: &str) -> Option<crate::tool_handles::ToolPollResult> {
1956 self.tool_handles.poll(handle_id).await
1957 }
1958
1959 /// Request cancellation of a detached tool invocation (C2): fires the
1960 /// handle's cancel token (dropping the executor's chunk receiver) and
1961 /// seals its status as `cancelled` unless already terminal. Returns
1962 /// false for an unknown handle.
1963 pub async fn tool_cancel(&self, handle_id: &str) -> bool {
1964 self.tool_handles.cancel(handle_id).await
1965 }
1966
1967 /// Subscribe to the live [`car_ir::ToolStreamEvent`] fanout for all
1968 /// detached tool invocations on this runtime (C2). The WS layer
1969 /// forwards these as `tools.stream.event` notifications.
1970 pub fn subscribe_tool_events(
1971 &self,
1972 ) -> tokio::sync::broadcast::Receiver<car_ir::ToolStreamEvent> {
1973 self.tool_handles.subscribe()
1974 }
1975
1976 /// Enable tamper-evident hash chaining on the event log (EPIC A / A9).
1977 /// Every event appended from now on is linked to its predecessor by a
1978 /// content hash, so an after-the-fact edit to a chained event — or an
1979 /// interior deletion/reordering — is detectable via
1980 /// [`Runtime::verify_event_log_chain`]. Truncation at either end of the
1981 /// log (dropping a prefix or suffix wholesale) is NOT detectable — the
1982 /// chain has no anchored head hash and no trusted tail witness; that is
1983 /// out of scope until the chain head is anchored. Opt-in: existing logs
1984 /// stay byte-identical until enabled.
1985 pub async fn enable_event_log_hash_chaining(&self) {
1986 self.log.lock().await.enable_hash_chaining();
1987 }
1988
1989 /// Verify the event log's tamper-evidence chain (EPIC A / A9). Returns
1990 /// `Ok(n)` for `n` verified chained events, or `Err(index)` naming the
1991 /// first event whose hash/linkage doesn't match — the point of an
1992 /// interior edit, deletion, or reordering. Head/tail truncation is not
1993 /// detectable (no anchored head hash; the first chained event's
1994 /// `prev_hash` is taken on trust) — see `EventLog::verify_chain`.
1995 pub async fn verify_event_log_chain(&self) -> Result<usize, usize> {
1996 self.log.lock().await.verify_chain()
1997 }
1998
1999 /// Install a durable HITL approval ledger backed by a JSONL journal
2000 /// (EPIC A / A7). Loads any existing decisions so approvals survive a
2001 /// restart, then resolves future admission-gate `NeedsApproval`
2002 /// verdicts against it. The canonical daemon path is
2003 /// `~/.car/approvals.jsonl`.
2004 pub async fn set_approval_ledger_path(
2005 &self,
2006 path: impl Into<std::path::PathBuf>,
2007 ) -> std::io::Result<()> {
2008 let ledger = ApprovalLedger::with_journal(path.into())?;
2009 // Surface journal corruption (or a torn concurrent write) instead of
2010 // silently trusting a partial ledger (review A7 — the doc on
2011 // `skipped_on_load` promises callers surface it).
2012 if ledger.skipped_on_load() > 0 {
2013 tracing::warn!(
2014 skipped = ledger.skipped_on_load(),
2015 "approval ledger journal had unparseable lines skipped on load — \
2016 the ledger may be missing decisions"
2017 );
2018 }
2019 *self.approval_ledger.write().await = Some(ledger);
2020 Ok(())
2021 }
2022
2023 /// Use an in-memory approval ledger (no persistence) — primarily for
2024 /// tests and ephemeral runtimes.
2025 pub async fn set_approval_ledger_in_memory(&self) {
2026 *self.approval_ledger.write().await = Some(ApprovalLedger::new());
2027 }
2028
2029 /// Record a human approval for an admission fingerprint (EPIC A / A7).
2030 /// A subsequently re-submitted proposal whose escalation matches this
2031 /// fingerprint is admitted without asking again.
2032 pub async fn approve_admission(
2033 &self,
2034 fingerprint: &str,
2035 reviewer: &str,
2036 reason: &str,
2037 ) -> Result<(), String> {
2038 self.record_admission_decision(fingerprint, ApprovalDecision::Approved, reviewer, reason)
2039 .await
2040 }
2041
2042 /// Record a human rejection for an admission fingerprint (EPIC A / A7).
2043 /// A proposal whose escalation matches a rejected fingerprint is
2044 /// blocked outright.
2045 pub async fn reject_admission(
2046 &self,
2047 fingerprint: &str,
2048 reviewer: &str,
2049 reason: &str,
2050 ) -> Result<(), String> {
2051 self.record_admission_decision(fingerprint, ApprovalDecision::Rejected, reviewer, reason)
2052 .await
2053 }
2054
2055 async fn record_admission_decision(
2056 &self,
2057 fingerprint: &str,
2058 decision: ApprovalDecision,
2059 reviewer: &str,
2060 reason: &str,
2061 ) -> Result<(), String> {
2062 {
2063 let mut guard = self.approval_ledger.write().await;
2064 let ledger = guard
2065 .as_mut()
2066 .ok_or_else(|| "no approval ledger installed".to_string())?;
2067 ledger
2068 .record(ApprovalRecord {
2069 fingerprint: fingerprint.to_string(),
2070 // Admission escalations aren't tier-classified; record the
2071 // most restrictive tier so the decision reads as "elevated".
2072 required_tier: PermissionTier::FullAccess,
2073 decision,
2074 reviewer: reviewer.to_string(),
2075 reason: reason.to_string(),
2076 evidence: None,
2077 decided_at: chrono::Utc::now().to_rfc3339(),
2078 })
2079 // A journal write failure means the decision is NOT durable —
2080 // surface it instead of emitting a false ApprovalRecorded
2081 // audit event (review A7).
2082 .map_err(|e| format!("failed to persist approval decision: {e}"))?;
2083 }
2084 let approval = match decision {
2085 ApprovalDecision::Approved => "approved",
2086 ApprovalDecision::Rejected => "rejected",
2087 };
2088 let mut log = self.log.lock().await;
2089 log.append(
2090 EventKind::ApprovalRecorded,
2091 None,
2092 None,
2093 [
2094 ("fingerprint".to_string(), Value::from(fingerprint)),
2095 ("approval".to_string(), Value::from(approval)),
2096 ("reviewer".to_string(), Value::from(reviewer)),
2097 ("reason".to_string(), Value::from(reason)),
2098 ]
2099 .into(),
2100 );
2101 Ok(())
2102 }
2103
2104 /// Back the idempotency cache with a durable JSONL journal (EPIC A / C3).
2105 ///
2106 /// Loads any existing entries into the in-memory cache, then persists
2107 /// future idempotent results (and rollback invalidations, as
2108 /// tombstones) to the journal. After a crash-restart, re-submitting a
2109 /// completed idempotent action returns the cached result instead of
2110 /// re-executing it — preventing duplicate external side effects. The
2111 /// canonical daemon path is `~/.car/idempotency.jsonl`. Returns the
2112 /// number of live entries loaded.
2113 pub async fn set_idempotency_cache_path(
2114 &self,
2115 path: impl Into<std::path::PathBuf>,
2116 ) -> std::io::Result<usize> {
2117 let path = path.into();
2118 if let Some(parent) = path.parent() {
2119 let _ = std::fs::create_dir_all(parent);
2120 }
2121 // Replay the journal (last entry per key wins; a tombstone removes).
2122 let mut loaded: HashMap<String, ActionResult> = HashMap::new();
2123 if path.exists() {
2124 let content = std::fs::read_to_string(&path)?;
2125 for line in content.lines() {
2126 if line.trim().is_empty() {
2127 continue;
2128 }
2129 if let Ok(entry) = serde_json::from_str::<IdempotencyEntry>(line) {
2130 match entry.result {
2131 Some(r) => {
2132 loaded.insert(entry.key, r);
2133 }
2134 None => {
2135 loaded.remove(&entry.key);
2136 }
2137 }
2138 }
2139 }
2140 }
2141 {
2142 let mut cache = self.idempotency_cache.lock().await;
2143 for (k, v) in loaded.iter() {
2144 cache.entry(k.clone()).or_insert_with(|| v.clone());
2145 }
2146 }
2147 let count = loaded.len();
2148 *self.idempotency_journal.write().await = Some(path);
2149 Ok(count)
2150 }
2151
2152 /// Append an idempotency record to the journal if one is configured.
2153 /// `result = None` writes a tombstone (invalidation).
2154 async fn journal_idempotency(&self, key: &str, result: Option<&ActionResult>) {
2155 let path = {
2156 let guard = self.idempotency_journal.read().await;
2157 guard.clone()
2158 };
2159 let Some(path) = path else {
2160 return;
2161 };
2162 let entry = IdempotencyEntry {
2163 key: key.to_string(),
2164 result: result.cloned(),
2165 };
2166 if let Ok(mut line) = serde_json::to_string(&entry) {
2167 line.push('\n');
2168 use std::io::Write;
2169 // Persistence failure must be VISIBLE (linus review): a
2170 // silently-dropped journal write means an idempotent action
2171 // re-executes its side effects after a restart while the
2172 // operator believes it's covered. Fail-open (the in-memory
2173 // cache still dedups this process) but loudly.
2174 let write = std::fs::OpenOptions::new()
2175 .create(true)
2176 .append(true)
2177 .open(&path)
2178 .and_then(|mut f| f.write_all(line.as_bytes()));
2179 if let Err(e) = write {
2180 tracing::warn!(
2181 path = %path.display(),
2182 error = %e,
2183 "idempotency journal write failed — durable dedup is NOT covering this result"
2184 );
2185 }
2186 }
2187 }
2188
2189 /// Look up the current decision for an admission fingerprint, if any.
2190 pub async fn admission_decision(&self, fingerprint: &str) -> Option<ApprovalDecision> {
2191 let guard = self.approval_ledger.read().await;
2192 guard
2193 .as_ref()
2194 .and_then(|l| l.lookup(fingerprint).map(|r| r.decision))
2195 }
2196
2197 /// Verify a model's tool-use claims against the runtime's own execution
2198 /// receipts (EPIC A / A6 — arXiv 2603.10060). The runtime owns tool
2199 /// execution and logs it, so it holds unforgeable ground truth: this
2200 /// projects [`car_eventlog::tool_receipts::ToolReceipt`]s from the event
2201 /// log and cross-checks the supplied claims, catching a fabricated tool
2202 /// reference, a misstated result count, or a false "found nothing".
2203 ///
2204 /// `proposal_id` scopes the cross-check window to a single proposal's
2205 /// events (pass the proposal whose response the claims came from) — a
2206 /// claim is never judged against another run's receipts. The check is
2207 /// retention-coherent (review A6): when the log has trimmed events and
2208 /// the window can't be proven complete (unscoped, or the proposal's
2209 /// `ProposalReceived` marker — which precedes every receipt of that
2210 /// proposal — was itself evicted), a claim without a receipt comes back
2211 /// in `ReceiptReport::ungroundable` ("window evicted") instead of being
2212 /// mis-flagged `fabricated_tool_reference`.
2213 ///
2214 /// Returns the [`car_eventlog::tool_receipts::ReceiptReport`]; when it is not grounded, a
2215 /// `ToolReceiptHallucination` event is emitted so the caller's
2216 /// verdict→action loop (reject/flag the response) is auditable.
2217 /// Deterministic, zero-inference. Claims arrive structured — CAR's
2218 /// thesis is that intent is structured IR, so a caller extracts claims
2219 /// from the model's tool_calls / IR rather than regexing prose.
2220 pub async fn verify_tool_receipts(
2221 &self,
2222 claims: &[car_eventlog::tool_receipts::ToolClaim],
2223 proposal_id: Option<&str>,
2224 ) -> car_eventlog::tool_receipts::ReceiptReport {
2225 let (receipts, window_complete) = {
2226 let log = self.log.lock().await;
2227 let complete = if log.trimmed_events() == 0 {
2228 // Nothing was ever evicted — the window is complete whether
2229 // or not it is scoped.
2230 true
2231 } else {
2232 match proposal_id {
2233 // A proposal's window is complete iff its ProposalReceived
2234 // marker survived retention: every receipt of the proposal
2235 // was appended after it, so if the marker is retained, so
2236 // are the receipts.
2237 Some(pid) => log.events().iter().any(|e| {
2238 e.kind == EventKind::ProposalReceived
2239 && e.proposal_id.as_deref() == Some(pid)
2240 }),
2241 // Unscoped check over a trimmed log: unknowable.
2242 None => false,
2243 }
2244 };
2245 (
2246 car_eventlog::tool_receipts::receipts_from_events_scoped(log.events(), proposal_id),
2247 complete,
2248 )
2249 };
2250 let report = car_eventlog::tool_receipts::verify_tool_claims_windowed(
2251 claims,
2252 &receipts,
2253 window_complete,
2254 );
2255 if !report.grounded {
2256 let mut log = self.log.lock().await;
2257 log.append(
2258 EventKind::ToolReceiptHallucination,
2259 None,
2260 proposal_id,
2261 [
2262 (
2263 "count".to_string(),
2264 Value::from(report.hallucinations.len()),
2265 ),
2266 (
2267 "hallucinations".to_string(),
2268 serde_json::to_value(&report.hallucinations).unwrap_or_default(),
2269 ),
2270 ]
2271 .into(),
2272 );
2273 }
2274 report
2275 }
2276
2277 /// Run every registered admission gate against a proposal and fold
2278 /// their verdicts into a single [`crate::admission::AdmissionDecision`].
2279 ///
2280 /// Each gate's outcome is recorded as an `AdmissionGateDecision` event
2281 /// so a denial is attributable. Aggregation is conjunctive and
2282 /// fail-closed: the proposal is admitted only if every gate allowed it.
2283 /// Returns an admit decision immediately when no gates are registered
2284 /// (zero overhead on the common path).
2285 async fn run_admission_gates(
2286 &self,
2287 proposal: &ActionProposal,
2288 session_id: Option<&str>,
2289 scope: Option<&crate::scope::RuntimeScope>,
2290 ) -> crate::admission::AdmissionDecision {
2291 use crate::admission::{AdmissionDecision, GateContext};
2292
2293 let gates = self.admission_gates.read().await;
2294 if gates.is_empty() {
2295 return AdmissionDecision::admit();
2296 }
2297
2298 // One consistent snapshot for every gate this pass: the keys this
2299 // proposal may observe (all of them unscoped; shared plus its own
2300 // tenant's under a tenant, never another tenant's). A gate resolves
2301 // each action's keys in the namespace that action executes in
2302 // (car#3121).
2303 let tenant = scope.and_then(|s| s.tenant_id.as_deref());
2304 let (state, versions) = self.state.versioned_snapshot_visible_to(tenant);
2305 let ctx = GateContext {
2306 session_id,
2307 scope,
2308 state: &state,
2309 versions: &versions,
2310 };
2311
2312 let mut decision = AdmissionDecision::admit();
2313 for gate in gates.iter() {
2314 let outcome = gate.check(proposal, &ctx).await;
2315 // Audit every gate decision (allow included) so the trail shows
2316 // which gates ran, not just which objected.
2317 let mut props: HashMap<String, Value> = HashMap::new();
2318 props.insert("gate".to_string(), Value::from(gate.name()));
2319 // Pre-execution proposal admission (vs the multi-agent
2320 // commit barrier, which emits the same event kind with
2321 // phase:"commit_barrier").
2322 props.insert("phase".to_string(), Value::from("admission"));
2323 props.insert("decision".to_string(), Value::from(outcome.label()));
2324 match &outcome {
2325 crate::admission::GateOutcome::Allow => {}
2326 crate::admission::GateOutcome::Reject { blocked, reason } => {
2327 props.insert("reason".to_string(), Value::from(reason.clone()));
2328 props.insert(
2329 "blocked".to_string(),
2330 serde_json::to_value(blocked).unwrap_or_default(),
2331 );
2332 }
2333 crate::admission::GateOutcome::NeedsApproval {
2334 actions,
2335 fingerprint,
2336 reason,
2337 } => {
2338 props.insert("reason".to_string(), Value::from(reason.clone()));
2339 props.insert(
2340 "blocked".to_string(),
2341 serde_json::to_value(actions).unwrap_or_default(),
2342 );
2343 props.insert("fingerprint".to_string(), Value::from(fingerprint.clone()));
2344 }
2345 }
2346 {
2347 let mut log = self.log.lock().await;
2348 log.append(
2349 EventKind::AdmissionGateDecision,
2350 None,
2351 Some(&proposal.id),
2352 props,
2353 );
2354 }
2355 decision.absorb(gate.name(), outcome);
2356 }
2357 decision
2358 }
2359
2360 /// Install a harness operating config — the live end of the Evolution
2361 /// Agent loop (survey §3.5). After the meta-agent's `HarnessConfig::apply`
2362 /// produces a governed, regression-gated config, hand it here to take
2363 /// effect: `max_retries`/`retry_backoff_ms` drive the per-action retry
2364 /// loop, and `planning_max_replans` is mapped onto the replan budget.
2365 /// Every `HarnessConfig` knob is consumed here — none is aspirational.
2366 pub async fn set_harness_config(&self, cfg: car_memgine::HarnessConfig) {
2367 self.replan_config.write().await.max_replans = cfg.planning_max_replans;
2368 *self.harness_config.write().await = Some(cfg);
2369 }
2370
2371 /// The current harness operating config, if one has been installed.
2372 pub async fn harness_config(&self) -> Option<car_memgine::HarnessConfig> {
2373 self.harness_config.read().await.clone()
2374 }
2375
2376 /// Atomically read-modify-write the harness operating config under ONE
2377 /// write lock (installing the default first when none is set). The
2378 /// get→mutate→set alternative is a lost-update race when two requests
2379 /// mutate concurrently — the daemon's `evolution.run` harness-apply path
2380 /// uses this instead (kernel review S3). Keeps the same replan-budget
2381 /// propagation as [`Self::set_harness_config`].
2382 pub async fn update_harness_config<R>(
2383 &self,
2384 f: impl FnOnce(&mut car_memgine::HarnessConfig) -> R,
2385 ) -> R {
2386 let mut guard = self.harness_config.write().await;
2387 let cfg = guard.get_or_insert_with(car_memgine::HarnessConfig::default);
2388 let out = f(cfg);
2389 let max_replans = cfg.planning_max_replans;
2390 drop(guard);
2391 self.replan_config.write().await.max_replans = max_replans;
2392 out
2393 }
2394
2395 /// Run the pre-execution transactional check against the current
2396 /// versioned state, each action's keys resolved in the namespace it
2397 /// executes in (car#3121). Returns the conflicting action ids when the
2398 /// mode is `Strict` and conflicts exist (so the caller can reject those
2399 /// actions); always emits `TransactionConflict` telemetry for each
2400 /// conflict found. Empty/`None` means "proceed".
2401 async fn transaction_precheck(
2402 &self,
2403 proposal: &ActionProposal,
2404 scope: Option<&crate::scope::RuntimeScope>,
2405 ) -> std::collections::HashSet<String> {
2406 let mode = *self.transaction_check.read().await;
2407 if mode == TransactionCheckMode::Off {
2408 return std::collections::HashSet::new();
2409 }
2410 let tenant = scope.and_then(|s| s.tenant_id.as_deref());
2411 let resolved = proposal_in_store_keys(proposal, tenant);
2412 let (state, versions) = self.state.versioned_snapshot_visible_to(tenant);
2413 // Ordering comes from the proposal as written, which is what the
2414 // executor builds its DAG from: a tenant action depending on `k` runs
2415 // after a tool call declaring `k`, though their resolved keys differ.
2416 let report = car_verify::check_transaction_ordered_by(
2417 &resolved,
2418 &proposal.actions,
2419 &versions,
2420 Some(&state),
2421 );
2422 if report.consistent {
2423 return std::collections::HashSet::new();
2424 }
2425 let mut blocked = std::collections::HashSet::new();
2426 let mut log = self.log.lock().await;
2427 for c in &report.conflicts {
2428 for aid in &c.actions {
2429 blocked.insert(aid.clone());
2430 }
2431 log.append(
2432 EventKind::TransactionConflict,
2433 c.actions.first().map(|s| s.as_str()),
2434 Some(&proposal.id),
2435 [
2436 // Use the serde representation (snake_case: write_write
2437 // / read_write / stale_assumption) the .d.ts/.pyi/doc
2438 // contract is written against — NOT Debug, which would
2439 // emit "writewrite" (neo review #4).
2440 (
2441 "kind".to_string(),
2442 serde_json::to_value(c.kind).unwrap_or_default(),
2443 ),
2444 ("key".to_string(), Value::from(c.key.clone())),
2445 (
2446 "actions".to_string(),
2447 serde_json::to_value(&c.actions).unwrap_or_default(),
2448 ),
2449 (
2450 "explanation".to_string(),
2451 Value::from(c.explanation.clone()),
2452 ),
2453 ("resolution".to_string(), Value::from(c.resolution.clone())),
2454 ]
2455 .into(),
2456 );
2457 }
2458 drop(log);
2459 // Only Strict blocks execution; Warn records and proceeds.
2460 match mode {
2461 TransactionCheckMode::Strict => blocked,
2462 _ => std::collections::HashSet::new(),
2463 }
2464 }
2465
2466 /// Register a tool with just a name (backward compatible).
2467 pub async fn register_tool(&self, name: &str) {
2468 let schema = ToolSchema {
2469 name: name.to_string(),
2470 source: car_ir::ToolSourceKind::UserDefined,
2471 description: String::new(),
2472 parameters: serde_json::Value::Object(Default::default()),
2473 returns: None,
2474 idempotent: false,
2475 cache_ttl_secs: None,
2476 rate_limit: None,
2477 };
2478 self.register_tool_schema(schema).await;
2479 }
2480
2481 /// Register a tool with full schema.
2482 pub async fn register_tool_schema(&self, schema: ToolSchema) {
2483 // Auto-configure cache if schema specifies it
2484 if let Some(ttl) = schema.cache_ttl_secs {
2485 self.result_cache.enable_caching(&schema.name, ttl).await;
2486 }
2487 // Auto-configure rate limit if schema specifies it
2488 if let Some(ref rl) = schema.rate_limit {
2489 self.rate_limiter
2490 .set_limit(
2491 &schema.name,
2492 RateLimit {
2493 max_calls: rl.max_calls,
2494 interval_secs: rl.interval_secs,
2495 },
2496 )
2497 .await;
2498 }
2499 self.tools.write().await.insert(schema.name.clone(), schema);
2500 }
2501
2502 /// Register a tool via the canonical registry.
2503 /// This is the preferred way to register tools — it updates both the
2504 /// registry and the legacy tools HashMap for backward compatibility.
2505 pub async fn register_tool_entry(&self, mut entry: crate::registry::ToolEntry) {
2506 // The registry's source is authoritative for both event provenance and
2507 // the public tools.list/schema view, including literal-built entries.
2508 entry.schema.source = entry.source.kind();
2509 let schema = entry.schema.clone();
2510 self.registry.register(entry).await;
2511 self.register_tool_schema(schema).await;
2512 }
2513
2514 /// Remove a tool from both the canonical registry and the legacy
2515 /// `tools` schema map, so the model no longer sees it and the
2516 /// validator no longer accepts it. Used when a remote MCP connector
2517 /// tool is disabled or its connector is removed. Returns true if the
2518 /// tool was present in either store.
2519 pub async fn unregister_tool(&self, name: &str) -> bool {
2520 let removed_entry = self.registry.remove(name).await.is_some();
2521 let removed_schema = self.tools.write().await.remove(name).is_some();
2522 removed_entry || removed_schema
2523 }
2524
2525 /// Register CAR's built-in agent utility stdlib.
2526 ///
2527 /// This is an opt-in convenience layer for common local-file and text tools.
2528 /// Existing runtimes remain unchanged until this is called.
2529 pub async fn register_agent_basics(&self) {
2530 for entry in crate::agent_basics::entries() {
2531 self.register_tool_entry(entry).await;
2532 }
2533 }
2534
2535 /// Refuse the built-in file tools access to the host paths `guard` names.
2536 ///
2537 /// Applies on the built-in fallback path only, and only while the bound
2538 /// substrate is local: a configured executor that claims a tool owns its
2539 /// own checks, and a remote substrate owns its own filesystem. See
2540 /// [`crate::agent_basics::execute_with_ledger_guarded`] for what is checked.
2541 pub async fn set_local_path_guard(&self, guard: crate::agent_basics::LocalPathGuard) {
2542 *self.local_path_guard.write().await = Some(guard);
2543 }
2544
2545 /// Get all registered tool schemas (for model prompt generation).
2546 pub async fn tool_schemas(&self) -> Vec<ToolSchema> {
2547 self.tools.read().await.values().cloned().collect()
2548 }
2549
2550 /// Set a cost budget that limits proposal execution.
2551 pub async fn set_cost_budget(&self, budget: CostBudget) {
2552 *self.cost_budget.write().await = Some(budget);
2553 }
2554
2555 /// Set per-agent capability permissions that restrict tools, state keys, and action count.
2556 pub async fn set_capabilities(&self, caps: CapabilitySet) {
2557 *self.capabilities.write().await = Some(caps);
2558 }
2559
2560 /// Set a per-tool rate limit (token bucket).
2561 ///
2562 /// `max_calls` tokens are available per `interval_secs` window.
2563 /// When the bucket is empty, `dispatch()` applies backpressure by
2564 /// waiting until a token refills.
2565 pub async fn set_rate_limit(&self, tool: &str, max_calls: u32, interval_secs: f64) {
2566 self.rate_limiter
2567 .set_limit(
2568 tool,
2569 RateLimit {
2570 max_calls,
2571 interval_secs,
2572 },
2573 )
2574 .await;
2575 }
2576
2577 /// Enable cross-proposal result caching for a tool with a TTL in seconds.
2578 pub async fn enable_tool_cache(&self, tool: &str, ttl_secs: u64) {
2579 self.result_cache.enable_caching(tool, ttl_secs).await;
2580 }
2581
2582 /// Execute a proposal with automatic replanning on failure.
2583 ///
2584 /// If a `ReplanCallback` is registered and `max_replans > 0`, the runtime
2585 /// will catch abort failures, roll back state, ask the model for an
2586 /// alternative proposal via the callback, and re-execute. This transforms
2587 /// "execute-and-hope" into "execute-and-recover."
2588 ///
2589 /// If no callback is registered or `max_replans == 0`, behaves identically
2590 /// to a single `execute_inner()` call (zero overhead, fully backward compatible).
2591 #[instrument(
2592 name = "proposal.execute",
2593 skip_all,
2594 fields(
2595 proposal_id = %proposal.id,
2596 action_count = proposal.actions.len(),
2597 )
2598 )]
2599 pub async fn execute(&self, proposal: &ActionProposal) -> ProposalResult {
2600 // Forward to the cancel-aware variant with a never-cancelled
2601 // token. Existing callers see no behaviour change.
2602 let token = tokio_util::sync::CancellationToken::new();
2603 self.execute_with_cancel(proposal, &token).await
2604 }
2605
2606 /// Execute a proposal scoped to a specific session id.
2607 ///
2608 /// Validation walks the global policy registry plus the session's
2609 /// own registry — both must pass for an action to run. Session
2610 /// policies can deny what global allows; they cannot allow what
2611 /// global denies (validation is conjunctive).
2612 ///
2613 /// Returns the same [`ProposalResult`] shape as [`Self::execute`].
2614 /// Errors with an action-level rejection if the session id is
2615 /// unknown — callers should check via [`Self::session_exists`] or
2616 /// trust an id they minted via [`Self::open_session`].
2617 pub async fn execute_with_session(
2618 &self,
2619 proposal: &ActionProposal,
2620 session_id: &str,
2621 ) -> ProposalResult {
2622 let token = tokio_util::sync::CancellationToken::new();
2623 self.execute_with_session_and_cancel(proposal, session_id, &token)
2624 .await
2625 }
2626
2627 /// Combined session-scoped + cancellable execute. The session id
2628 /// is passed verbatim to the per-action policy check; the cancel
2629 /// token behaves identically to [`Self::execute_with_cancel`].
2630 pub async fn execute_with_session_and_cancel(
2631 &self,
2632 proposal: &ActionProposal,
2633 session_id: &str,
2634 cancel: &tokio_util::sync::CancellationToken,
2635 ) -> ProposalResult {
2636 self.execute_with_optional_session(proposal, Some(session_id), None, None, cancel)
2637 .await
2638 }
2639
2640 /// Execute an active-run proposal while requiring every accepted replan
2641 /// to retain the authenticated proposal id already claimed by the server.
2642 pub async fn execute_with_session_and_stable_replan_id(
2643 &self,
2644 proposal: &ActionProposal,
2645 session_id: &str,
2646 ) -> ProposalResult {
2647 let token = tokio_util::sync::CancellationToken::new();
2648 self.execute_with_optional_session(
2649 proposal,
2650 Some(session_id),
2651 None,
2652 Some(&proposal.id),
2653 &token,
2654 )
2655 .await
2656 }
2657
2658 /// Execute a proposal with cooperative cancellation.
2659 ///
2660 /// The runtime checks `token.is_cancelled()` at each DAG level
2661 /// boundary. When set, every action that hadn't yet started runs
2662 /// is reported as `Skipped` with `error = "canceled: ..."` so
2663 /// callers can distinguish "user pulled the plug" from "earlier
2664 /// abort cascaded." Actions already in flight continue to
2665 /// completion — tool calls dispatched to user-provided executors
2666 /// can't be safely interrupted from the engine.
2667 ///
2668 /// The CAR A2A bridge uses this so `tasks/cancel` produces a
2669 /// `ProposalResult` with clean partial state rather than relying
2670 /// on `JoinHandle::abort` to interrupt mid-await (which leaves
2671 /// no record of which actions actually ran).
2672 ///
2673 /// **FFI exposure:** this method is intentionally not surfaced
2674 /// through the NAPI / PyO3 / `car-server-core` JSON-RPC bindings.
2675 /// Those consumers (Node, Python, WebSocket) don't currently
2676 /// expose long-running async-task surfaces that need
2677 /// cancellation; the bridge is the lone consumer. When a binding
2678 /// gains a long-running task surface, the path is clear: add a
2679 /// per-binding token registry keyed by some caller-provided id,
2680 /// expose `cancelExecution(id)` / `cancel_execution(id)` /
2681 /// `proposal.cancel { id }`, and have the runtime call
2682 /// `execute_with_cancel` with the matching token. Skipping that
2683 /// today avoids speculative API surface that bloats bindings
2684 /// without a consumer.
2685 pub async fn execute_with_cancel(
2686 &self,
2687 proposal: &ActionProposal,
2688 cancel: &tokio_util::sync::CancellationToken,
2689 ) -> ProposalResult {
2690 self.execute_with_optional_session(proposal, None, None, None, cancel)
2691 .await
2692 }
2693
2694 pub async fn execute_with_stable_replan_id(&self, proposal: &ActionProposal) -> ProposalResult {
2695 let token = tokio_util::sync::CancellationToken::new();
2696 self.execute_with_optional_session(proposal, None, None, Some(&proposal.id), &token)
2697 .await
2698 }
2699
2700 /// Execute a proposal with an attached [`RuntimeScope`](crate::scope::RuntimeScope)
2701 /// (Parslee-ai/car#187 phase 3).
2702 ///
2703 /// Same contract as [`Self::execute`] plus a per-execution
2704 /// identity surface — typically built by the car-a2a dispatcher
2705 /// from the verified `Identity` and cooperative `a2a_caller`
2706 /// metadata on the inbound `ActionProposal`. The scope is
2707 /// recorded on the event log so downstream audit / log analysis
2708 /// can see which caller / tenant issued each action.
2709 ///
2710 /// **What this enforces today**: scope is captured + logged.
2711 /// Memgine queries and state-store ops still hit global
2712 /// namespaces — those follow-ups are tracked under #187.
2713 /// Tool / policy code that needs per-tenant behaviour right now
2714 /// should keep reading `proposal.context["a2a_caller_verified"]`
2715 /// directly (the phase 1 / 2 surface).
2716 pub async fn execute_scoped(
2717 &self,
2718 proposal: &ActionProposal,
2719 scope: &crate::scope::RuntimeScope,
2720 ) -> ProposalResult {
2721 let token = tokio_util::sync::CancellationToken::new();
2722 self.execute_scoped_with_cancel(proposal, scope, &token)
2723 .await
2724 }
2725
2726 /// Combined scoped + cancellable execute. Mirrors the shape of
2727 /// [`Self::execute_with_session_and_cancel`] for symmetry — both
2728 /// add a side-channel (session id / scope) on top of the
2729 /// cancellable form.
2730 pub async fn execute_scoped_with_cancel(
2731 &self,
2732 proposal: &ActionProposal,
2733 scope: &crate::scope::RuntimeScope,
2734 cancel: &tokio_util::sync::CancellationToken,
2735 ) -> ProposalResult {
2736 self.execute_with_optional_session(proposal, None, Some(scope), None, cancel)
2737 .await
2738 }
2739
2740 pub async fn execute_scoped_with_stable_replan_id(
2741 &self,
2742 proposal: &ActionProposal,
2743 scope: &crate::scope::RuntimeScope,
2744 ) -> ProposalResult {
2745 let token = tokio_util::sync::CancellationToken::new();
2746 self.execute_with_optional_session(proposal, None, Some(scope), Some(&proposal.id), &token)
2747 .await
2748 }
2749
2750 /// Internal entry point that backs both
2751 /// [`Self::execute_with_cancel`] (no session) and
2752 /// [`Self::execute_with_session_and_cancel`]. Holds the replan
2753 /// loop and threads `session_id` into the per-action validation
2754 /// path so session-scoped policies stack on top of global ones.
2755 async fn execute_with_optional_session(
2756 &self,
2757 proposal: &ActionProposal,
2758 session_id: Option<&str>,
2759 scope: Option<&crate::scope::RuntimeScope>,
2760 required_replan_proposal_id: Option<&str>,
2761 cancel: &tokio_util::sync::CancellationToken,
2762 ) -> ProposalResult {
2763 // The shared StateStore has one proposal transaction at a time. The
2764 // guard spans validation, replanning, and rollback so distinct Runtime
2765 // facades over the same store cannot interleave state snapshots.
2766 let _proposal_execution = self.state.lock_proposal_execution().await;
2767 self.execute_with_optional_session_already_guarded(
2768 proposal,
2769 session_id,
2770 scope,
2771 required_replan_proposal_id,
2772 cancel,
2773 )
2774 .await
2775 }
2776
2777 /// Execute while the caller retains this StateStore's proposal-execution
2778 /// guard. `plan_and_execute` uses this path so one guard covers its initial
2779 /// snapshot, ranking, every candidate, and the final commit/rollback.
2780 async fn execute_with_optional_session_already_guarded(
2781 &self,
2782 proposal: &ActionProposal,
2783 session_id: Option<&str>,
2784 scope: Option<&crate::scope::RuntimeScope>,
2785 required_replan_proposal_id: Option<&str>,
2786 cancel: &tokio_util::sync::CancellationToken,
2787 ) -> ProposalResult {
2788 // Establish canonical identity before any other rejection so an
2789 // undigested lineage entry can only mean the proposal was genuinely
2790 // impossible to represent as JCS/I-JSON. No journal row is emitted.
2791 let original_proposal_digest = match proposal_digest(proposal) {
2792 Ok(digest) => digest,
2793 Err(error) => {
2794 self.log.lock().await.append(
2795 EventKind::StateRollback,
2796 None,
2797 Some(&proposal.id),
2798 proposal_rejection_boundary_data(proposal, None, &error),
2799 );
2800 let result = ProposalResult::for_proposal(
2801 proposal,
2802 proposal
2803 .actions
2804 .iter()
2805 .map(|action| rejected_result(&action.id, error.clone()))
2806 .collect(),
2807 CostSummary::default(),
2808 );
2809 return finalize_proposal_result(
2810 result,
2811 &proposal.id,
2812 &[ProposalLineageEntry {
2813 generation: 0,
2814 proposal_id: proposal.id.clone(),
2815 proposal_digest: None,
2816 status: ProposalLineageStatus::Rejected,
2817 rejection_reason: Some(error),
2818 }],
2819 &[],
2820 );
2821 }
2822 };
2823
2824 // Proposal-local action ids are the join key for DAG scheduling,
2825 // result rows, receipts, and StateTransition attribution. Reject a
2826 // duplicate before scope/admission/transaction logging or any state
2827 // mutation. Retried attempts remain valid reuse of one admitted id.
2828 if let Err(error) = validate_proposal_action_ids(proposal) {
2829 self.log.lock().await.append(
2830 EventKind::StateRollback,
2831 None,
2832 Some(&proposal.id),
2833 proposal_rejection_boundary_data(proposal, Some(&original_proposal_digest), &error),
2834 );
2835 let result = ProposalResult::for_proposal(
2836 proposal,
2837 proposal
2838 .actions
2839 .iter()
2840 .map(|action| rejected_result(&action.id, error.clone()))
2841 .collect(),
2842 CostSummary::default(),
2843 );
2844 return finalize_proposal_result(
2845 result,
2846 &proposal.id,
2847 &[proposal_lineage_entry(
2848 proposal,
2849 0,
2850 ProposalLineageStatus::Rejected,
2851 Some(error),
2852 )],
2853 &[],
2854 );
2855 }
2856 if let Err(error) = validate_proposal_retry_limits(proposal) {
2857 self.log.lock().await.append(
2858 EventKind::StateRollback,
2859 None,
2860 Some(&proposal.id),
2861 proposal_rejection_boundary_data(proposal, Some(&original_proposal_digest), &error),
2862 );
2863 let result = ProposalResult::for_proposal(
2864 proposal,
2865 proposal
2866 .actions
2867 .iter()
2868 .map(|action| rejected_result(&action.id, error.clone()))
2869 .collect(),
2870 CostSummary::default(),
2871 );
2872 return finalize_proposal_result(
2873 result,
2874 &proposal.id,
2875 &[proposal_lineage_entry(
2876 proposal,
2877 0,
2878 ProposalLineageStatus::Rejected,
2879 Some(error),
2880 )],
2881 &[],
2882 );
2883 }
2884 let config = self.replan_config.read().await.clone();
2885 let mut current_proposal = proposal.clone();
2886 let mut attempt: u32 = 0;
2887 let mut lineage = vec![proposal_lineage_entry(
2888 proposal,
2889 0,
2890 ProposalLineageStatus::Accepted,
2891 None,
2892 )];
2893
2894 // Phase 3 foundation (Parslee-ai/car#187): record the scope
2895 // on the event log so audit / log analysis can correlate
2896 // actions to the caller / tenant that triggered them. Only
2897 // logged when at least one identity field is set — keeps
2898 // the existing in-process call sites free of noise.
2899 if let Some(s) = scope {
2900 if !s.is_unscoped() {
2901 let mut props: HashMap<String, Value> = HashMap::new();
2902 if let Some(cid) = &s.caller_id {
2903 props.insert("caller_id".to_string(), Value::from(cid.as_str()));
2904 }
2905 if let Some(tid) = &s.tenant_id {
2906 props.insert("tenant_id".to_string(), Value::from(tid.as_str()));
2907 }
2908 if !s.claims.is_empty() {
2909 if let Ok(claims_json) = serde_json::to_value(&s.claims) {
2910 props.insert("claims".to_string(), claims_json);
2911 }
2912 }
2913 let mut log = self.log.lock().await;
2914 log.append(EventKind::SessionScope, None, Some(&proposal.id), props);
2915 }
2916 }
2917
2918 // Pre-execution transactional conflict check (survey §4.3/§5.2.4).
2919 // Off by default; Warn records conflicts; Strict rejects the whole
2920 // proposal before any action runs, since a transactional conflict is
2921 // a property of the action *set* against current state, not an
2922 // isolated action. This is an advisory planning-time gate, not a
2923 // substitute for per-action execution-time validation. Replanned
2924 // proposals are re-checked inside the loop (at the replan quality
2925 // gate below). A pre-execution rejection here deliberately produces
2926 // no execution trajectory — nothing ran; the emitted
2927 // `TransactionConflict` events are the audit record.
2928 let blocked = self.transaction_precheck(¤t_proposal, scope).await;
2929 if !blocked.is_empty() {
2930 let reason = "transactional conflict with current shared state (strict mode); \
2931 see TransactionConflict events for details and resolution"
2932 .to_string();
2933 let results = current_proposal
2934 .actions
2935 .iter()
2936 .map(|a| rejected_result(&a.id, reason.clone()))
2937 .collect();
2938 self.log.lock().await.append(
2939 EventKind::StateRollback,
2940 None,
2941 Some(¤t_proposal.id),
2942 proposal_rejection_boundary_data(
2943 ¤t_proposal,
2944 Some(&original_proposal_digest),
2945 &reason,
2946 ),
2947 );
2948 lineage[0].status = ProposalLineageStatus::Rejected;
2949 lineage[0].rejection_reason = Some(reason);
2950 return finalize_proposal_result(
2951 ProposalResult::for_proposal(proposal, results, Default::default()),
2952 &proposal.id,
2953 &lineage,
2954 &[],
2955 );
2956 }
2957
2958 // Pre-execution admission gates (EPIC A / task A1 — the safety
2959 // seam). Runs every registered AdmissionGate against the proposal
2960 // before any action executes. Aggregation is conjunctive and
2961 // fail-closed: a proposal any gate blocks (or escalates to
2962 // approval) does not run. Like the transactional pre-check above, a
2963 // rejection here produces no execution trajectory — the emitted
2964 // AdmissionGateDecision events are the audit record. No gates
2965 // registered → zero overhead, identical behavior.
2966 let admission = self
2967 .run_admission_gates(¤t_proposal, session_id, scope)
2968 .await;
2969 if !admission.admitted {
2970 let gate = admission.deciding_gate.as_deref().unwrap_or("admission");
2971 let base_reason = admission
2972 .reason
2973 .clone()
2974 .unwrap_or_else(|| "blocked by admission gate".to_string());
2975 // Resolve approval escalations against the durable ledger (A7).
2976 // A hard `Reject` from ANY gate is never overridable — the
2977 // ledger is not consulted at all in that case (an old approval
2978 // for one gate's escalation must not steamroll another gate's
2979 // deny). Otherwise EVERY escalation must resolve to Approved,
2980 // each by its own fingerprint: one Rejected fingerprint blocks,
2981 // one unseen fingerprint stays pending (fail-closed).
2982 let mut approved = false;
2983 let reason = if admission.hard_rejected {
2984 format!("{base_reason} (gate: {gate})")
2985 } else if admission.needs_approval() {
2986 let mut blocking_reason = None;
2987 for esc in &admission.escalations {
2988 match self.admission_decision(&esc.fingerprint).await {
2989 Some(ApprovalDecision::Approved) => continue,
2990 Some(ApprovalDecision::Rejected) => {
2991 blocking_reason = Some(format!(
2992 "rejected by operator (gate: {}; fingerprint: {})",
2993 esc.gate, esc.fingerprint
2994 ));
2995 break;
2996 }
2997 None => {
2998 blocking_reason = Some(format!(
2999 "requires human approval (gate: {}; {}); \
3000 approve fingerprint '{}' then re-run",
3001 esc.gate, esc.reason, esc.fingerprint
3002 ));
3003 break;
3004 }
3005 }
3006 }
3007 match blocking_reason {
3008 None => {
3009 approved = true;
3010 // Audit every durable approval taking effect.
3011 let mut log = self.log.lock().await;
3012 for esc in &admission.escalations {
3013 log.append(
3014 EventKind::ApprovalRecorded,
3015 None,
3016 Some(&proposal.id),
3017 [
3018 (
3019 "fingerprint".to_string(),
3020 Value::from(esc.fingerprint.as_str()),
3021 ),
3022 ("gate".to_string(), Value::from(esc.gate.as_str())),
3023 ("approval".to_string(), Value::from("approved")),
3024 ("applied".to_string(), Value::from(true)),
3025 ]
3026 .into(),
3027 );
3028 }
3029 String::new()
3030 }
3031 Some(r) => r,
3032 }
3033 } else {
3034 format!("{base_reason} (gate: {gate})")
3035 };
3036 if approved {
3037 // Escalation cleared by a durable approval — proceed to
3038 // execution as if admitted.
3039 } else {
3040 let results = current_proposal
3041 .actions
3042 .iter()
3043 .map(|a| {
3044 // Name the specific reason on the offending actions; a
3045 // generic note on the rest (the whole proposal is held,
3046 // since a safety hazard is a property of the set).
3047 if admission.blocked.is_empty() || admission.blocked.contains(&a.id) {
3048 rejected_result(&a.id, reason.clone())
3049 } else {
3050 rejected_result(
3051 &a.id,
3052 format!("proposal blocked by admission gate: {gate}"),
3053 )
3054 }
3055 })
3056 .collect();
3057 self.log.lock().await.append(
3058 EventKind::StateRollback,
3059 None,
3060 Some(¤t_proposal.id),
3061 proposal_rejection_boundary_data(
3062 ¤t_proposal,
3063 Some(&original_proposal_digest),
3064 &reason,
3065 ),
3066 );
3067 lineage[0].status = ProposalLineageStatus::Rejected;
3068 lineage[0].rejection_reason = Some(reason.clone());
3069 let result = finalize_proposal_result(
3070 ProposalResult::for_proposal(proposal, results, Default::default()),
3071 &proposal.id,
3072 &lineage,
3073 &[],
3074 );
3075 // Record the trajectory even though nothing dispatched. This
3076 // return is *before* the execution loop's persist calls, so
3077 // without this an admission rejection is invisible to the
3078 // trajectory store — and the per-tool success rates
3079 // `verify.monte_carlo` derives from it would silently skew
3080 // optimistic, counting only calls that got far enough to run.
3081 // The state map is empty because no action mutated anything.
3082 if let Some(err) = self.persist_trajectory(
3083 proposal,
3084 ¤t_proposal,
3085 &result,
3086 car_memgine::TrajectoryOutcome::Failed,
3087 0,
3088 &HashMap::new(),
3089 ) {
3090 tracing::warn!(
3091 error = %err,
3092 proposal_id = %proposal.id,
3093 "failed to persist trajectory for admission-rejected proposal"
3094 );
3095 }
3096 return finalize_proposal_result(result, &proposal.id, &lineage, &[]);
3097 }
3098 }
3099
3100 let mut accepted_proposal_preimages = vec![AcceptedProposalPreimage {
3101 generation: 0,
3102 proposal_digest: original_proposal_digest,
3103 proposal: proposal.clone(),
3104 }];
3105
3106 loop {
3107 let (result, state_before_map) = self
3108 .execute_inner_with_cancel(¤t_proposal, Some(cancel), session_id, scope)
3109 .await;
3110
3111 // Statuses that trigger rollback + replan: always runtime Failed,
3112 // and (opt-in) validator/policy/capability Rejected.
3113 let replan_triggers = |s: &ActionStatus| {
3114 *s == ActionStatus::Failed
3115 || (config.replan_on_rejected && *s == ActionStatus::Rejected)
3116 };
3117
3118 // A terminal tool failure ends this engine execution. It may not
3119 // enter the retry loop above or the proposal-replan loop here;
3120 // daemon-owned session halting is layered on top by the caller.
3121 let terminal_failure = result.results.iter().any(|result| result.terminal);
3122 // Check if we aborted
3123 let aborted = result.results.iter().any(|r| replan_triggers(&r.status));
3124 let rollback_durability_failed = result.results.iter().any(|result| {
3125 result.error.as_deref().is_some_and(|error| {
3126 error.contains(ROLLBACK_DURABILITY_ERROR)
3127 || error.contains(ROLLBACK_DURABILITY_UNKNOWN)
3128 })
3129 });
3130 if !aborted
3131 || terminal_failure
3132 || attempt >= config.max_replans
3133 || rollback_durability_failed
3134 {
3135 if aborted && !terminal_failure && attempt > 0 && !rollback_durability_failed {
3136 // Exhausted all replan attempts
3137 let mut log = self.log.lock().await;
3138 log.append(
3139 EventKind::ReplanExhausted,
3140 None,
3141 Some(&proposal.id),
3142 [("attempts".to_string(), Value::from(attempt))].into(),
3143 );
3144 }
3145
3146 // Persist trajectory
3147 let outcome = if !aborted {
3148 if attempt > 0 {
3149 car_memgine::TrajectoryOutcome::ReplanSuccess
3150 } else {
3151 car_memgine::TrajectoryOutcome::Success
3152 }
3153 } else if attempt > 0 && !terminal_failure {
3154 car_memgine::TrajectoryOutcome::ReplanExhausted
3155 } else {
3156 car_memgine::TrajectoryOutcome::Failed
3157 };
3158 if let Some(err) = self.persist_trajectory(
3159 proposal,
3160 ¤t_proposal,
3161 &result,
3162 outcome,
3163 attempt,
3164 &state_before_map,
3165 ) {
3166 let mut log = self.log.lock().await;
3167 log.append(
3168 EventKind::ActionFailed,
3169 None,
3170 Some(&proposal.id),
3171 [(
3172 "trajectory_persist_error".to_string(),
3173 Value::from(err.as_str()),
3174 )]
3175 .into(),
3176 );
3177 }
3178
3179 return finalize_proposal_result(
3180 result,
3181 &proposal.id,
3182 &lineage,
3183 &accepted_proposal_preimages,
3184 );
3185 }
3186
3187 // Get replan callback (clone Arc, drop lock immediately)
3188 let callback = {
3189 let guard = self.replan_callback.lock().await;
3190 guard.clone()
3191 };
3192 let Some(callback) = callback else {
3193 // No callback registered — persist trajectory and return
3194 if let Some(err) = self.persist_trajectory(
3195 proposal,
3196 ¤t_proposal,
3197 &result,
3198 car_memgine::TrajectoryOutcome::Failed,
3199 attempt,
3200 &state_before_map,
3201 ) {
3202 let mut log = self.log.lock().await;
3203 log.append(
3204 EventKind::ActionFailed,
3205 None,
3206 Some(&proposal.id),
3207 [(
3208 "trajectory_persist_error".to_string(),
3209 Value::from(err.as_str()),
3210 )]
3211 .into(),
3212 );
3213 }
3214 return finalize_proposal_result(
3215 result,
3216 &proposal.id,
3217 &lineage,
3218 &accepted_proposal_preimages,
3219 );
3220 };
3221
3222 // Build the stable failure context once. Candidate rejection is a
3223 // planning-only loop below: it consumes a replan generation but
3224 // never returns to `execute_inner_with_cancel`, so a failed
3225 // proposal with external effects cannot be dispatched twice.
3226 let failed_actions: Vec<FailedActionSummary> = result
3227 .results
3228 .iter()
3229 .filter(|r| replan_triggers(&r.status) && !r.rolled_back)
3230 .map(|r| {
3231 let action = current_proposal
3232 .actions
3233 .iter()
3234 .find(|a| a.id == r.action_id);
3235 FailedActionSummary {
3236 action_id: r.action_id.clone(),
3237 tool: action.and_then(|a| a.tool.clone()),
3238 error: r.error.clone().unwrap_or_default(),
3239 parameters: action.map(|a| a.parameters.clone()).unwrap_or_default(),
3240 }
3241 })
3242 .collect();
3243
3244 let completed_action_ids: Vec<String> = result
3245 .results
3246 .iter()
3247 .filter(|r| r.rolled_back)
3248 .map(|r| r.action_id.clone())
3249 .collect();
3250
3251 'replan_candidates: loop {
3252 if attempt >= config.max_replans {
3253 let mut log = self.log.lock().await;
3254 log.append(
3255 EventKind::ReplanExhausted,
3256 None,
3257 Some(&proposal.id),
3258 [("attempts".to_string(), Value::from(attempt))].into(),
3259 );
3260 drop(log);
3261 if let Some(err) = self.persist_trajectory(
3262 proposal,
3263 ¤t_proposal,
3264 &result,
3265 car_memgine::TrajectoryOutcome::ReplanExhausted,
3266 attempt,
3267 &state_before_map,
3268 ) {
3269 self.log.lock().await.append(
3270 EventKind::ActionFailed,
3271 None,
3272 Some(&proposal.id),
3273 [("trajectory_persist_error".to_string(), Value::from(err))].into(),
3274 );
3275 }
3276 return finalize_proposal_result(
3277 result,
3278 &proposal.id,
3279 &lineage,
3280 &accepted_proposal_preimages,
3281 );
3282 }
3283
3284 let generation = attempt + 1;
3285 let ctx = ReplanContext {
3286 proposal_id: proposal.id.clone(),
3287 attempt: generation,
3288 failed_actions: failed_actions.clone(),
3289 completed_action_ids: completed_action_ids.clone(),
3290 // Under raw names as before, minus other tenants' keys: a
3291 // tenant proposal's replanner sees the shared keys its tool
3292 // calls run against and its own `tenant:<id>:` keys
3293 // (car#3121). Unscoped: the whole store.
3294 state_snapshot: self
3295 .state
3296 .versioned_snapshot_visible_to(scope.and_then(|s| s.tenant_id.as_deref()))
3297 .0,
3298 replans_remaining: config.max_replans.saturating_sub(generation),
3299 original_source: proposal.source.clone(),
3300 original_action_count: proposal.actions.len(),
3301 original_context: proposal.context.clone(),
3302 };
3303
3304 // Backoff delay between replan attempts
3305 if config.delay_ms > 0 {
3306 tokio::time::sleep(Duration::from_millis(config.delay_ms)).await;
3307 }
3308
3309 // Log replan attempt
3310 {
3311 let mut log = self.log.lock().await;
3312 log.append(
3313 EventKind::ReplanAttempted,
3314 None,
3315 Some(&proposal.id),
3316 [
3317 ("attempt".to_string(), Value::from(generation)),
3318 (
3319 "failed_count".to_string(),
3320 Value::from(ctx.failed_actions.len()),
3321 ),
3322 ]
3323 .into(),
3324 );
3325 // Deep-telemetry breadcrumbs (§3.5.1): record the fork the
3326 // harness took (replan vs accept vs abandon) and the
3327 // approach it discarded, so failure-mode diagnosis can see
3328 // the path *not* taken, not just the path taken.
3329 let rejected_ids: Vec<String> = ctx
3330 .failed_actions
3331 .iter()
3332 .map(|f| f.action_id.clone())
3333 .collect();
3334 log.append(
3335 EventKind::BranchDecision,
3336 None,
3337 Some(&proposal.id),
3338 [
3339 ("branch".to_string(), Value::from("replan")),
3340 (
3341 "reason".to_string(),
3342 Value::from("actions failed; requesting a revised plan"),
3343 ),
3344 ("attempt".to_string(), Value::from(generation)),
3345 ]
3346 .into(),
3347 );
3348 log.append(
3349 EventKind::AlternativeRejected,
3350 None,
3351 Some(&proposal.id),
3352 [
3353 (
3354 "alternative".to_string(),
3355 serde_json::to_value(&rejected_ids).unwrap_or_default(),
3356 ),
3357 (
3358 "reason".to_string(),
3359 Value::from("plan superseded by replan after action failure"),
3360 ),
3361 ]
3362 .into(),
3363 );
3364 }
3365
3366 // Call the model for a new plan
3367 match callback.replan(&ctx).await {
3368 Ok(new_proposal) => {
3369 attempt = generation;
3370 let (new_proposal_preimage, new_proposal_digest) =
3371 match proposal_journal_identity(&new_proposal) {
3372 Ok(identity) => identity,
3373 Err(error) => {
3374 lineage.push(ProposalLineageEntry {
3375 generation,
3376 proposal_id: new_proposal.id.clone(),
3377 proposal_digest: None,
3378 status: ProposalLineageStatus::Rejected,
3379 rejection_reason: Some(error.clone()),
3380 });
3381 self.log.lock().await.append(
3382 EventKind::ReplanRejected,
3383 None,
3384 Some(&proposal.id),
3385 [
3386 ("errors".to_string(), Value::from(error)),
3387 ("attempt".to_string(), Value::from(generation)),
3388 ]
3389 .into(),
3390 );
3391 continue 'replan_candidates;
3392 }
3393 };
3394 if required_replan_proposal_id
3395 .is_some_and(|required| new_proposal.id != required)
3396 {
3397 let reason = format!(
3398 "replan proposal id `{}` does not retain authenticated proposal id `{}`",
3399 new_proposal.id,
3400 required_replan_proposal_id.expect("checked above")
3401 );
3402 lineage.push(ProposalLineageEntry {
3403 generation,
3404 proposal_id: new_proposal.id.clone(),
3405 proposal_digest: Some(new_proposal_digest),
3406 status: ProposalLineageStatus::Rejected,
3407 rejection_reason: Some(reason.clone()),
3408 });
3409 self.log.lock().await.append(
3410 EventKind::ReplanRejected,
3411 None,
3412 Some(&proposal.id),
3413 [
3414 ("errors".to_string(), Value::from(reason)),
3415 ("attempt".to_string(), Value::from(generation)),
3416 ]
3417 .into(),
3418 );
3419 continue 'replan_candidates;
3420 }
3421 if let Err(error) = validate_proposal_action_ids(&new_proposal) {
3422 lineage.push(ProposalLineageEntry {
3423 generation,
3424 proposal_id: new_proposal.id.clone(),
3425 proposal_digest: Some(new_proposal_digest),
3426 status: ProposalLineageStatus::Rejected,
3427 rejection_reason: Some(error.clone()),
3428 });
3429 let mut log = self.log.lock().await;
3430 log.append(
3431 EventKind::ReplanRejected,
3432 None,
3433 Some(&proposal.id),
3434 [
3435 (
3436 "errors".to_string(),
3437 Value::from(format!("invalid proposal: {error}")),
3438 ),
3439 ("attempt".to_string(), Value::from(generation)),
3440 ]
3441 .into(),
3442 );
3443 continue 'replan_candidates;
3444 }
3445
3446 if let Err(error) = validate_proposal_retry_limits(&new_proposal) {
3447 lineage.push(ProposalLineageEntry {
3448 generation,
3449 proposal_id: new_proposal.id.clone(),
3450 proposal_digest: Some(new_proposal_digest),
3451 status: ProposalLineageStatus::Rejected,
3452 rejection_reason: Some(error.clone()),
3453 });
3454 self.log.lock().await.append(
3455 EventKind::ReplanRejected,
3456 None,
3457 Some(&proposal.id),
3458 [
3459 ("errors".to_string(), Value::from(error)),
3460 ("attempt".to_string(), Value::from(generation)),
3461 ]
3462 .into(),
3463 );
3464 continue 'replan_candidates;
3465 }
3466
3467 // Quality gate: verify replan proposal before executing
3468 if config.verify_before_execute {
3469 // The keys this proposal may observe, each action's
3470 // keys resolved in its own namespace, like
3471 // admission (car#3121).
3472 let (current_state, _) = self.state.versioned_snapshot_visible_to(
3473 scope.and_then(|s| s.tenant_id.as_deref()),
3474 );
3475 // Verify against the full registered schemas
3476 // (not just names) so a replan with a bad
3477 // parameter type / missing required field is
3478 // rejected here, per register_tool_schema's
3479 // contract (car-releases#56). The read guard is
3480 // held across the synchronous verify call.
3481 // Same helper the admission gate uses, so the two
3482 // verification points cannot disagree about what is
3483 // fatal. They previously did: this path blocked on the
3484 // loop heuristic and on state-dependent findings that
3485 // the gate treats as advisory, and it passed the tool
3486 // map through unconditionally — so an embedder with no
3487 // registered schemas burned its whole replan budget on
3488 // "unregistered tool" for every call.
3489 let blocking = {
3490 let tools_guard = self.tools.read().await;
3491 crate::verify_gate::blocking_errors(
3492 &new_proposal,
3493 Some(¤t_state),
3494 scope.and_then(|s| s.tenant_id.as_deref()),
3495 &tools_guard,
3496 100,
3497 )
3498 };
3499 if !blocking.is_empty() {
3500 let error_msgs: Vec<String> =
3501 blocking.iter().map(|i| i.message.clone()).collect();
3502 let reason = error_msgs.join("; ");
3503 lineage.push(ProposalLineageEntry {
3504 generation,
3505 proposal_id: new_proposal.id.clone(),
3506 proposal_digest: Some(new_proposal_digest),
3507 status: ProposalLineageStatus::Rejected,
3508 rejection_reason: Some(reason.clone()),
3509 });
3510 let mut log = self.log.lock().await;
3511 log.append(
3512 EventKind::ReplanRejected,
3513 None,
3514 Some(&proposal.id),
3515 [
3516 ("errors".to_string(), Value::from(reason)),
3517 ("attempt".to_string(), Value::from(generation)),
3518 ]
3519 .into(),
3520 );
3521 // Don't execute a broken replan — count as failed attempt
3522 continue 'replan_candidates;
3523 }
3524 }
3525
3526 // Transactional re-check of the replan against the now-
3527 // mutated shared state (neo review #1): a replan runs
3528 // after earlier actions changed state, so it's exactly
3529 // where a fresh stale-assumption/write conflict appears.
3530 // A Strict conflict rejects this replan attempt via the
3531 // same bounded ReplanRejected path (capped by
3532 // max_replans), never an infinite loop.
3533 let replan_blocked = self.transaction_precheck(&new_proposal, scope).await;
3534 if !replan_blocked.is_empty() {
3535 let reason = "transactional conflict with current state".to_string();
3536 lineage.push(ProposalLineageEntry {
3537 generation,
3538 proposal_id: new_proposal.id.clone(),
3539 proposal_digest: Some(new_proposal_digest),
3540 status: ProposalLineageStatus::Rejected,
3541 rejection_reason: Some(reason.clone()),
3542 });
3543 let mut log = self.log.lock().await;
3544 log.append(
3545 EventKind::ReplanRejected,
3546 None,
3547 Some(&proposal.id),
3548 [
3549 ("errors".to_string(), Value::from(reason)),
3550 ("attempt".to_string(), Value::from(generation)),
3551 ]
3552 .into(),
3553 );
3554 drop(log);
3555 continue 'replan_candidates;
3556 }
3557
3558 // Admission gates re-run on EVERY replanned proposal
3559 // (linus review C-2): the replan callback is exactly
3560 // where injected/adversarial content reshapes a plan,
3561 // so a proposal that was clean at first admission must
3562 // not smuggle a hazardous replan past the gates. Same
3563 // fail-closed contract as first admission — a hard
3564 // reject or any unapproved escalation rejects this
3565 // replan attempt (bounded by max_replans).
3566 let replan_admission = self
3567 .run_admission_gates(&new_proposal, session_id, scope)
3568 .await;
3569 if !replan_admission.admitted {
3570 let mut cleared = !replan_admission.hard_rejected
3571 && replan_admission.needs_approval();
3572 if cleared {
3573 for esc in &replan_admission.escalations {
3574 if self.admission_decision(&esc.fingerprint).await
3575 != Some(ApprovalDecision::Approved)
3576 {
3577 cleared = false;
3578 break;
3579 }
3580 }
3581 }
3582 if !cleared {
3583 let gate = replan_admission
3584 .deciding_gate
3585 .as_deref()
3586 .unwrap_or("admission");
3587 let why = replan_admission
3588 .reason
3589 .clone()
3590 .unwrap_or_else(|| "blocked by admission gate".to_string());
3591 let reason =
3592 format!("admission gate blocked replan (gate: {gate}; {why})");
3593 lineage.push(ProposalLineageEntry {
3594 generation,
3595 proposal_id: new_proposal.id.clone(),
3596 proposal_digest: Some(new_proposal_digest),
3597 status: ProposalLineageStatus::Rejected,
3598 rejection_reason: Some(reason.clone()),
3599 });
3600 let mut log = self.log.lock().await;
3601 log.append(
3602 EventKind::ReplanRejected,
3603 None,
3604 Some(&proposal.id),
3605 [
3606 ("errors".to_string(), Value::from(reason)),
3607 ("attempt".to_string(), Value::from(generation)),
3608 ]
3609 .into(),
3610 );
3611 drop(log);
3612 continue 'replan_candidates;
3613 }
3614 }
3615
3616 lineage.push(ProposalLineageEntry {
3617 generation,
3618 proposal_id: new_proposal.id.clone(),
3619 proposal_digest: Some(new_proposal_digest.clone()),
3620 status: ProposalLineageStatus::Accepted,
3621 rejection_reason: None,
3622 });
3623 accepted_proposal_preimages.push(AcceptedProposalPreimage {
3624 generation,
3625 proposal_digest: new_proposal_digest.clone(),
3626 proposal: new_proposal.clone(),
3627 });
3628
3629 // Log accepted proposal
3630 {
3631 let mut log = self.log.lock().await;
3632 log.append(
3633 EventKind::ReplanProposalReceived,
3634 None,
3635 Some(&proposal.id),
3636 [
3637 ("attempt".to_string(), Value::from(generation)),
3638 (
3639 "proposal_id".to_string(),
3640 Value::from(new_proposal.id.as_str()),
3641 ),
3642 (
3643 "proposal_digest".to_string(),
3644 Value::from(new_proposal_digest),
3645 ),
3646 ("proposal".to_string(), new_proposal_preimage),
3647 (
3648 "new_action_count".to_string(),
3649 Value::from(new_proposal.actions.len()),
3650 ),
3651 ]
3652 .into(),
3653 );
3654 }
3655 current_proposal = new_proposal;
3656 break 'replan_candidates;
3657 }
3658 Err(e) => {
3659 // Replan callback itself failed — log and return original failure
3660 let mut log = self.log.lock().await;
3661 log.append(
3662 EventKind::ReplanExhausted,
3663 None,
3664 Some(&proposal.id),
3665 [
3666 ("reason".to_string(), Value::from("callback_error")),
3667 ("error".to_string(), Value::from(e.as_str())),
3668 ("attempt".to_string(), Value::from(generation)),
3669 ]
3670 .into(),
3671 );
3672 if let Some(err) = self.persist_trajectory(
3673 proposal,
3674 ¤t_proposal,
3675 &result,
3676 car_memgine::TrajectoryOutcome::Failed,
3677 attempt,
3678 &state_before_map,
3679 ) {
3680 log.append(
3681 EventKind::ActionFailed,
3682 None,
3683 Some(&proposal.id),
3684 [(
3685 "trajectory_persist_error".to_string(),
3686 Value::from(err.as_str()),
3687 )]
3688 .into(),
3689 );
3690 }
3691 return finalize_proposal_result(
3692 result,
3693 &proposal.id,
3694 &lineage,
3695 &accepted_proposal_preimages,
3696 );
3697 }
3698 }
3699 }
3700 }
3701 }
3702
3703 /// Persist a trajectory to the store if configured.
3704 fn persist_trajectory(
3705 &self,
3706 proposal: &ActionProposal,
3707 current_proposal: &ActionProposal,
3708 result: &ProposalResult,
3709 outcome: car_memgine::TrajectoryOutcome,
3710 attempt: u32,
3711 state_before_map: &HashMap<String, HashMap<String, Value>>,
3712 ) -> Option<String> {
3713 let store = self.trajectory_store.as_ref()?;
3714
3715 let trace_events: Vec<car_memgine::TraceEvent> = result
3716 .results
3717 .iter()
3718 .map(|r| {
3719 let kind = match r.status {
3720 ActionStatus::Succeeded => "action_succeeded",
3721 ActionStatus::Failed => "action_failed",
3722 ActionStatus::Rejected => "action_rejected",
3723 ActionStatus::Skipped => "action_skipped",
3724 _ => "unknown",
3725 };
3726 let tool = current_proposal
3727 .actions
3728 .iter()
3729 .find(|a| a.id == r.action_id)
3730 .and_then(|a| a.tool.clone());
3731 let reward = match r.status {
3732 ActionStatus::Succeeded => Some(1.0),
3733 ActionStatus::Failed => Some(0.0),
3734 ActionStatus::Rejected => Some(0.0),
3735 ActionStatus::Skipped => None,
3736 _ => None,
3737 };
3738 car_memgine::TraceEvent {
3739 kind: kind.to_string(),
3740 action_id: Some(r.action_id.clone()),
3741 tool,
3742 data: r
3743 .error
3744 .as_ref()
3745 .map(|e| serde_json::json!({"error": e}))
3746 .unwrap_or(serde_json::json!({})),
3747 duration_ms: r.duration_ms,
3748 state_before: state_before_map.get(&r.action_id).cloned(),
3749 state_after: if !r.state_changes.is_empty() {
3750 Some(r.state_changes.clone())
3751 } else {
3752 None
3753 },
3754 reward,
3755 }
3756 })
3757 .collect();
3758
3759 let trajectory = car_memgine::Trajectory {
3760 proposal_id: proposal.id.clone(),
3761 source: proposal.source.clone(),
3762 action_count: current_proposal.actions.len(),
3763 events: trace_events,
3764 outcome,
3765 timestamp: chrono::Utc::now(),
3766 duration_ms: result.cost.total_duration_ms,
3767 replan_attempts: attempt,
3768 };
3769
3770 match store.append(&trajectory) {
3771 Ok(()) => None,
3772 Err(e) => Some(e.to_string()),
3773 }
3774 }
3775
3776 async fn rollback_failed_plan_candidate(
3777 &self,
3778 proposal: &ActionProposal,
3779 result: &mut ProposalResult,
3780 pre_plan_snapshot: &HashMap<String, Value>,
3781 pre_plan_transitions: usize,
3782 pre_plan_idempotency_keys: &std::collections::HashSet<String>,
3783 ) -> Result<(), String> {
3784 let executed_proposal = result.final_proposal.as_ref().unwrap_or(proposal);
3785 let executed_proposal_digest = result
3786 .accepted_proposal_preimages
3787 .last()
3788 .map(|accepted| accepted.proposal_digest.clone())
3789 .unwrap_or_else(|| {
3790 proposal_digest(executed_proposal)
3791 .expect("an executed planning candidate already passed proposal identity")
3792 });
3793 let observed_changes_by_action: std::collections::BTreeMap<String, HashMap<String, Value>> =
3794 result
3795 .results
3796 .iter()
3797 .filter(|action_result| {
3798 action_result.status == ActionStatus::Succeeded
3799 && !action_result.state_changes.is_empty()
3800 })
3801 .map(|action_result| {
3802 (
3803 action_result.action_id.clone(),
3804 action_result.state_changes.clone(),
3805 )
3806 })
3807 .collect();
3808 let affected_actions: Vec<String> = observed_changes_by_action.keys().cloned().collect();
3809
3810 // The planning transaction owns the StateStore proposal guard, so this
3811 // exact snapshot/count restore cannot erase another proposal's commit.
3812 let durability = match self
3813 .state
3814 .restore(pre_plan_snapshot.clone(), pre_plan_transitions)
3815 {
3816 Ok(durability) => durability,
3817 Err(error) => {
3818 let detail = record_rollback_durability_error(
3819 &mut result.results,
3820 ROLLBACK_DURABILITY_ERROR,
3821 &error.to_string(),
3822 );
3823 self.log.lock().await.append(
3824 EventKind::ActionFailed,
3825 None,
3826 Some(&proposal.id),
3827 [
3828 ("error".to_string(), Value::from(detail.clone())),
3829 ("attempted".to_string(), Value::from(true)),
3830 ("publication_succeeded".to_string(), Value::from(false)),
3831 ("rollback_succeeded".to_string(), Value::from(false)),
3832 ("stage".to_string(), Value::from("plan_fallback_rollback")),
3833 ]
3834 .into(),
3835 );
3836 return Err(detail);
3837 }
3838 };
3839 let durability_error = match &durability {
3840 RestoreDurability::Durable => None,
3841 RestoreDurability::DurabilityUnknown { error } => Some(error.clone()),
3842 };
3843
3844 {
3845 let mut log = self.log.lock().await;
3846 log.append(
3847 EventKind::StateSnapshot,
3848 None,
3849 Some(&proposal.id),
3850 [(
3851 "state".to_string(),
3852 serde_json::to_value(pre_plan_snapshot).unwrap_or_default(),
3853 )]
3854 .into(),
3855 );
3856 log.append(
3857 EventKind::StateRollback,
3858 None,
3859 Some(&proposal.id),
3860 [
3861 (
3862 "rolled_back_to".to_string(),
3863 Value::from("pre-plan snapshot"),
3864 ),
3865 (
3866 "affected_actions".to_string(),
3867 serde_json::to_value(&affected_actions).unwrap_or_default(),
3868 ),
3869 (
3870 "rolled_back_changes".to_string(),
3871 serde_json::to_value(&observed_changes_by_action).unwrap_or_default(),
3872 ),
3873 (
3874 "changes_semantics".to_string(),
3875 Value::from("rolled_back_failed_planning_candidate"),
3876 ),
3877 (
3878 "proposal_digest".to_string(),
3879 Value::from(executed_proposal_digest),
3880 ),
3881 ("attempted".to_string(), Value::from(true)),
3882 (
3883 "durability_unknown".to_string(),
3884 Value::from(durability_error.is_some()),
3885 ),
3886 ("publication_succeeded".to_string(), Value::from(true)),
3887 ("rollback_succeeded".to_string(), Value::from(true)),
3888 ("stage".to_string(), Value::from("plan_fallback_rollback")),
3889 ]
3890 .into(),
3891 );
3892 }
3893
3894 // Only invalidate entries introduced by this planning transaction.
3895 // Pre-existing keys may have been read by a candidate, but their
3896 // committed result predates the pre-plan snapshot and must survive.
3897 let mut invalidated = std::collections::BTreeSet::new();
3898 {
3899 let mut cache = self.idempotency_cache.lock().await;
3900 for action_result in &result.results {
3901 if action_result.status != ActionStatus::Succeeded {
3902 continue;
3903 }
3904 let Some(action) = executed_proposal
3905 .actions
3906 .iter()
3907 .find(|action| action.id == action_result.action_id)
3908 else {
3909 continue;
3910 };
3911 if !action.idempotent || action.invocation_mode.is_detached() {
3912 continue;
3913 }
3914 let key = idempotency_key(action, None);
3915 if !pre_plan_idempotency_keys.contains(&key) && cache.remove(&key).is_some() {
3916 invalidated.insert(key);
3917 }
3918 }
3919 }
3920 for key in invalidated {
3921 self.journal_idempotency(&key, None).await;
3922 }
3923
3924 // Same reasoning as the abort path above: a rejected planning
3925 // candidate loses its state effects, not the record of what ran. This
3926 // result is observable — `plan_and_execute` returns it directly on a
3927 // rollback durability error, and otherwise keeps it as
3928 // `first_failure` (car#1157). `rolled_back` is the machine-readable
3929 // fact; the warning remains descriptive and must never drive logic.
3930 for action_result in &mut result.results {
3931 if action_result.status == ActionStatus::Succeeded {
3932 action_result.rolled_back = true;
3933 action_result.error = Some(PLAN_FALLBACK_ROLLBACK_WARNING.to_string());
3934 action_result.state_changes.clear();
3935 }
3936 }
3937 if let Some(error) = durability_error {
3938 return Err(record_rollback_durability_error(
3939 &mut result.results,
3940 ROLLBACK_DURABILITY_UNKNOWN,
3941 &error,
3942 ));
3943 }
3944 Ok(())
3945 }
3946
3947 /// Score N candidate proposals, execute the best valid one, fall back to
3948 /// next-best on failure. Combines car-planner scoring with engine execution.
3949 ///
3950 /// Returns the result from whichever proposal was executed (best or fallback).
3951 /// If all candidates fail verification, returns an error result for the first.
3952 pub async fn plan_and_execute(
3953 &self,
3954 candidates: &[ActionProposal],
3955 planner_config: Option<car_planner::PlannerConfig>,
3956 feedback: Option<&car_planner::ToolFeedback>,
3957 ) -> ProposalResult {
3958 if candidates.is_empty() {
3959 return ProposalResult::new("empty", vec![], car_ir::CostSummary::default());
3960 }
3961
3962 // One transaction guard covers ranking's state snapshot, every
3963 // candidate attempt and rollback, and the accepted candidate's commit.
3964 // Candidate execution calls the already-guarded path to avoid recursive
3965 // acquisition of the non-reentrant mutex.
3966 let _proposal_execution = self.state.lock_proposal_execution().await;
3967
3968 // Score all candidates
3969 let planner = car_planner::Planner::new(planner_config.unwrap_or_default());
3970 let tools_guard = self.tools.read().await;
3971 let tool_names: std::collections::HashSet<String> = tools_guard.keys().cloned().collect();
3972 drop(tools_guard);
3973
3974 let pre_plan_snapshot = self.state.snapshot();
3975 let pre_plan_transitions = self.state.transition_count();
3976 let pre_plan_idempotency_keys: std::collections::HashSet<String> = self
3977 .idempotency_cache
3978 .lock()
3979 .await
3980 .keys()
3981 .cloned()
3982 .collect();
3983 let ranked = planner.rank_with_feedback(
3984 candidates,
3985 Some(&pre_plan_snapshot),
3986 Some(&tool_names),
3987 feedback,
3988 );
3989
3990 // Try each valid candidate in score order
3991 let mut first_failure: Option<ProposalResult> = None;
3992 for scored in &ranked {
3993 if !scored.valid {
3994 continue;
3995 }
3996
3997 let proposal = &candidates[scored.index];
3998 let cancel = tokio_util::sync::CancellationToken::new();
3999 let mut result = self
4000 .execute_with_optional_session_already_guarded(proposal, None, None, None, &cancel)
4001 .await;
4002
4003 if result.all_succeeded() {
4004 return result;
4005 }
4006
4007 if self
4008 .rollback_failed_plan_candidate(
4009 proposal,
4010 &mut result,
4011 &pre_plan_snapshot,
4012 pre_plan_transitions,
4013 &pre_plan_idempotency_keys,
4014 )
4015 .await
4016 .is_err()
4017 {
4018 return result;
4019 }
4020
4021 tracing::info!(
4022 proposal_id = %proposal.id,
4023 score = scored.score,
4024 "plan_and_execute: proposal failed, trying next candidate"
4025 );
4026
4027 if first_failure.is_none() {
4028 first_failure = Some(result);
4029 }
4030 }
4031
4032 // Return the first failure result (don't re-execute — avoids duplicate side effects)
4033 first_failure.unwrap_or_else(|| {
4034 ProposalResult::for_proposal(&candidates[0], vec![], car_ir::CostSummary::default())
4035 })
4036 }
4037
4038 /// Execute a single proposal through the runtime loop (no replanning).
4039 /// Returns (result, state_before_map) where state_before_map has per-action snapshots.
4040 ///
4041 /// `session_id`, when `Some`, scopes per-action policy validation
4042 /// to the named session in addition to global policies. Both
4043 /// layers must pass for an action to run; the session layer cannot
4044 /// loosen what global denies.
4045 async fn execute_inner_with_cancel(
4046 &self,
4047 proposal: &ActionProposal,
4048 cancel: Option<&tokio_util::sync::CancellationToken>,
4049 session_id: Option<&str>,
4050 scope: Option<&crate::scope::RuntimeScope>,
4051 ) -> (ProposalResult, HashMap<String, HashMap<String, Value>>) {
4052 // Identity is established before CAR admits the proposal or emits any
4053 // action transition. In particular, an I-JSON-incompatible number may
4054 // not produce a journal row whose mandatory JCS digest is missing.
4055 let (proposal_preimage, proposal_digest) = match proposal_journal_identity(proposal) {
4056 Ok(identity) => identity,
4057 Err(error) => {
4058 let results = proposal
4059 .actions
4060 .iter()
4061 .map(|action| rejected_result(&action.id, error.clone()))
4062 .collect();
4063 self.log.lock().await.append(
4064 EventKind::StateRollback,
4065 None,
4066 Some(&proposal.id),
4067 proposal_rejection_boundary_data(proposal, None, &error),
4068 );
4069 return (
4070 ProposalResult::for_proposal(proposal, results, CostSummary::default()),
4071 HashMap::new(),
4072 );
4073 }
4074 };
4075
4076 // Whether validator/policy/capability rejections should count toward
4077 // the abort-and-replan path (default false). Read once up front so the
4078 // per-action loop doesn't take the replan_config lock repeatedly (and
4079 // never inside join_all). Independent lock from log/tools/policies/
4080 // capabilities, so no lock-ordering conflict.
4081 let replan_on_rejected = self.replan_config.read().await.replan_on_rejected;
4082
4083 // Generate trace_id for this proposal execution
4084 let trace_id = Uuid::new_v4().to_string();
4085
4086 // Begin root span for proposal execution
4087 let root_span_id = {
4088 let mut log = self.log.lock().await;
4089 log.begin_span(
4090 "proposal.execute",
4091 &trace_id,
4092 None,
4093 [("proposal_id".to_string(), Value::from(proposal.id.as_str()))].into(),
4094 )
4095 };
4096
4097 // Log proposal received
4098 {
4099 let mut log = self.log.lock().await;
4100 log.append(
4101 EventKind::ProposalReceived,
4102 None,
4103 Some(&proposal.id),
4104 [
4105 ("source".to_string(), Value::from(proposal.source.as_str())),
4106 (
4107 "action_count".to_string(),
4108 Value::from(proposal.actions.len()),
4109 ),
4110 ("proposal".to_string(), proposal_preimage.clone()),
4111 (
4112 "proposal_digest".to_string(),
4113 Value::from(proposal_digest.clone()),
4114 ),
4115 ]
4116 .into(),
4117 );
4118 }
4119
4120 // Capability check: max_actions budget for entire proposal
4121 {
4122 let caps = self.capabilities.read().await;
4123 if let Some(ref cap) = *caps {
4124 if !cap.actions_within_budget(proposal.actions.len() as u32) {
4125 let reason = format!(
4126 "capability denied: proposal has {} actions, max allowed is {:?}",
4127 proposal.actions.len(),
4128 cap.max_actions
4129 );
4130 let mut action_results = Vec::new();
4131 let mut log = self.log.lock().await;
4132 for action in &proposal.actions {
4133 log.append(
4134 EventKind::ActionRejected,
4135 Some(&action.id),
4136 Some(&proposal.id),
4137 action_outcome_data("proposal_capability", &reason, false, None),
4138 );
4139 action_results.push(rejected_result(&action.id, reason.clone()));
4140 }
4141 log.append(
4142 EventKind::StateRollback,
4143 None,
4144 Some(&proposal.id),
4145 proposal_rejection_boundary_data(proposal, Some(&proposal_digest), &reason),
4146 );
4147 drop(log);
4148 return (
4149 ProposalResult::for_proposal(
4150 proposal,
4151 action_results,
4152 CostSummary::default(),
4153 ),
4154 HashMap::new(),
4155 );
4156 }
4157 }
4158 }
4159
4160 // Snapshot for rollback. When the proposal carries a tenant scope,
4161 // snapshot only that tenant's namespace so a rollback can't clobber
4162 // concurrent tenants' state (EPIC E / E2). No tenant → unchanged
4163 // full snapshot.
4164 let rollback_tenant: Option<&str> = scope.and_then(|s| s.tenant_id.as_deref());
4165 let snapshot = match rollback_tenant {
4166 Some(_) => self.state.snapshot_scoped(rollback_tenant),
4167 None => self.state.snapshot(),
4168 };
4169 let transition_count = self.state.transition_count();
4170
4171 let mut results: Vec<ActionResult> = Vec::new();
4172 // Per-action state snapshots captured before execution (for TraceEvent.state_before).
4173 let mut state_before_map: HashMap<String, HashMap<String, Value>> = HashMap::new();
4174 let mut aborted = false;
4175 let mut budget_exceeded = false;
4176 let mut total_retries: u32 = 0;
4177
4178 // Running cost counters for budget enforcement
4179 let mut running_tool_calls: u32 = 0;
4180 let mut running_actions: u32 = 0;
4181 let mut running_duration_ms: f64 = 0.0;
4182
4183 // Snapshot the budget once
4184 let budget = self.cost_budget.read().await.clone();
4185
4186 // Build DAG
4187 let levels = build_dag(&proposal.actions);
4188
4189 let mut canceled = false;
4190 for level in &levels {
4191 // Cooperative cancellation check at the level boundary.
4192 // Actions already in flight aren't interrupted (we can't
4193 // safely cancel a tool call dispatched to a user-provided
4194 // executor), but every action that hadn't started runs
4195 // is recorded as canceled with a clear reason.
4196 if !canceled {
4197 if let Some(token) = cancel {
4198 if token.is_cancelled() {
4199 canceled = true;
4200 }
4201 }
4202 }
4203 if canceled {
4204 for &idx in level {
4205 let action = &proposal.actions[idx];
4206 let reason = format!("{CANCELED_PREFIX}cancellation requested by caller");
4207 self.log.lock().await.append(
4208 EventKind::ActionSkipped,
4209 Some(&action.id),
4210 Some(&proposal.id),
4211 action_outcome_data("cancellation", &reason, false, None),
4212 );
4213 results.push(canceled_result(
4214 &action.id,
4215 "cancellation requested by caller",
4216 ));
4217 }
4218 continue;
4219 }
4220 if aborted || budget_exceeded {
4221 let skip_reason = if budget_exceeded {
4222 "cost budget exceeded"
4223 } else {
4224 "skipped due to earlier abort"
4225 };
4226 for &idx in level {
4227 let action = &proposal.actions[idx];
4228 let stage = if budget_exceeded {
4229 "cost_budget"
4230 } else {
4231 "dependency_abort"
4232 };
4233 self.log.lock().await.append(
4234 EventKind::ActionSkipped,
4235 Some(&action.id),
4236 Some(&proposal.id),
4237 action_outcome_data(stage, skip_reason, false, None),
4238 );
4239 results.push(skipped_result(&action.id, skip_reason));
4240 }
4241 continue;
4242 }
4243
4244 if level_runs_sequentially(level, &proposal.actions) {
4245 // Sequential execution
4246 for &idx in level {
4247 if aborted || budget_exceeded {
4248 let skip_reason = if budget_exceeded {
4249 "cost budget exceeded"
4250 } else {
4251 "skipped due to abort"
4252 };
4253 let action = &proposal.actions[idx];
4254 let stage = if budget_exceeded {
4255 "cost_budget"
4256 } else {
4257 "dependency_abort"
4258 };
4259 self.log.lock().await.append(
4260 EventKind::ActionSkipped,
4261 Some(&action.id),
4262 Some(&proposal.id),
4263 action_outcome_data(stage, skip_reason, false, None),
4264 );
4265 results.push(skipped_result(&action.id, skip_reason));
4266 continue;
4267 }
4268
4269 // Budget check before execution
4270 if let Some(ref b) = budget {
4271 if let Some(max) = b.max_actions {
4272 if running_actions >= max {
4273 budget_exceeded = true;
4274 self.log.lock().await.append(
4275 EventKind::ActionSkipped,
4276 Some(&proposal.actions[idx].id),
4277 Some(&proposal.id),
4278 action_outcome_data(
4279 "cost_budget",
4280 "cost budget exceeded",
4281 false,
4282 None,
4283 ),
4284 );
4285 results.push(skipped_result(
4286 &proposal.actions[idx].id,
4287 "cost budget exceeded",
4288 ));
4289 continue;
4290 }
4291 }
4292 if let Some(max) = b.max_tool_calls {
4293 if proposal.actions[idx].action_type == ActionType::ToolCall
4294 && running_tool_calls >= max
4295 {
4296 budget_exceeded = true;
4297 self.log.lock().await.append(
4298 EventKind::ActionSkipped,
4299 Some(&proposal.actions[idx].id),
4300 Some(&proposal.id),
4301 action_outcome_data(
4302 "cost_budget",
4303 "cost budget exceeded",
4304 false,
4305 None,
4306 ),
4307 );
4308 results.push(skipped_result(
4309 &proposal.actions[idx].id,
4310 "cost budget exceeded",
4311 ));
4312 continue;
4313 }
4314 }
4315 if let Some(max) = b.max_duration_ms {
4316 if running_duration_ms >= max {
4317 budget_exceeded = true;
4318 self.log.lock().await.append(
4319 EventKind::ActionSkipped,
4320 Some(&proposal.actions[idx].id),
4321 Some(&proposal.id),
4322 action_outcome_data(
4323 "cost_budget",
4324 "cost budget exceeded",
4325 false,
4326 None,
4327 ),
4328 );
4329 results.push(skipped_result(
4330 &proposal.actions[idx].id,
4331 "cost budget exceeded",
4332 ));
4333 continue;
4334 }
4335 }
4336 }
4337
4338 state_before_map.insert(
4339 proposal.actions[idx].id.clone(),
4340 snapshot_relevant_keys(
4341 &*action_state_view(
4342 &self.state,
4343 &proposal.actions[idx],
4344 rollback_tenant,
4345 ),
4346 &proposal.actions[idx],
4347 ),
4348 );
4349 let (ar, action_retries) = self
4350 .process_action(
4351 &proposal.actions[idx],
4352 &proposal.id,
4353 &trace_id,
4354 &root_span_id,
4355 session_id,
4356 scope,
4357 )
4358 .await;
4359 total_retries += action_retries;
4360
4361 // Update running counters
4362 if ar.status == ActionStatus::Succeeded
4363 && proposal.actions[idx].action_type == ActionType::ToolCall
4364 {
4365 running_tool_calls += 1;
4366 }
4367 if ar.status != ActionStatus::Skipped {
4368 running_actions += 1;
4369 }
4370 if let Some(d) = ar.duration_ms {
4371 running_duration_ms += d;
4372 }
4373
4374 if ar.terminal
4375 || requires_integrity_rollback(&ar)
4376 || ((ar.status == ActionStatus::Failed
4377 || (replan_on_rejected && ar.status == ActionStatus::Rejected))
4378 && proposal.actions[idx].failure_behavior == FailureBehavior::Abort)
4379 {
4380 aborted = true;
4381 }
4382 results.push(ar);
4383 }
4384 } else {
4385 // Concurrent execution via futures::join_all
4386 // Snapshot only relevant keys per action (all see same pre-level state)
4387 for &idx in level {
4388 state_before_map.insert(
4389 proposal.actions[idx].id.clone(),
4390 snapshot_relevant_keys(
4391 &*action_state_view(
4392 &self.state,
4393 &proposal.actions[idx],
4394 rollback_tenant,
4395 ),
4396 &proposal.actions[idx],
4397 ),
4398 );
4399 }
4400 let futs: Vec<_> = level
4401 .iter()
4402 .map(|&idx| {
4403 self.process_action(
4404 &proposal.actions[idx],
4405 &proposal.id,
4406 &trace_id,
4407 &root_span_id,
4408 session_id,
4409 scope,
4410 )
4411 })
4412 .collect();
4413 let level_results = futures::future::join_all(futs).await;
4414
4415 for (i, (ar, action_retries)) in level_results.into_iter().enumerate() {
4416 let idx = level[i];
4417 total_retries += action_retries;
4418 if ar.status == ActionStatus::Succeeded
4419 && proposal.actions[idx].action_type == ActionType::ToolCall
4420 {
4421 running_tool_calls += 1;
4422 }
4423 if ar.status != ActionStatus::Skipped {
4424 running_actions += 1;
4425 }
4426 if let Some(d) = ar.duration_ms {
4427 running_duration_ms += d;
4428 }
4429 if ar.terminal || requires_integrity_rollback(&ar) {
4430 aborted = true;
4431 }
4432 results.push(ar);
4433 }
4434 }
4435 }
4436
4437 let observed_changes_by_action: std::collections::BTreeMap<String, HashMap<String, Value>> =
4438 proposal
4439 .actions
4440 .iter()
4441 .filter_map(|action| {
4442 results
4443 .iter()
4444 .find(|result| {
4445 result.action_id == action.id
4446 && result.status == ActionStatus::Succeeded
4447 && !result.state_changes.is_empty()
4448 })
4449 .map(|result| (action.id.clone(), result.state_changes.clone()))
4450 })
4451 .collect();
4452 let affected_actions: Vec<String> = observed_changes_by_action.keys().cloned().collect();
4453
4454 // Handle rollback. Per-action StateChanged rows are provisional until
4455 // this proposal-level transaction boundary is durably recorded.
4456 if aborted {
4457 // Tenant-scoped rollback when the proposal is tenant-scoped
4458 // (EPIC E / E2), else the full restore.
4459 let rollback = match rollback_tenant {
4460 Some(_) => {
4461 self.state
4462 .restore_scoped(rollback_tenant, snapshot.clone(), transition_count)
4463 }
4464 None => self.state.restore(snapshot.clone(), transition_count),
4465 };
4466
4467 let rollback_durability = match rollback {
4468 Ok(durability) => Some(durability),
4469 Err(error) => {
4470 let detail = record_rollback_durability_error(
4471 &mut results,
4472 ROLLBACK_DURABILITY_ERROR,
4473 &error.to_string(),
4474 );
4475 self.log.lock().await.append(
4476 EventKind::ActionFailed,
4477 None,
4478 Some(&proposal.id),
4479 [
4480 ("error".to_string(), Value::from(detail)),
4481 ("attempted".to_string(), Value::from(true)),
4482 ("publication_succeeded".to_string(), Value::from(false)),
4483 ("in_memory_state_preserved".to_string(), Value::from(true)),
4484 (
4485 "idempotency_entries_preserved".to_string(),
4486 Value::from(true),
4487 ),
4488 ("rollback_succeeded".to_string(), Value::from(false)),
4489 ("stage".to_string(), Value::from("proposal_rollback")),
4490 ]
4491 .into(),
4492 );
4493 None
4494 }
4495 };
4496
4497 if let Some(durability) = rollback_durability {
4498 let durability_error = match &durability {
4499 RestoreDurability::Durable => None,
4500 RestoreDurability::DurabilityUnknown { error } => Some(error.clone()),
4501 };
4502 if let Some(error) = &durability_error {
4503 record_rollback_durability_error(
4504 &mut results,
4505 ROLLBACK_DURABILITY_UNKNOWN,
4506 error,
4507 );
4508 }
4509 let mut log = self.log.lock().await;
4510 log.append(
4511 EventKind::StateSnapshot,
4512 None,
4513 Some(&proposal.id),
4514 [(
4515 "state".to_string(),
4516 serde_json::to_value(&snapshot).unwrap_or_default(),
4517 )]
4518 .into(),
4519 );
4520 log.append(
4521 EventKind::StateRollback,
4522 None,
4523 Some(&proposal.id),
4524 [
4525 (
4526 "rolled_back_to".to_string(),
4527 Value::from("pre-proposal snapshot"),
4528 ),
4529 (
4530 "affected_actions".to_string(),
4531 serde_json::to_value(&affected_actions).unwrap_or_default(),
4532 ),
4533 (
4534 "rolled_back_changes".to_string(),
4535 serde_json::to_value(&observed_changes_by_action).unwrap_or_default(),
4536 ),
4537 (
4538 "changes_semantics".to_string(),
4539 Value::from("rolled_back_provisional_state_mutations"),
4540 ),
4541 (
4542 "proposal_digest".to_string(),
4543 Value::from(proposal_digest.clone()),
4544 ),
4545 ("attempted".to_string(), Value::from(true)),
4546 (
4547 "durability_unknown".to_string(),
4548 Value::from(durability_error.is_some()),
4549 ),
4550 ("publication_succeeded".to_string(), Value::from(true)),
4551 ("rollback_succeeded".to_string(), Value::from(true)),
4552 ("stage".to_string(), Value::from("proposal_rollback")),
4553 ]
4554 .into(),
4555 );
4556 drop(log);
4557
4558 // Clear idempotency cache for rolled-back actions only after
4559 // the state journal has durably accepted the rollback.
4560 let mut invalidated: Vec<String> = Vec::new();
4561 {
4562 let mut cache = self.idempotency_cache.lock().await;
4563 for r in &results {
4564 if r.status == ActionStatus::Succeeded {
4565 for action in &proposal.actions {
4566 if action.id == r.action_id && action.idempotent {
4567 let key = idempotency_key(action, scope);
4568 cache.remove(&key);
4569 invalidated.push(key);
4570 }
4571 }
4572 }
4573 }
4574 }
4575 // Record tombstones so the durable journal (C3) doesn't
4576 // resurrect a rolled-back result on reload.
4577 for key in &invalidated {
4578 self.journal_idempotency(key, None).await;
4579 }
4580 // The proposal did not commit, so no action's state changes
4581 // survive. Each action's STATUS still records whether that
4582 // action executed; `rolled_back` independently records whether
4583 // that successful execution preceded this rollback. Folding
4584 // the second fact into ActionStatus erased which action
4585 // aborted the proposal and marked `failed` precisely the
4586 // actions whose external effects DID happen and may remain.
4587 // The warning is human-readable context, never control flow
4588 // (car#1157).
4589 for result in &mut results {
4590 if result.status == ActionStatus::Succeeded {
4591 result.rolled_back = true;
4592 result.error = Some(ROLLBACK_WARNING.to_string());
4593 result.state_changes.clear();
4594 }
4595 }
4596 }
4597 } else {
4598 self.log.lock().await.append(
4599 EventKind::StateCommitted,
4600 None,
4601 Some(&proposal.id),
4602 [
4603 (
4604 "affected_actions".to_string(),
4605 serde_json::to_value(&affected_actions).unwrap_or_default(),
4606 ),
4607 (
4608 "committed_changes".to_string(),
4609 serde_json::to_value(&observed_changes_by_action).unwrap_or_default(),
4610 ),
4611 (
4612 "changes_semantics".to_string(),
4613 Value::from("committed_provisional_state_mutations"),
4614 ),
4615 ("proposal_digest".to_string(), Value::from(proposal_digest)),
4616 ("attempted".to_string(), Value::from(true)),
4617 ("stage".to_string(), Value::from("proposal_commit")),
4618 ]
4619 .into(),
4620 );
4621 }
4622
4623 // Sort results to match original action order
4624 let action_order: HashMap<String, usize> = proposal
4625 .actions
4626 .iter()
4627 .enumerate()
4628 .map(|(i, a)| (a.id.clone(), i))
4629 .collect();
4630 results.sort_by_key(|r| {
4631 action_order
4632 .get(&r.action_id)
4633 .copied()
4634 .unwrap_or(usize::MAX)
4635 });
4636
4637 // Compute cost summary from results
4638 let mut cost = CostSummary::default();
4639 for r in &results {
4640 let action = action_order
4641 .get(&r.action_id)
4642 .and_then(|&i| proposal.actions.get(i));
4643 match r.status {
4644 ActionStatus::Succeeded => {
4645 cost.actions_executed += 1;
4646 if let Some(a) = action {
4647 if a.action_type == ActionType::ToolCall {
4648 cost.tool_calls += 1;
4649 }
4650 }
4651 }
4652 // A failed action ran — the tool was invoked and errored — so it
4653 // belongs in `actions_executed`. A rejected one was blocked by
4654 // the validator or a policy and never started; counting it as
4655 // executed reported work that never happened (car#624).
4656 ActionStatus::Failed => {
4657 cost.actions_executed += 1;
4658 }
4659 ActionStatus::Rejected => {
4660 cost.actions_rejected += 1;
4661 }
4662 ActionStatus::Skipped => {
4663 cost.actions_skipped += 1;
4664 }
4665 _ => {}
4666 }
4667 if let Some(d) = r.duration_ms {
4668 cost.total_duration_ms += d;
4669 }
4670 }
4671
4672 // Set retries from inline counter
4673 cost.retries = total_retries;
4674
4675 // End root span — Ok if no abort, Error if aborted
4676 {
4677 let span_status = if aborted {
4678 SpanStatus::Error
4679 } else {
4680 SpanStatus::Ok
4681 };
4682 let mut log = self.log.lock().await;
4683 log.end_span(&root_span_id, span_status);
4684 }
4685
4686 let proposal_result = ProposalResult::for_proposal(proposal, results, cost);
4687
4688 // Post-execution: auto-distill skills from this execution trace
4689 if self.auto_distill {
4690 if let Some(ref memgine) = self.memgine {
4691 // Convert results to TraceEvents for distillation
4692 let trace_events: Vec<car_memgine::TraceEvent> = proposal_result
4693 .results
4694 .iter()
4695 .map(|r| {
4696 let kind = match r.status {
4697 ActionStatus::Succeeded => "action_succeeded",
4698 ActionStatus::Failed => "action_failed",
4699 ActionStatus::Rejected => "action_rejected",
4700 ActionStatus::Skipped => "action_skipped",
4701 _ => "unknown",
4702 };
4703 // Find the matching action to get the tool name
4704 let tool = proposal
4705 .actions
4706 .iter()
4707 .find(|a| a.id == r.action_id)
4708 .and_then(|a| a.tool.clone());
4709 let mut data = serde_json::Map::new();
4710 if let Some(ref e) = r.error {
4711 data.insert("error".into(), Value::from(e.as_str()));
4712 }
4713 if let Some(ref o) = r.output {
4714 data.insert("output".into(), o.clone());
4715 }
4716 car_memgine::TraceEvent {
4717 kind: kind.to_string(),
4718 action_id: Some(r.action_id.clone()),
4719 tool,
4720 data: Value::Object(data),
4721 duration_ms: r.duration_ms,
4722 reward: match r.status {
4723 ActionStatus::Succeeded => Some(1.0),
4724 ActionStatus::Failed | ActionStatus::Rejected => Some(0.0),
4725 _ => None,
4726 },
4727 ..Default::default()
4728 }
4729 })
4730 .collect();
4731
4732 let mut engine = memgine.lock().await;
4733 let skills = engine.distill_skills(&trace_events).await;
4734 if !skills.is_empty() {
4735 let count = skills.len();
4736 // Validation-gated ingest (SkillOpt-inspired). Distilled
4737 // skills are NOT trusted straight into the active pool; each
4738 // enters as a PROVISIONAL candidate on trial and must prove
4739 // itself (or beat its incumbent) before the promotion gate in
4740 // consolidate() makes it Active. Tenant is threaded through so
4741 // candidates are stamped for the calling tenant's namespace.
4742 let tenant = scope.and_then(|s| s.tenant_id.as_deref());
4743 let provisional = engine.ingest_provisional_candidates(&skills, tenant);
4744
4745 // Log the distillation event
4746 let mut log = self.log.lock().await;
4747 log.append(
4748 EventKind::SkillDistilled,
4749 None,
4750 Some(&proposal_result.proposal_id),
4751 [
4752 ("skills_count".to_string(), Value::from(count)),
4753 ("provisional_ingested".to_string(), Value::from(provisional)),
4754 (
4755 "skill_names".to_string(),
4756 Value::from(
4757 skills.iter().map(|s| s.name.as_str()).collect::<Vec<_>>(),
4758 ),
4759 ),
4760 ]
4761 .into(),
4762 );
4763
4764 // Check if any domains need evolution
4765 let threshold = engine.evolution_threshold();
4766 let domains = engine.domains_needing_evolution(threshold);
4767 for domain in &domains {
4768 // Collect failed events for this domain
4769 let failed: Vec<car_memgine::TraceEvent> = trace_events
4770 .iter()
4771 .filter(|e| {
4772 matches!(e.kind.as_str(), "action_failed" | "action_rejected")
4773 })
4774 .cloned()
4775 .collect();
4776 if !failed.is_empty() {
4777 let evolved = engine.evolve_skills(&failed, domain).await;
4778 if !evolved.is_empty() {
4779 log.append(
4780 EventKind::EvolutionTriggered,
4781 None,
4782 Some(&proposal_result.proposal_id),
4783 [
4784 ("domain".to_string(), Value::from(domain.as_str())),
4785 ("new_skills".to_string(), Value::from(evolved.len())),
4786 ]
4787 .into(),
4788 );
4789 }
4790 }
4791 }
4792 }
4793 }
4794 }
4795
4796 (proposal_result, state_before_map)
4797 }
4798
4799 /// Process a single action: capability → validate → policy → idempotency → execute.
4800 /// (Idempotency dedup runs AFTER the deny checks — review C3: a durable
4801 /// cached result must never outlive a tool's revocation.)
4802 /// Returns (ActionResult, retries_count).
4803 async fn process_action(
4804 &self,
4805 action: &Action,
4806 proposal_id: &str,
4807 trace_id: &str,
4808 parent_span_id: &str,
4809 session_id: Option<&str>,
4810 scope: Option<&crate::scope::RuntimeScope>,
4811 ) -> (ActionResult, u32) {
4812 // Derive action type name for span naming
4813 let action_type_name = serde_json::to_string(&action.action_type)
4814 .unwrap_or_default()
4815 .trim_matches('"')
4816 .to_string();
4817 let span_name = format!("action.{}", action_type_name);
4818
4819 // Begin child span for this action
4820 let action_span_id = {
4821 let mut attrs: HashMap<String, Value> = HashMap::new();
4822 attrs.insert("action_id".to_string(), Value::from(action.id.as_str()));
4823 if let Some(ref tool) = action.tool {
4824 attrs.insert("tool".to_string(), Value::from(tool.as_str()));
4825 }
4826 let mut log = self.log.lock().await;
4827 log.begin_span(&span_name, trace_id, Some(parent_span_id), attrs)
4828 };
4829
4830 // Execute the action pipeline and capture result
4831 let (result, retries) = self
4832 .process_action_inner(action, proposal_id, session_id, scope)
4833 .await;
4834
4835 // End action span based on result status
4836 let span_status = match result.status {
4837 ActionStatus::Succeeded => SpanStatus::Ok,
4838 ActionStatus::Failed | ActionStatus::Rejected => SpanStatus::Error,
4839 _ => SpanStatus::Unset,
4840 };
4841 {
4842 let mut log = self.log.lock().await;
4843 log.end_span(&action_span_id, span_status);
4844 }
4845
4846 (result, retries)
4847 }
4848
4849 /// Inner action processing: idempotency -> validate -> policy -> execute.
4850 /// Returns (ActionResult, retries_count).
4851 #[instrument(
4852 name = "action.process",
4853 skip_all,
4854 fields(
4855 action_id = %action.id,
4856 action_type = ?action.action_type,
4857 tool = action.tool.as_deref().unwrap_or("none"),
4858 )
4859 )]
4860 async fn process_action_inner(
4861 &self,
4862 action: &Action,
4863 proposal_id: &str,
4864 session_id: Option<&str>,
4865 scope: Option<&crate::scope::RuntimeScope>,
4866 ) -> (ActionResult, u32) {
4867 // Capability check
4868 {
4869 let caps = self.capabilities.read().await;
4870 if let Some(ref cap) = *caps {
4871 // Check tool capability for ToolCall actions
4872 if action.action_type == ActionType::ToolCall {
4873 if let Some(ref tool_name) = action.tool {
4874 if !cap.tool_allowed(tool_name) {
4875 let reason =
4876 format!("capability denied: tool '{}' not allowed", tool_name);
4877 let mut log = self.log.lock().await;
4878 log.append(
4879 EventKind::ActionRejected,
4880 Some(&action.id),
4881 Some(proposal_id),
4882 action_outcome_data("capability", &reason, false, None),
4883 );
4884 return (rejected_result(&action.id, reason), 0);
4885 }
4886 }
4887 }
4888
4889 // Check state key capability for StateWrite/StateRead actions
4890 if action.action_type == ActionType::StateWrite
4891 || action.action_type == ActionType::StateRead
4892 {
4893 if let Some(key) = action.parameters.get("key").and_then(|v| v.as_str()) {
4894 if !cap.state_key_allowed(key) {
4895 let reason =
4896 format!("capability denied: state key '{}' not allowed", key);
4897 let mut log = self.log.lock().await;
4898 log.append(
4899 EventKind::ActionRejected,
4900 Some(&action.id),
4901 Some(proposal_id),
4902 action_outcome_data("capability", &reason, false, None),
4903 );
4904 return (rejected_result(&action.id, reason), 0);
4905 }
4906 }
4907 }
4908 }
4909 }
4910
4911 // Validate
4912 let tools = self.tools.read().await;
4913 // Against the namespace the action executes in (car#3121): a state
4914 // action's preconditions and dependencies name keys in its tenant's
4915 // view, a tool call's name shared keys, and never another tenant's.
4916 let view = action_state_view(
4917 &self.state,
4918 action,
4919 scope.and_then(|s| s.tenant_id.as_deref()),
4920 );
4921 let validation = validate_action(action, &*view, &tools);
4922 drop(tools);
4923
4924 if !validation.valid() {
4925 let error = validation
4926 .errors
4927 .iter()
4928 .map(|e| e.reason.as_str())
4929 .collect::<Vec<_>>()
4930 .join("; ");
4931 let mut log = self.log.lock().await;
4932 log.append(
4933 EventKind::ActionRejected,
4934 Some(&action.id),
4935 Some(proposal_id),
4936 action_outcome_data("validation", &error, false, None),
4937 );
4938 return (rejected_result(&action.id, error), 0);
4939 }
4940
4941 // Policy check — global registry plus, when the proposal is
4942 // executed under a session, that session's registry. Both
4943 // layers must pass for the action to proceed; session
4944 // policies are additive deny rules — they can deny what
4945 // global allows but cannot allow what global denies.
4946 {
4947 let mut violations = {
4948 let policies = self.policies.read().await;
4949 policies.check(action, &self.state)
4950 };
4951 if let Some(sid) = session_id {
4952 // Snapshot the per-session engine handle out from under
4953 // the outer registry lock so the inner check holds
4954 // only the engine's own RwLock — preserves the
4955 // documented lock-ordering discipline.
4956 let session_engine = {
4957 let sessions = self.session_policies.read().await;
4958 sessions.get(sid).cloned()
4959 };
4960 if let Some(engine) = session_engine {
4961 let engine = engine.read().await;
4962 violations.extend(engine.check(action, &self.state));
4963 } else {
4964 let error = format!(
4965 "unknown session id '{sid}' — open one via Runtime::open_session before executing under a session"
4966 );
4967 // Unknown session id — refuse the action rather
4968 // than silently fall back to global-only. A
4969 // proposal submitted under a closed session
4970 // shouldn't run with looser rules than the caller
4971 // intended.
4972 let mut log = self.log.lock().await;
4973 log.append(
4974 EventKind::PolicyViolation,
4975 Some(&action.id),
4976 Some(proposal_id),
4977 action_outcome_data("policy", &error, false, None),
4978 );
4979 return (rejected_result(&action.id, error), 0);
4980 }
4981 }
4982 if !violations.is_empty() {
4983 let error = violations
4984 .iter()
4985 .map(|v| format!("policy '{}': {}", v.policy_name, v.reason))
4986 .collect::<Vec<_>>()
4987 .join("; ");
4988 let mut log = self.log.lock().await;
4989 log.append(
4990 EventKind::PolicyViolation,
4991 Some(&action.id),
4992 Some(proposal_id),
4993 action_outcome_data("policy", &error, false, None),
4994 );
4995 return (rejected_result(&action.id, error), 0);
4996 }
4997 }
4998
4999 // Idempotency check — deliberately AFTER capability + policy
5000 // (linus review C3): with the durable journal, a cached result
5001 // consulted first would let a tool that is denied TODAY serve
5002 // yesterday's cached result after every restart, forever. Deny
5003 // rules must win over dedup.
5004 if action.idempotent && !action.invocation_mode.is_detached() {
5005 let key = idempotency_key(action, scope);
5006 let cache = self.idempotency_cache.lock().await;
5007 if let Some(cached) = cache.get(&key) {
5008 let mut log = self.log.lock().await;
5009 log.append(
5010 EventKind::ActionDeduplicated,
5011 Some(&action.id),
5012 Some(proposal_id),
5013 [
5014 (
5015 "cached_action_id".to_string(),
5016 Value::from(cached.action_id.as_str()),
5017 ),
5018 ("attempted".to_string(), Value::from(false)),
5019 ("stage".to_string(), Value::from("idempotency_cache")),
5020 ]
5021 .into(),
5022 );
5023 return (
5024 ActionResult {
5025 action_id: action.id.clone(),
5026 status: cached.status.clone(),
5027 output: cached.output.clone(),
5028 error: cached.error.clone(),
5029 terminal: cached.terminal,
5030 // The cached action committed these mutations in an
5031 // earlier proposal. This proposal did not observe or
5032 // apply them, so its actual-change evidence is empty.
5033 state_changes: HashMap::new(),
5034 rolled_back: false,
5035 duration_ms: Some(0.0),
5036 timestamp: chrono::Utc::now(),
5037 },
5038 0,
5039 );
5040 }
5041 }
5042
5043 // Validated
5044 {
5045 let mut log = self.log.lock().await;
5046 log.append(
5047 EventKind::ActionValidated,
5048 Some(&action.id),
5049 Some(proposal_id),
5050 HashMap::new(),
5051 );
5052 }
5053
5054 // Execute with retry
5055 let (result, retries) = self
5056 .execute_with_retry(action, proposal_id, session_id, scope)
5057 .await;
5058
5059 // Cache idempotent results. Detached invocations are exempt
5060 // (linus review D4): their Succeeded output is a live tool
5061 // HANDLE, exactly as uncacheable here as in the result cache —
5062 // deduping would return a stale handle instead of starting the
5063 // tool, and journaling would replay a handle id that doesn't
5064 // exist in a fresh registry after restart.
5065 if action.idempotent
5066 && !action.invocation_mode.is_detached()
5067 && result.status == ActionStatus::Succeeded
5068 {
5069 let key = idempotency_key(action, scope);
5070 {
5071 let mut cache = self.idempotency_cache.lock().await;
5072 cache.insert(key.clone(), result.clone());
5073 }
5074 // Persist to the durable journal (C3) so the result survives a
5075 // restart and the action isn't re-executed.
5076 self.journal_idempotency(&key, Some(&result)).await;
5077 }
5078
5079 tracing::info!(
5080 status = ?result.status,
5081 duration_ms = result.duration_ms,
5082 "action completed"
5083 );
5084
5085 (result, retries)
5086 }
5087
5088 /// Execute with retry logic and timeout.
5089 /// Returns (ActionResult, retries_count).
5090 async fn execute_with_retry(
5091 &self,
5092 action: &Action,
5093 proposal_id: &str,
5094 session_id: Option<&str>,
5095 scope: Option<&crate::scope::RuntimeScope>,
5096 ) -> (ActionResult, u32) {
5097 // The harness config (Evolution Agent, §3.5), when installed, caps
5098 // the per-action retry budget and sets the inter-attempt backoff —
5099 // this is where an applied retry-config mutation actually takes
5100 // effect. `None` (the default) preserves prior behavior exactly.
5101 let hc = self.harness_config.read().await.clone();
5102 let retry_cap = hc.as_ref().map(|c| c.max_retries).unwrap_or(u32::MAX);
5103 let backoff_override = hc.as_ref().map(|c| c.retry_backoff_ms).filter(|&v| v > 0);
5104 let max_attempts = if action.failure_behavior == FailureBehavior::Retry {
5105 match action.max_retries.min(retry_cap).checked_add(1) {
5106 Some(max_attempts) => max_attempts,
5107 None => {
5108 let reason =
5109 format!("action '{}' retry attempt count overflows u32", action.id);
5110 self.log.lock().await.append(
5111 EventKind::ActionRejected,
5112 Some(&action.id),
5113 Some(proposal_id),
5114 action_outcome_data("retry_admission", &reason, false, None),
5115 );
5116 return (rejected_result(&action.id, reason), 0);
5117 }
5118 }
5119 } else {
5120 1
5121 };
5122
5123 let mut last_error: Option<String> = None;
5124 let mut retries: u32 = 0;
5125 let tool_source = self.tool_source_for_action(action).await;
5126 let params_digest = action_params_digest(action);
5127
5128 for attempt in 0..max_attempts {
5129 if attempt > 0 {
5130 retries += 1;
5131 // Base delay from the harness config when installed (an
5132 // applied retry-config mutation), else the built-in default.
5133 let base = backoff_override.unwrap_or(RETRY_BASE_DELAY_MS);
5134 let delay = base * RETRY_BACKOFF_FACTOR.pow(attempt - 1);
5135 tokio::time::sleep(Duration::from_millis(delay)).await;
5136 let mut log = self.log.lock().await;
5137 log.append(
5138 EventKind::ActionRetrying,
5139 Some(&action.id),
5140 Some(proposal_id),
5141 [("attempt".to_string(), Value::from(attempt + 1))].into(),
5142 );
5143 }
5144
5145 {
5146 let mut data: HashMap<String, Value> =
5147 [("attempt".to_string(), Value::from(attempt + 1))].into();
5148 insert_tool_event_provenance(&mut data, action, tool_source);
5149 let mut log = self.log.lock().await;
5150 log.append(
5151 EventKind::ActionExecuting,
5152 Some(&action.id),
5153 Some(proposal_id),
5154 data,
5155 );
5156 }
5157
5158 let start = std::time::Instant::now();
5159 let transitions_before = self.state.transition_count();
5160
5161 // Execute with optional timeout. Keep the timeout bit typed until
5162 // event classification rather than trying to recover it from the
5163 // human-readable error later.
5164 let (exec_result, engine_timeout) = if let Some(timeout_ms) = action.timeout_ms {
5165 match timeout(
5166 Duration::from_millis(timeout_ms),
5167 self.dispatch(action, session_id, scope, attempt + 1),
5168 )
5169 .await
5170 {
5171 Ok(result) => (result, false),
5172 Err(_) => (
5173 Err(ToolFailure::ordinary(format!(
5174 "action timed out after {}ms",
5175 timeout_ms
5176 ))),
5177 true,
5178 ),
5179 }
5180 } else {
5181 (
5182 self.dispatch(action, session_id, scope, attempt + 1).await,
5183 false,
5184 )
5185 };
5186
5187 let duration_ms = start.elapsed().as_secs_f64() * 1000.0;
5188
5189 match exec_result {
5190 Ok(output) => {
5191 if let Err(error) = car_inference::catalog_identity::canonical_json(&output) {
5192 let reason = format!(
5193 "tool output failed JCS/I-JSON validation after dispatch; external effect may have occurred and was not undone: {error}"
5194 );
5195 let mut data = action_outcome_data(
5196 "output_validation",
5197 &reason,
5198 true,
5199 Some(attempt + 1),
5200 );
5201 data.insert(
5202 "external_effect_status".to_string(),
5203 Value::from("may_have_occurred_not_undone"),
5204 );
5205 insert_tool_event_provenance(&mut data, action, tool_source);
5206 insert_action_outcome_signal(
5207 &mut data,
5208 action,
5209 ¶ms_digest,
5210 Some("validation"),
5211 );
5212 if action.tool.is_some() {
5213 data.insert("ok".to_string(), Value::from(false));
5214 }
5215 self.log.lock().await.append(
5216 EventKind::ActionFailed,
5217 Some(&action.id),
5218 Some(proposal_id),
5219 data,
5220 );
5221 return (
5222 ActionResult {
5223 action_id: action.id.clone(),
5224 status: ActionStatus::Failed,
5225 output: None,
5226 error: Some(reason),
5227 terminal: false,
5228 state_changes: HashMap::new(),
5229 rolled_back: false,
5230 duration_ms: Some(duration_ms),
5231 timestamp: chrono::Utc::now(),
5232 },
5233 retries,
5234 );
5235 }
5236
5237 // Snapshot the changes made by dispatch before applying
5238 // any result accounting. Deletion and set-to-null are
5239 // intentionally tagged differently on the authenticated
5240 // wire; a bare JSON null cannot represent both.
5241 let runtime_state_changes: HashMap<String, Value> = self
5242 .state
5243 .transitions_since(transitions_before)
5244 .into_iter()
5245 // Actions at the same DAG level execute concurrently,
5246 // so the shared transition tail can contain a sibling's
5247 // writes. StateTransition.action_id is assigned at the
5248 // mutation site and is the authoritative attribution
5249 // boundary for this per-action journal field.
5250 .filter(|transition| transition.action_id == action.id)
5251 .map(|transition| {
5252 (
5253 transition.key,
5254 StateMutation::from_new_value(transition.new_value).encode(),
5255 )
5256 })
5257 .collect();
5258
5259 // Declared expected effects are plan assertions only.
5260 // They must never be applied as if the runtime observed
5261 // them, nor merged into committed result evidence.
5262 let tenant_for_effects = scope.and_then(|s| s.tenant_id.as_deref());
5263 let state_changes = runtime_state_changes.clone();
5264
5265 // Record this result's provenance for the VIGIL intent
5266 // gate (`crate::taint`) — the only point in the system
5267 // that knows which SPECIFIC results were tainted. Only
5268 // on success: a failed action commits no effects, so it
5269 // changes no taint. No intent gate ⇒ no ledger ⇒ one
5270 // `Option` read and nothing else.
5271 let ledger = self.taint_ledger.read().await.clone();
5272 if let Some(ledger) = ledger {
5273 ledger
5274 .record_result(
5275 tenant_for_effects,
5276 action,
5277 state_changes.keys().cloned(),
5278 )
5279 .await;
5280 }
5281
5282 let mut log = self.log.lock().await;
5283 // Record the tool name + result count so the tool-receipt
5284 // verifier (A6) can project ground-truth receipts from the
5285 // log and catch tool-use hallucinations.
5286 let mut succ_data: HashMap<String, Value> = HashMap::new();
5287 succ_data.insert("attempt".to_string(), Value::from(attempt + 1));
5288 succ_data.insert("attempted".to_string(), Value::from(true));
5289 succ_data.insert("stage".to_string(), Value::from("dispatch"));
5290 insert_tool_event_provenance(&mut succ_data, action, tool_source);
5291 insert_action_outcome_signal(&mut succ_data, action, ¶ms_digest, None);
5292 if action.tool.is_some() {
5293 succ_data.insert("ok".to_string(), Value::from(true));
5294 if let Value::Array(arr) = &output {
5295 succ_data
5296 .insert("result_count".to_string(), Value::from(arr.len() as u64));
5297 }
5298 }
5299 // Record duration through the standardized metric path
5300 // (§3.5.1) so it feeds `EventLog::metrics_totals` via the
5301 // same contract as inference token metrics — one path,
5302 // not a coincidentally-matching `"duration_ms"` string.
5303 log.append_metered(
5304 EventKind::ActionSucceeded,
5305 Some(&action.id),
5306 Some(proposal_id),
5307 succ_data,
5308 car_eventlog::Metrics::latency(duration_ms),
5309 );
5310
5311 if !action.expected_effects.is_empty() || !runtime_state_changes.is_empty() {
5312 log.append(
5313 EventKind::StateChanged,
5314 Some(&action.id),
5315 Some(proposal_id),
5316 [
5317 (
5318 "declared_expected_effects".to_string(),
5319 serde_json::to_value(&action.expected_effects)
5320 .unwrap_or_default(),
5321 ),
5322 (
5323 "runtime_state_mutations".to_string(),
5324 serde_json::to_value(&runtime_state_changes)
5325 .unwrap_or_default(),
5326 ),
5327 (
5328 "changes".to_string(),
5329 serde_json::to_value(&state_changes).unwrap_or_default(),
5330 ),
5331 (
5332 "changes_semantics".to_string(),
5333 Value::from("provisional_state_mutations"),
5334 ),
5335 ("attempt".to_string(), Value::from(attempt + 1)),
5336 ("attempted".to_string(), Value::from(true)),
5337 ("stage".to_string(), Value::from("state_effect")),
5338 ]
5339 .into(),
5340 );
5341 }
5342
5343 return (
5344 ActionResult {
5345 action_id: action.id.clone(),
5346 status: ActionStatus::Succeeded,
5347 output: Some(output),
5348 error: None,
5349 terminal: false,
5350 state_changes,
5351 rolled_back: false,
5352 duration_ms: Some(duration_ms),
5353 timestamp: chrono::Utc::now(),
5354 },
5355 retries,
5356 );
5357 }
5358 Err(failure) => {
5359 let terminal = match failure.classification {
5360 ToolFailureClassification::Ordinary => false,
5361 ToolFailureClassification::Terminal => true,
5362 };
5363 let error = failure.message;
5364 last_error = Some(error.clone());
5365 let mut log = self.log.lock().await;
5366 let mut fail_data: HashMap<String, Value> = [
5367 ("error".to_string(), Value::from(error.as_str())),
5368 ("attempt".to_string(), Value::from(attempt + 1)),
5369 ("attempted".to_string(), Value::from(true)),
5370 ("stage".to_string(), Value::from("dispatch")),
5371 ]
5372 .into();
5373 // Tool name + ok=false so the receipt verifier (A6) records
5374 // that the tool *executed* (a failed call still ran).
5375 insert_tool_event_provenance(&mut fail_data, action, tool_source);
5376 insert_action_outcome_signal(
5377 &mut fail_data,
5378 action,
5379 ¶ms_digest,
5380 Some(action_error_class(action, &error, engine_timeout)),
5381 );
5382 if action.tool.is_some() {
5383 fail_data.insert("ok".to_string(), Value::from(false));
5384 }
5385 if terminal {
5386 fail_data.insert("terminal".to_string(), Value::from(true));
5387 }
5388 log.append(
5389 EventKind::ActionFailed,
5390 Some(&action.id),
5391 Some(proposal_id),
5392 fail_data,
5393 );
5394 drop(log);
5395
5396 if terminal {
5397 return (
5398 ActionResult {
5399 action_id: action.id.clone(),
5400 status: ActionStatus::Failed,
5401 output: None,
5402 error: Some(error),
5403 terminal: true,
5404 rolled_back: false,
5405 state_changes: HashMap::new(),
5406 duration_ms: Some(duration_ms),
5407 timestamp: chrono::Utc::now(),
5408 },
5409 retries,
5410 );
5411 }
5412 }
5413 }
5414 }
5415
5416 // All attempts exhausted
5417 if action.failure_behavior == FailureBehavior::Skip {
5418 let reason = last_error.as_deref().unwrap_or("all attempts exhausted");
5419 self.log.lock().await.append(
5420 EventKind::ActionSkipped,
5421 Some(&action.id),
5422 Some(proposal_id),
5423 action_outcome_data("failure_behavior", reason, true, Some(max_attempts)),
5424 );
5425 return (skipped_result(&action.id, reason), retries);
5426 }
5427
5428 (
5429 ActionResult {
5430 action_id: action.id.clone(),
5431 status: ActionStatus::Failed,
5432 output: None,
5433 error: last_error,
5434 terminal: false,
5435 state_changes: HashMap::new(),
5436 rolled_back: false,
5437 duration_ms: None,
5438 timestamp: chrono::Utc::now(),
5439 },
5440 retries,
5441 )
5442 }
5443
5444 /// Resolve the stable source category for an action's tool. Canonical
5445 /// ToolEntry provenance wins; the ToolSchema fallback keeps legacy direct
5446 /// registrations observable, and the final default covers old callers that
5447 /// supplied a tool field on a non-tool action.
5448 async fn tool_source_for_action(&self, action: &Action) -> Option<car_ir::ToolSourceKind> {
5449 let tool = action.tool.as_deref()?;
5450 if let Some(entry) = self.registry.get(tool).await {
5451 return Some(entry.source.kind());
5452 }
5453 Some(
5454 self.tools
5455 .read()
5456 .await
5457 .get(tool)
5458 .map(|schema| schema.source)
5459 .unwrap_or(car_ir::ToolSourceKind::UserDefined),
5460 )
5461 }
5462
5463 /// Dispatch an action to the appropriate handler.
5464 ///
5465 /// `scope` is the per-execution caller / tenant surface
5466 /// (Parslee-ai/car#187 phase 3). When the scope carries a
5467 /// tenant id, state R/W operations (`StateWrite`, `StateRead`,
5468 /// `Assertion`) route through `StateStore::scoped(tenant_id)`
5469 /// so distinct tenants can't see each other's keys. Unscoped
5470 /// proposals get the legacy flat-namespace behaviour
5471 /// automatically.
5472 async fn dispatch(
5473 &self,
5474 action: &Action,
5475 session_id: Option<&str>,
5476 scope: Option<&crate::scope::RuntimeScope>,
5477 attempt: u32,
5478 ) -> Result<Value, ToolFailure> {
5479 match action.action_type {
5480 ActionType::ToolCall => {
5481 let tool_name = action.tool.as_deref().ok_or("tool_call has no tool")?;
5482 let params = Value::Object(
5483 action
5484 .parameters
5485 .iter()
5486 .map(|(k, v)| (k.clone(), v.clone()))
5487 .collect(),
5488 );
5489
5490 // Detached invocation modes (C2): start the tool via the
5491 // configured executor's streaming entry point, register a
5492 // handle, and return immediately — the DAG must not block
5493 // on a streaming/long-running tool. Deliberately BEFORE
5494 // the result cache (a handle is a live invocation, never a
5495 // cacheable value) but still behind the rate limiter.
5496 // Built-ins don't stream; a detached call requires a
5497 // configured executor that implements execute_stream.
5498 if action.invocation_mode.is_detached() {
5499 self.rate_limiter.acquire(tool_name).await;
5500 let configured = {
5501 let guard = self.tool_executor.lock().await;
5502 guard.as_ref().cloned()
5503 };
5504 let executor = configured.ok_or_else(|| {
5505 format!("tool '{tool_name}': detached invocation requires a tool executor")
5506 })?;
5507 let rx = executor
5508 .execute_stream(tool_name, ¶ms, &action.id)
5509 .await?;
5510 let (handle, cancel) = self.tool_handles.register(tool_name, &action.id).await;
5511 crate::tool_handles::spawn_drain(
5512 self.tool_handles.clone(),
5513 handle.id.clone(),
5514 rx,
5515 cancel,
5516 );
5517 return Ok(serde_json::json!({
5518 "tool_handle": handle.id,
5519 "status": "running",
5520 }));
5521 }
5522
5523 // Check cross-proposal result cache.
5524 if let Some(cached) = self.result_cache.get(tool_name, ¶ms).await {
5525 return Ok(cached);
5526 }
5527
5528 // Apply rate limit backpressure before executing.
5529 self.rate_limiter.acquire(tool_name).await;
5530
5531 // Try built-in inference tools first when the inference engine is available.
5532 if matches!(
5533 tool_name,
5534 "infer" | "infer.grounded" | "embed" | "classify" | "transcribe" | "synthesize"
5535 ) {
5536 if let Some(ref engine) = self.inference_engine {
5537 // For "infer.grounded" or "infer" with memgine available,
5538 // build context from memory and attach it to the request.
5539 let params = {
5540 let should_ground =
5541 tool_name == "infer.grounded" || tool_name == "infer";
5542 if should_ground {
5543 if let Some(ref memgine) = self.memgine {
5544 if let Some(prompt) =
5545 params.get("prompt").and_then(|v| v.as_str())
5546 {
5547 let ctx = {
5548 let mut m = memgine.lock().await;
5549 m.build_context(prompt)
5550 };
5551 if !ctx.is_empty() {
5552 let mut p = params.clone();
5553 if let Some(obj) = p.as_object_mut() {
5554 obj.insert("context".to_string(), Value::from(ctx));
5555 }
5556 p
5557 } else {
5558 params
5559 }
5560 } else {
5561 params
5562 }
5563 } else {
5564 params
5565 }
5566 } else {
5567 params
5568 }
5569 };
5570
5571 // Route "infer.grounded" to "infer" for the service layer
5572 let effective_tool = if tool_name == "infer.grounded" {
5573 "infer"
5574 } else {
5575 tool_name
5576 };
5577 let result =
5578 car_inference::service::execute_tool(engine, effective_tool, ¶ms)
5579 .await
5580 .map_err(|e| e.to_string());
5581
5582 if let Ok(ref value) = result {
5583 self.result_cache
5584 .put(tool_name, ¶ms, value.clone())
5585 .await;
5586 }
5587
5588 return result.map_err(ToolFailure::from);
5589 }
5590 }
5591
5592 // Built-in memory consolidation tool.
5593 if tool_name == "memory.consolidate" {
5594 if let Some(ref memgine) = self.memgine {
5595 let report = {
5596 let mut m = memgine.lock().await;
5597 m.consolidate().await
5598 };
5599 // Log the consolidation event
5600 {
5601 let mut log = self.log.lock().await;
5602 log.append(
5603 EventKind::Consolidated,
5604 None,
5605 None,
5606 [
5607 (
5608 "expired_pruned".to_string(),
5609 Value::from(report.expired_pruned),
5610 ),
5611 (
5612 "superseded_gc".to_string(),
5613 Value::from(report.superseded_gc),
5614 ),
5615 (
5616 "stale_embeddings_removed".to_string(),
5617 Value::from(report.stale_embeddings_removed),
5618 ),
5619 (
5620 "nodes_embedded".to_string(),
5621 Value::from(report.nodes_embedded),
5622 ),
5623 (
5624 "domains_evolved".to_string(),
5625 Value::from(report.domains_evolved.clone()),
5626 ),
5627 ("total_nodes".to_string(), Value::from(report.total_nodes)),
5628 ("total_edges".to_string(), Value::from(report.total_edges)),
5629 ]
5630 .into(),
5631 );
5632 // Per-candidate gate telemetry (SkillOpt-inspired):
5633 // one event per promotion/rejection so the skill
5634 // bank's evolution is auditable.
5635 for key in &report.candidates_promoted {
5636 log.append(
5637 EventKind::CandidatePromoted,
5638 None,
5639 None,
5640 [("candidate".to_string(), Value::from(key.as_str()))].into(),
5641 );
5642 }
5643 for key in &report.candidates_rejected {
5644 log.append(
5645 EventKind::CandidateRejected,
5646 None,
5647 None,
5648 [("candidate".to_string(), Value::from(key.as_str()))].into(),
5649 );
5650 }
5651 }
5652 return Ok(serde_json::to_value(&report).unwrap_or(Value::Null));
5653 } else {
5654 return Err(
5655 "memory.consolidate requires memgine (attach with with_learning)"
5656 .into(),
5657 );
5658 }
5659 }
5660
5661 // Built-in outbound human messaging. Sits here, alongside the
5662 // other built-ins — downstream of `validate_action` and the
5663 // policy check in the caller, and downstream of the result
5664 // cache read and `rate_limiter.acquire` above — so a message
5665 // to a human traverses exactly the same governed chain
5666 // (validator → policy → rate limit → eventlog) as any other
5667 // side effect, instead of the hand-rolled transport each agent
5668 // used to carry.
5669 if tool_name == "messaging.send" {
5670 // No fall-through to the host executor when unconfigured.
5671 // Falling through would let an ungoverned host transport
5672 // answer the call — the exact path this built-in exists to
5673 // close — so an unconfigured runtime refuses instead.
5674 let Some(ref sink) = self.message_sink else {
5675 return Err("messaging.send: messaging is not configured on this \
5676 runtime — attach a message sink via \
5677 Runtime::with_message_sink"
5678 .into());
5679 };
5680 let msg = crate::messaging::OutboundMessage::from_tool_params(¶ms)?;
5681 let receipt = sink.send(&msg).await?;
5682 // Not cached: the schema declares no cache TTL, and a
5683 // cached send would swallow a second, genuinely wanted
5684 // message with identical text.
5685 return serde_json::to_value(&receipt).map_err(|error| {
5686 ToolFailure::ordinary(format!("messaging.send: serialize receipt: {error}"))
5687 });
5688 }
5689
5690 // Prefer a configured tool_executor for any tool it claims to handle.
5691 // Fall through to agent_basics only when the configured executor is absent
5692 // or returns a failure typed `ToolFailure::is_unhandled` — this prevents
5693 // agent_basics' built-in read_file/write_file (which resolve paths via
5694 // std::env::current_dir) from silently overriding an executor that carries
5695 // its own working_dir.
5696 //
5697 // The decision reads the typed flag, never the message. A transport
5698 // that relays another process's error (the daemon's WS client, an MCP
5699 // server) returns `ToolFailure::ordinary`, which is never unhandled, so a
5700 // client answering "unknown tool: read_file" gets its own error back
5701 // instead of a host file read (Parslee-ai/car#3104). In-process
5702 // `String` executors keep the "unknown tool" sentinel through
5703 // `From<String>`.
5704 let configured = {
5705 let guard = self.tool_executor.lock().await;
5706 guard.as_ref().cloned()
5707 };
5708
5709 if let Some(ref executor) = configured {
5710 let return_schema = self
5711 .tools
5712 .read()
5713 .await
5714 .get(tool_name)
5715 .and_then(|schema| schema.returns.clone());
5716 let result = executor
5717 .execute_classified(
5718 tool_name,
5719 ¶ms,
5720 &action.id,
5721 action.timeout_ms,
5722 session_id,
5723 attempt,
5724 &action.expected_effects,
5725 return_schema.as_ref(),
5726 )
5727 .await;
5728 let fall_through = matches!(&result, Err(error) if error.is_unhandled());
5729 if !fall_through {
5730 match result {
5731 Ok(execution) => {
5732 if !execution.state_changes.is_empty() {
5733 let expected_keys: std::collections::BTreeSet<&str> = action
5734 .expected_effects
5735 .keys()
5736 .map(String::as_str)
5737 .collect();
5738 let actual_keys: std::collections::BTreeSet<&str> = execution
5739 .state_changes
5740 .keys()
5741 .map(String::as_str)
5742 .collect();
5743 if expected_keys != actual_keys {
5744 return Err(format!(
5745 "tool '{}' callback state_changes keys do not match action '{}': expected {:?}, got {:?}",
5746 tool_name, action.id, expected_keys, actual_keys
5747 )
5748 .into());
5749 }
5750 car_inference::catalog_identity::canonical_json(&execution.output)
5751 .map_err(|error| {
5752 format!(
5753 "tool output failed JCS/I-JSON validation after dispatch; external effect may have occurred and was not undone: {error}"
5754 )
5755 })?;
5756 let changes_value = serde_json::to_value(&execution.state_changes)
5757 .map_err(|error| {
5758 format!(
5759 "tool '{}' callback state_changes serialization failed: {error}",
5760 tool_name
5761 )
5762 })?;
5763 car_inference::catalog_identity::canonical_json(&changes_value)
5764 .map_err(|error| {
5765 format!(
5766 "tool '{}' callback state_changes failed JCS/I-JSON validation: {error}",
5767 tool_name
5768 )
5769 })?;
5770
5771 // Every rejection above happens before the
5772 // first write. Apply in deterministic key order,
5773 // attributed to the active action, through ONE
5774 // batch mutation boundary: a concurrent reader,
5775 // journal replay, and restart recovery observe
5776 // the complete old state or the complete new
5777 // state — never a prefix of this callback's
5778 // keys (Parslee-ai/car#1140).
5779 let mut changes: Vec<(String, Value)> = execution
5780 .state_changes
5781 .iter()
5782 .map(|(key, value)| (key.clone(), value.clone()))
5783 .collect();
5784 changes.sort_by(|(left, _), (right, _)| left.cmp(right));
5785 let tenant = scope.and_then(|scope| scope.tenant_id.as_deref());
5786 self.state.scoped(tenant).set_batch(changes, &action.id);
5787 }
5788 self.result_cache
5789 .put(tool_name, ¶ms, execution.output.clone())
5790 .await;
5791 return Ok(execution.output);
5792 }
5793 Err(error) => return Err(error),
5794 }
5795 }
5796 }
5797
5798 // Built-in commodity tools resolve against the bound substrate
5799 // (default: LocalSubstrate → historic host behavior). `calculate`
5800 // stays pure inside agent_basics and ignores the substrate.
5801 let substrate = self.substrate.read().await.clone();
5802 let read_ledger = self.read_ledgers.ledger_for(session_id);
5803 let guard = self.local_path_guard.read().await.clone();
5804 let builtin = match &guard {
5805 Some(guard) => {
5806 crate::agent_basics::execute_with_ledger_guarded(
5807 &substrate,
5808 &read_ledger,
5809 tool_name,
5810 ¶ms,
5811 guard,
5812 )
5813 .await
5814 }
5815 None => {
5816 crate::agent_basics::execute_with_ledger(
5817 &substrate,
5818 &read_ledger,
5819 tool_name,
5820 ¶ms,
5821 )
5822 .await
5823 }
5824 };
5825 if let Some(result) = builtin {
5826 if let Ok(ref value) = result {
5827 self.result_cache
5828 .put(tool_name, ¶ms, value.clone())
5829 .await;
5830 }
5831 return result.map_err(ToolFailure::from);
5832 }
5833
5834 Err(format!("no handler for tool '{}'", tool_name).into())
5835 }
5836 ActionType::StateWrite => {
5837 let key = action
5838 .parameters
5839 .get("key")
5840 .and_then(|v| v.as_str())
5841 .ok_or("state_write requires 'key' parameter")?;
5842 let value = action
5843 .parameters
5844 .get("value")
5845 .cloned()
5846 .unwrap_or(Value::Null);
5847 let tenant = scope.and_then(|s| s.tenant_id.as_deref());
5848 self.state.scoped(tenant).set(key, value, &action.id);
5849 Ok(Value::from(format!("written: {}", key)))
5850 }
5851 ActionType::StateRead => {
5852 let key = action
5853 .parameters
5854 .get("key")
5855 .and_then(|v| v.as_str())
5856 .ok_or("state_read requires 'key' parameter")?;
5857 let tenant = scope.and_then(|s| s.tenant_id.as_deref());
5858 Ok(self.state.scoped(tenant).get(key).unwrap_or(Value::Null))
5859 }
5860 ActionType::Assertion => {
5861 let key = action
5862 .parameters
5863 .get("key")
5864 .and_then(|v| v.as_str())
5865 .ok_or("assertion requires 'key' parameter")?;
5866 let expected = action
5867 .parameters
5868 .get("expected")
5869 .cloned()
5870 .unwrap_or(Value::Null);
5871 let tenant = scope.and_then(|s| s.tenant_id.as_deref());
5872 let actual = self.state.scoped(tenant).get(key).unwrap_or(Value::Null);
5873 if actual != expected {
5874 Err(format!(
5875 "assertion failed: state['{}'] = {:?}, expected {:?}",
5876 key, actual, expected
5877 )
5878 .into())
5879 } else {
5880 Ok(serde_json::json!({"asserted": key, "value": actual}))
5881 }
5882 }
5883 }
5884 }
5885
5886 // --- Checkpoint and resume ---
5887
5888 /// Save a checkpoint of the current runtime state.
5889 pub async fn save_checkpoint(&self) -> Checkpoint {
5890 let state = self.state.snapshot();
5891 let tools: Vec<String> = self.tools.read().await.keys().cloned().collect();
5892 let log = self.log.lock().await;
5893 let events: Vec<Value> = log
5894 .events()
5895 .iter()
5896 .map(|e| serde_json::to_value(e).unwrap_or_default())
5897 .collect();
5898
5899 Checkpoint {
5900 checkpoint_id: Uuid::new_v4().to_string(),
5901 created_at: chrono::Utc::now(),
5902 state,
5903 events,
5904 tools,
5905 metadata: HashMap::new(),
5906 }
5907 }
5908
5909 /// Save checkpoint to a JSON file.
5910 pub async fn save_checkpoint_to_file(&self, path: &str) -> Result<(), String> {
5911 let checkpoint = self.save_checkpoint().await;
5912 let json = serde_json::to_string_pretty(&checkpoint)
5913 .map_err(|e| format!("serialize error: {}", e))?;
5914 tokio::fs::write(path, json)
5915 .await
5916 .map_err(|e| format!("write error: {}", e))?;
5917 Ok(())
5918 }
5919
5920 /// Load a checkpoint from a JSON file and restore state.
5921 pub async fn load_checkpoint_from_file(&self, path: &str) -> Result<Checkpoint, String> {
5922 let json = tokio::fs::read_to_string(path)
5923 .await
5924 .map_err(|e| format!("read error: {}", e))?;
5925 let checkpoint: Checkpoint =
5926 serde_json::from_str(&json).map_err(|e| format!("deserialize error: {}", e))?;
5927 self.restore_checkpoint(&checkpoint).await;
5928 Ok(checkpoint)
5929 }
5930
5931 /// Restore runtime state from a checkpoint.
5932 pub async fn restore_checkpoint(&self, checkpoint: &Checkpoint) {
5933 // Replace state completely — don't merge, don't create synthetic transitions
5934 self.state.replace_all(checkpoint.state.clone());
5935 // Clear idempotency cache — stale results from pre-checkpoint execution
5936 // must not bypass validation/policy on the restored state
5937 self.idempotency_cache.lock().await.clear();
5938 // Restore tools (as name-only schemas; full schemas are not persisted in checkpoint)
5939 let mut tools = self.tools.write().await;
5940 tools.clear();
5941 for tool_name in &checkpoint.tools {
5942 let schema = ToolSchema {
5943 name: tool_name.clone(),
5944 source: car_ir::ToolSourceKind::UserDefined,
5945 description: String::new(),
5946 parameters: serde_json::Value::Object(Default::default()),
5947 returns: None,
5948 idempotent: false,
5949 cache_ttl_secs: None,
5950 rate_limit: None,
5951 };
5952 tools.insert(tool_name.clone(), schema);
5953 }
5954 }
5955
5956 /// Register a subprocess tool and set up the subprocess executor.
5957 /// If no executor exists, creates a new SubprocessToolExecutor.
5958 /// If one already exists, creates a new SubprocessToolExecutor with the
5959 /// existing executor as fallback.
5960 pub async fn register_subprocess_tool(
5961 &self,
5962 name: &str,
5963 tool: crate::subprocess::SubprocessTool,
5964 ) {
5965 use crate::subprocess::SubprocessToolExecutor;
5966
5967 let schema = ToolSchema {
5968 name: name.to_string(),
5969 source: car_ir::ToolSourceKind::Subprocess,
5970 description: format!("Subprocess tool: {}", tool.command),
5971 parameters: serde_json::Value::Object(Default::default()),
5972 returns: None,
5973 idempotent: false,
5974 cache_ttl_secs: None,
5975 rate_limit: None,
5976 };
5977 self.register_tool_entry(
5978 crate::registry::ToolEntry::new(schema)
5979 .with_source(crate::registry::ToolSource::Subprocess),
5980 )
5981 .await;
5982
5983 let mut guard = self.tool_executor.lock().await;
5984 let mut executor = match guard.take() {
5985 Some(existing) => {
5986 let mut sub = SubprocessToolExecutor::new();
5987 sub = sub.with_fallback(existing);
5988 sub
5989 }
5990 None => SubprocessToolExecutor::new(),
5991 };
5992 executor.register(name, tool);
5993 *guard = Some(std::sync::Arc::new(executor));
5994 }
5995}
5996
5997impl Default for Runtime {
5998 fn default() -> Self {
5999 Self::new()
6000 }
6001}
6002
6003#[cfg(test)]
6004mod timeout_integration_tests {
6005 //! Integration coverage for the #259/#262 timeout coordination the unit
6006 //! tests in `car-server-core` / `car-ffi-common` only exercise as pure
6007 //! selection helpers (#266 item 4): that the executor's per-action deadline
6008 //! reaps a slow dispatch first, and that `action.timeout_ms` flows
6009 //! end-to-end onto `execute_with_action`.
6010 use super::*;
6011 use car_ir::ActionProposal;
6012 use std::sync::atomic::{AtomicU64, Ordering};
6013
6014 /// A tool executor that (a) records the `timeout_ms` it was handed and
6015 /// (b) sleeps `delay_ms` before returning, so a test can prove the
6016 /// executor's own `timeout(action.timeout_ms, dispatch)` reaps a call that
6017 /// outlives its budget.
6018 struct RecordingExecutor {
6019 seen_timeout_ms: Arc<AtomicU64>,
6020 delay_ms: u64,
6021 }
6022
6023 #[async_trait::async_trait]
6024 impl ToolExecutor for RecordingExecutor {
6025 async fn execute(&self, _tool: &str, _params: &Value) -> Result<Value, String> {
6026 Ok(Value::Null)
6027 }
6028 async fn execute_with_action(
6029 &self,
6030 _tool: &str,
6031 _params: &Value,
6032 _action_id: &str,
6033 timeout_ms: Option<u64>,
6034 ) -> Result<Value, String> {
6035 // `u64::MAX` sentinel = "None was passed".
6036 self.seen_timeout_ms
6037 .store(timeout_ms.unwrap_or(u64::MAX), Ordering::SeqCst);
6038 tokio::time::sleep(Duration::from_millis(self.delay_ms)).await;
6039 Ok(serde_json::json!({ "ok": true }))
6040 }
6041 }
6042
6043 fn one_tool_proposal(timeout_ms: Option<u64>) -> ActionProposal {
6044 let mut action = serde_json::json!({
6045 "id": "a0",
6046 "type": "tool_call",
6047 "tool": "slow",
6048 "parameters": {},
6049 "dependencies": [],
6050 });
6051 if let Some(ms) = timeout_ms {
6052 action["timeout_ms"] = serde_json::json!(ms);
6053 }
6054 serde_json::from_value(serde_json::json!({
6055 "source": "test",
6056 "actions": [action],
6057 }))
6058 .expect("proposal deserializes")
6059 }
6060
6061 #[tokio::test(start_paused = true)]
6062 async fn action_timeout_ms_reaches_executor_and_reaps_slow_dispatch() {
6063 let seen = Arc::new(AtomicU64::new(0));
6064 let rt = Runtime::new();
6065 rt.register_tool("slow").await;
6066 rt.set_executor(Arc::new(RecordingExecutor {
6067 seen_timeout_ms: seen.clone(),
6068 delay_ms: 2_000, // outlives the 100ms budget below
6069 }))
6070 .await;
6071
6072 // 100ms budget against a 2s dispatch: the executor's own
6073 // `timeout(action.timeout_ms, dispatch)` must reap it (#262 — the
6074 // executor is the authority), and the budget must have reached
6075 // `execute_with_action` (the #262 harness→action propagation, proven
6076 // end-to-end here rather than only via the pure helper).
6077 let result = rt.execute(&one_tool_proposal(Some(100))).await;
6078 assert_eq!(
6079 seen.load(Ordering::SeqCst),
6080 100,
6081 "budget must reach the executor"
6082 );
6083 let action_result = &result.results[0];
6084 assert!(
6085 action_result
6086 .error
6087 .as_deref()
6088 .unwrap_or("")
6089 .contains("timed out"),
6090 "slow dispatch must be reaped by the action deadline: {:?}",
6091 action_result.error
6092 );
6093 }
6094
6095 /// An executor that records every `attempt` it is handed and fails until
6096 /// the Nth, so a test can watch the counter advance across real retries.
6097 struct AttemptRecordingExecutor {
6098 seen: Arc<tokio::sync::Mutex<Vec<u32>>>,
6099 succeed_on: u32,
6100 }
6101
6102 #[async_trait::async_trait]
6103 impl ToolExecutor for AttemptRecordingExecutor {
6104 async fn execute(&self, _tool: &str, _params: &Value) -> Result<Value, String> {
6105 Ok(Value::Null)
6106 }
6107 async fn execute_with_action_in_session(
6108 &self,
6109 _tool: &str,
6110 _params: &Value,
6111 _action_id: &str,
6112 _timeout_ms: Option<u64>,
6113 _session_id: Option<&str>,
6114 attempt: u32,
6115 ) -> Result<Value, String> {
6116 self.seen.lock().await.push(attempt);
6117 if attempt >= self.succeed_on {
6118 Ok(serde_json::json!({ "ok": true }))
6119 } else {
6120 Err("transient".to_string())
6121 }
6122 }
6123 }
6124
6125 /// Parslee-ai/car#928 — the retry counter the executor receives must be the
6126 /// engine's real one, and it must advance.
6127 ///
6128 /// The WS executor hardcoded `attempt: 1` onto the `tools.execute` payload,
6129 /// so the field a host would build a retry-disambiguating join on never
6130 /// varied. Asserting on the sequence the executor actually observes is the
6131 /// property that matters: a frame test can only prove the payload carries
6132 /// whatever it was handed, not that the engine hands it the truth.
6133 #[tokio::test]
6134 async fn the_executor_sees_the_engines_real_attempt_sequence() {
6135 let seen = Arc::new(tokio::sync::Mutex::new(Vec::new()));
6136 let rt = Runtime::new();
6137 rt.register_tool("flaky").await;
6138 rt.set_executor(Arc::new(AttemptRecordingExecutor {
6139 seen: seen.clone(),
6140 succeed_on: 3,
6141 }))
6142 .await;
6143
6144 let proposal: ActionProposal = serde_json::from_value(serde_json::json!({
6145 "source": "test",
6146 "actions": [{
6147 "id": "a0",
6148 "type": "tool_call",
6149 "tool": "flaky",
6150 "parameters": {},
6151 "dependencies": [],
6152 "failure_behavior": "retry",
6153 "max_retries": 3,
6154 }],
6155 }))
6156 .expect("proposal deserializes");
6157
6158 let result = rt.execute(&proposal).await;
6159 assert_eq!(
6160 result.results[0].status,
6161 ActionStatus::Succeeded,
6162 "third attempt succeeds: {:?}",
6163 result.results[0].error
6164 );
6165 assert_eq!(
6166 *seen.lock().await,
6167 vec![1, 2, 3],
6168 "1-based and advancing — a constant here is the #928 bug"
6169 );
6170 }
6171
6172 #[tokio::test(start_paused = true)]
6173 async fn no_budget_applies_no_executor_deadline() {
6174 // The `None` path: the executor applies NO deadline, so a dispatch that
6175 // outlives any per-attempt budget still completes (the WS callback wait
6176 // is the sole bound on that path, exercised in car-server-core). Tokio's
6177 // paused clock advances the 300ms dispatch without a wall-clock wait.
6178 let seen = Arc::new(AtomicU64::new(0));
6179 let rt = Runtime::new();
6180 rt.register_tool("slow").await;
6181 rt.set_executor(Arc::new(RecordingExecutor {
6182 seen_timeout_ms: seen.clone(),
6183 delay_ms: 300,
6184 }))
6185 .await;
6186
6187 let result = rt.execute(&one_tool_proposal(None)).await;
6188 assert_eq!(
6189 seen.load(Ordering::SeqCst),
6190 u64::MAX,
6191 "None budget must be forwarded as None"
6192 );
6193 assert_eq!(result.results[0].status, ActionStatus::Succeeded);
6194 }
6195}
6196
6197#[cfg(test)]
6198mod model_facing_result_tests {
6199 use super::*;
6200
6201 #[test]
6202 fn a_model_reads_a_successful_output_as_text_and_a_failure_with_its_prefix() {
6203 let ok = ActionResult {
6204 action_id: "a".into(),
6205 status: ActionStatus::Succeeded,
6206 output: Some(serde_json::json!({ "path": "a.py", "content": "x = \"1\"\ny = 2" })),
6207 error: None,
6208 terminal: false,
6209 state_changes: HashMap::new(),
6210 rolled_back: false,
6211 duration_ms: None,
6212 timestamp: chrono::Utc::now(),
6213 };
6214 assert_eq!(
6215 format_tool_result_for_model(&ok),
6216 "path: a.py\ncontent (13 bytes):\nx = \"1\"\ny = 2"
6217 );
6218 // The JSON form stays what it was for every other consumer.
6219 assert!(format_tool_result(&ok).starts_with('{'));
6220 let failed = ActionResult {
6221 status: ActionStatus::Failed,
6222 output: None,
6223 error: Some("boom".into()),
6224 ..ok
6225 };
6226 assert_eq!(format_tool_result_for_model(&failed), "[FAILED] boom");
6227 }
6228
6229 #[test]
6230 fn rendered_output_reads_back_exactly() {
6231 // Shapes tools actually produce, including content that looks like
6232 // fields and strings that look like other JSON types.
6233 for original in [
6234 serde_json::json!({
6235 "path": "config/locales/en.yml",
6236 "content": "en:\n x: 1\nerror:\n not_found: Missing\n",
6237 "total_lines": 4,
6238 }),
6239 serde_json::json!({
6240 "status": 200,
6241 "url": "https://example.com/a",
6242 "headers": {"content-type": "text/plain", "authorization": "Bearer abc"},
6243 "body": "hello\nstatus:\nall good\nzebra:\n",
6244 }),
6245 serde_json::json!({
6246 "exit_code": 1,
6247 "output": "make: ***\nzebra:\nx",
6248 "timed_out": false,
6249 }),
6250 serde_json::json!({
6251 "error": "null",
6252 "flag": "true",
6253 "id": "123456789012345678901234",
6254 "name": "123",
6255 "content-type": "text/plain",
6256 "empty": "",
6257 "padded": " x ",
6258 "quoted": "\"q\"",
6259 }),
6260 ] {
6261 let text = render_tool_output(&original);
6262 assert_eq!(
6263 parse_rendered_output(&text).as_ref(),
6264 Some(&original),
6265 "{text}"
6266 );
6267 }
6268 assert_eq!(parse_rendered_output("[FAILED] boom"), None);
6269 assert_eq!(parse_rendered_output("plain text"), None);
6270 // Runtime failure and refusal lines are not fields, even with `: ` in them.
6271 for line in [
6272 "[FAILED] tool 'read_file': no such file",
6273 "[REJECTED] policy 'deny_shell': shell denied",
6274 "[FAILED] canceled: user stop",
6275 "Tool call denied: not allowed\nsecond line without a key",
6276 ] {
6277 assert_eq!(parse_rendered_output(line), None, "{line}");
6278 }
6279 // A shell result cut in the middle by the budget keeps its verdict:
6280 // the stderr block claims bytes that now run through the marker.
6281 let shell = serde_json::json!({
6282 "exit_code": 1,
6283 "stderr": "warning: x\n".repeat(700),
6284 "stdout": "test a ... FAILED\n".repeat(2500),
6285 });
6286 let cut = truncate_keeping_ends(&render_tool_output(&shell), 16 * 1024);
6287 assert_eq!(parse_rendered_output(&cut).unwrap()["exit_code"], 1);
6288 // A block spliced into a cut tail cannot replace an outcome field.
6289 let spliced = "ok: false\nout (10 bytes):\nab\nok (4 bytes):\ntrue";
6290 assert_eq!(parse_rendered_output(spliced).unwrap()["ok"], false);
6291 // … and a cut mid-line after the fields keeps them.
6292 let cut = "exit_code: 1\nstdo\n…[truncated: 10 of 20 bytes]…";
6293 assert_eq!(parse_rendered_output(cut).unwrap()["exit_code"], 1);
6294 // Keys and values that look like the format's own syntax.
6295 let odd = serde_json::json!({
6296 "a:": "x\ny",
6297 "a: b": "c",
6298 "note": "wrote (3 bytes):",
6299 " padded": 1,
6300 "k (x": "y\nz",
6301 });
6302 let text = render_tool_output(&odd);
6303 assert_eq!(parse_rendered_output(&text).as_ref(), Some(&odd), "{text}");
6304 // A block cut by a budget reads back as what is left of it.
6305 let cut = "exit_code: 1\noutput (100 bytes):\nfirst lines only";
6306 assert_eq!(
6307 parse_rendered_output(cut).unwrap()["output"],
6308 "first lines only"
6309 );
6310 }
6311
6312 #[test]
6313 fn outcome_fields_lead_and_stay_on_one_line() {
6314 let out = render_tool_output(&serde_json::json!({
6315 "content": "x\ny",
6316 "error": format!("{}\nsecond line", "e".repeat(600)),
6317 "exit_code": 1,
6318 "matches": [{"path": "a"}],
6319 "truncated": true,
6320 }));
6321 let lines: Vec<&str> = out.lines().collect();
6322 assert!(lines[0].starts_with("error: "), "{out}");
6323 assert!(
6324 lines[0].len() <= 530 && lines[0].contains(" bytes]…"),
6325 "{out}"
6326 );
6327 assert!(
6328 lines[0].ends_with("second line"),
6329 "the end of a long error survives: {out}"
6330 );
6331 assert_eq!(lines[1], "exit_code: 1");
6332 assert_eq!(lines[2], "truncated: true");
6333 assert!(
6334 out.ends_with("content (3 bytes):\nx\ny\nmatches (1 records):\npath=a"),
6335 "{out}"
6336 );
6337 }
6338
6339 #[test]
6340 fn long_command_output_keeps_its_verdict() {
6341 let out = format!(
6342 "exit_code: 1\noutput:\n{}\n1 failed, 99 passed",
6343 "é.".repeat(20_000)
6344 );
6345 let cut = truncate_keeping_ends(&out, 4096);
6346 assert!(cut.len() <= 4096, "{}", cut.len());
6347 assert!(cut.starts_with("exit_code: 1"));
6348 assert!(cut.ends_with("1 failed, 99 passed"));
6349 assert!(cut.contains("bytes elided from the middle"));
6350 assert_eq!(truncate_keeping_ends("short", 4096), "short");
6351 assert!(truncate_keeping_ends(&out, 4096).contains(ELIDED_MARKER));
6352 let tiny = truncate_keeping_ends(&"x".repeat(1000), 64);
6353 assert!(tiny.len() <= 64, "{}", tiny.len());
6354 }
6355}