car_engine/executor.rs
1//! Core execution engine — propose → validate → execute → commit.
2
3use crate::cache::ResultCache;
4use crate::capabilities::CapabilitySet;
5use crate::checkpoint::Checkpoint;
6use crate::rate_limit::{RateLimit, RateLimiter};
7use car_eventlog::{EventKind, EventLog, SpanStatus};
8use car_ir::{
9 build_dag, AcceptedProposalPreimage, Action, ActionProposal, ActionResult, ActionStatus,
10 ActionType, CostSummary, FailureBehavior, ProposalLineageEntry, ProposalLineageStatus,
11 ProposalResult, StateMutation, ToolFailure, ToolFailureClassification, ToolSchema,
12};
13use car_policy::{ApprovalDecision, ApprovalLedger, ApprovalRecord, PermissionTier, PolicyEngine};
14use car_state::{RestoreDurability, StateStore};
15use car_validator::validate_action;
16use serde_json::Value;
17use sha2::{Digest, Sha256};
18use std::collections::HashMap;
19use std::sync::Arc;
20use std::time::Duration;
21use tokio::sync::{Mutex as TokioMutex, RwLock as TokioRwLock};
22use tokio::time::timeout;
23use tracing::instrument;
24use uuid::Uuid;
25
26/// Retry backoff constants.
27const RETRY_BASE_DELAY_MS: u64 = 100;
28const RETRY_BACKOFF_FACTOR: u64 = 2;
29const ROLLBACK_WARNING: &str =
30 "proposal aborted; state effects were rolled back; external effects may remain and were not undone";
31const PLAN_FALLBACK_ROLLBACK_WARNING: &str =
32 "planning candidate rejected; state effects were rolled back; external effects may remain and were not undone";
33const ROLLBACK_DURABILITY_ERROR: &str =
34 "durable state rollback failed before publication; in-memory state and idempotency entries were preserved";
35const ROLLBACK_DURABILITY_UNKNOWN: &str =
36 "state rollback was published but parent-directory durability is unknown; idempotency entries were invalidated for exact retry";
37
38// ---------------------------------------------------------------------------
39// Replan types — failure recovery via model callback
40// ---------------------------------------------------------------------------
41
42/// Callback trait for replanning failed proposals.
43/// Implement this to let the runtime ask the model for an alternative plan
44/// when a proposal aborts.
45#[async_trait::async_trait]
46pub trait ReplanCallback: Send + Sync {
47 async fn replan(&self, ctx: &ReplanContext) -> Result<ActionProposal, String>;
48}
49
50/// Context provided to the replan callback so the model can generate an alternative.
51#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
52pub struct ReplanContext {
53 /// Original proposal ID.
54 pub proposal_id: String,
55 /// Which replan attempt this is (1-indexed; attempt 1 = first replan after initial failure).
56 pub attempt: u32,
57 /// Actions that failed and caused the abort.
58 pub failed_actions: Vec<FailedActionSummary>,
59 /// Action IDs that succeeded before the abort (now rolled back).
60 pub completed_action_ids: Vec<String>,
61 /// State snapshot after rollback.
62 pub state_snapshot: HashMap<String, Value>,
63 /// How many replans remain.
64 pub replans_remaining: u32,
65 /// Original proposal source (model name, agent, etc.).
66 pub original_source: String,
67 /// Total actions in the original proposal.
68 pub original_action_count: usize,
69 /// The original goal/context from the proposal (for generating alternatives).
70 pub original_context: HashMap<String, Value>,
71}
72
73/// Summary of a failed action, included in ReplanContext.
74#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
75pub struct FailedActionSummary {
76 pub action_id: String,
77 pub tool: Option<String>,
78 pub error: String,
79 pub parameters: HashMap<String, Value>,
80}
81
82/// Configuration for the replan loop.
83#[derive(Debug, Clone)]
84pub struct ReplanConfig {
85 /// Maximum number of replan attempts. 0 = disabled (default).
86 pub max_replans: u32,
87 /// Delay in milliseconds between replan attempts. Prevents burning through
88 /// attempts instantly if the model returns garbage fast. 0 = no delay.
89 pub delay_ms: u64,
90 /// If true, replan proposals are scored via car-planner's verify() before
91 /// execution. Proposals with errors are rejected without executing.
92 /// Prevents the engine from running a worse plan than the one that failed.
93 pub verify_before_execute: bool,
94 /// When true, validator/policy/capability rejections (ActionStatus::Rejected)
95 /// also trigger rollback + replan, not just runtime Failed actions.
96 /// Default false (conservative: preserves prior abort-only-on-Failed behavior).
97 pub replan_on_rejected: bool,
98}
99
100/// How the runtime treats a proposal that conflicts with the current
101/// shared state on a pre-execution transactional check (survey §4.3/§5.2.4
102/// — `car_verify::check_transaction` against the versioned `StateStore`).
103#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
104pub enum TransactionCheckMode {
105 /// Don't run the check (default — preserves prior behavior exactly).
106 #[default]
107 Off,
108 /// Run it and emit any conflicts as `TransactionConflict` telemetry, but
109 /// execute anyway.
110 Warn,
111 /// Run it; if any conflict is found, reject the proposal without
112 /// executing (the conflicting actions become `Rejected` results).
113 Strict,
114}
115
116impl Default for ReplanConfig {
117 fn default() -> Self {
118 Self {
119 max_replans: 0,
120 delay_ms: 0,
121 verify_before_execute: true,
122 replan_on_rejected: false,
123 }
124 }
125}
126
127/// Trait for tool execution. Implement this to provide tools to the runtime.
128///
129/// In-process: implement directly with function calls.
130/// Daemon mode: implement by sending JSON-RPC to the client.
131#[async_trait::async_trait]
132pub trait ToolExecutor: Send + Sync {
133 async fn execute(&self, tool: &str, params: &Value) -> Result<Value, String>;
134
135 /// Variant that also carries the originating proposal `Action.id`
136 /// and its `timeout_ms` budget.
137 ///
138 /// `action_id`: WS-based executors (`car-server-core::WsToolExecutor`)
139 /// use it so the daemon-initiated `tools.execute` request to the client
140 /// carries the same id the host's process-wide handler is keyed on
141 /// — without this round-trip the host can't disambiguate concurrent
142 /// callbacks for the same tool (Parslee-ai/car-releases#43 follow-up).
143 ///
144 /// `timeout_ms`: the action's per-call budget. WS executors MUST bound
145 /// their callback wait by this (falling back to a default when `None`)
146 /// so the daemon→host wait and the executor's own action deadline stay
147 /// coordinated — otherwise a hardcoded inner wait reaps a call the outer
148 /// deadline still permits (Parslee-ai/car#259). In-process executors
149 /// that don't need either can keep the default forward to [`execute`](crate::executor::ToolExecutor::execute).
150 async fn execute_with_action(
151 &self,
152 tool: &str,
153 params: &Value,
154 _action_id: &str,
155 _timeout_ms: Option<u64>,
156 ) -> Result<Value, String> {
157 self.execute(tool, params).await
158 }
159
160 /// Variant that also carries the Runtime execution session and the retry
161 /// attempt. Executors that retain safety-relevant state across calls (such
162 /// as a read-before-edit ledger) override this to keep that state isolated
163 /// by session. Existing executors retain their current behavior through the
164 /// default delegation.
165 ///
166 /// `attempt` is **1-based**: `1` on the first try, `2` on the first retry.
167 /// WS executors surface it on the `tools.execute` payload so a host can
168 /// tell which retry it is serving — `action_id` cannot, being neither
169 /// unique across attempts nor varying between them.
170 ///
171 /// It is threaded from `execute_with_retry`'s own counter rather than
172 /// synthesized here. It was previously hardcoded to `1` at the one place
173 /// that put it on the wire, which made the field a constant and any join
174 /// built on it silently degenerate (Parslee-ai/car#928).
175 async fn execute_with_action_in_session(
176 &self,
177 tool: &str,
178 params: &Value,
179 action_id: &str,
180 timeout_ms: Option<u64>,
181 _session_id: Option<&str>,
182 _attempt: u32,
183 ) -> Result<Value, String> {
184 self.execute_with_action(tool, params, action_id, timeout_ms)
185 .await
186 }
187
188 /// Execute one action while allowing a transport to return runtime-observed
189 /// state mutations alongside its ordinary tool output.
190 ///
191 /// The default preserves every direct/internal executor: its existing
192 /// output is wrapped with no mutations, and declared `expected_effects`
193 /// remain assertions rather than writes. A transport that overrides this
194 /// method must validate its mutation envelope against `expected_effects`
195 /// before returning. The Runtime repeats the key-set and I-JSON checks and
196 /// applies accepted changes through its own [`StateStore`].
197 async fn execute_with_action_state_in_session(
198 &self,
199 tool: &str,
200 params: &Value,
201 action_id: &str,
202 timeout_ms: Option<u64>,
203 session_id: Option<&str>,
204 attempt: u32,
205 _expected_effects: &HashMap<String, Value>,
206 _return_schema: Option<&Value>,
207 ) -> Result<ToolExecution, String> {
208 self.execute_with_action_in_session(
209 tool, params, action_id, timeout_ms, session_id, attempt,
210 )
211 .await
212 .map(ToolExecution::output_only)
213 }
214
215 /// Execute one action with typed failure evidence.
216 ///
217 /// This is a new default-delegating layer rather than a signature change
218 /// to the existing execution methods. Existing executors therefore retain
219 /// their exact behavior: every legacy string error becomes an ordinary,
220 /// non-terminal failure. Executors opt into fail-stop behavior only by
221 /// overriding this method and returning [`ToolFailure::terminal`].
222 ///
223 /// The runtime runs its built-in tool instead only for a failure that
224 /// [`ToolFailure::is_unhandled`]. The default conversion keeps the legacy
225 /// `"unknown tool"` string sentinel for in-process executors. An executor
226 /// that relays another process's errors (a WS client, an MCP server) must
227 /// override this method and return [`ToolFailure::ordinary`] for them, so a
228 /// peer's text can never pick the built-in (Parslee-ai/car#3104).
229 async fn execute_classified(
230 &self,
231 tool: &str,
232 params: &Value,
233 action_id: &str,
234 timeout_ms: Option<u64>,
235 session_id: Option<&str>,
236 attempt: u32,
237 expected_effects: &HashMap<String, Value>,
238 return_schema: Option<&Value>,
239 ) -> Result<ToolExecution, ToolFailure> {
240 self.execute_with_action_state_in_session(
241 tool,
242 params,
243 action_id,
244 timeout_ms,
245 session_id,
246 attempt,
247 expected_effects,
248 return_schema,
249 )
250 .await
251 .map_err(ToolFailure::from)
252 }
253
254 /// Streaming entry point for detached invocation modes (C2). Start
255 /// the tool and return a channel of [`car_ir::ToolStreamChunk`]s; the
256 /// runtime drains it into the per-runtime handle registry while the
257 /// DAG proceeds. End the stream with a terminal chunk (`done` /
258 /// `error`); dropping the sender without one is reported as failure.
259 /// A cooperative executor should stop work when the receiver returned
260 /// here is dropped (that's what cancellation looks like from its side).
261 ///
262 /// Default: unsupported — existing one-shot executors compile and
263 /// behave unchanged; a detached action against them is rejected with
264 /// this error.
265 async fn execute_stream(
266 &self,
267 tool: &str,
268 _params: &Value,
269 _action_id: &str,
270 ) -> Result<tokio::sync::mpsc::Receiver<car_ir::ToolStreamChunk>, String> {
271 Err(format!(
272 "tool '{tool}': this executor does not support streaming/long-running invocation"
273 ))
274 }
275}
276
277/// Result of a tool dispatch before the Runtime commits observed state.
278#[derive(Debug, Clone, PartialEq)]
279pub struct ToolExecution {
280 pub output: Value,
281 pub state_changes: HashMap<String, Value>,
282}
283
284impl ToolExecution {
285 pub fn output_only(output: Value) -> Self {
286 Self {
287 output,
288 state_changes: HashMap::new(),
289 }
290 }
291}
292
293/// Deterministic key for idempotency deduplication.
294/// Deterministic key for idempotency deduplication. Carries the tenant
295/// dimension (linus review): without it, tenant A's cached result for
296/// an identical idempotent action was served to tenant B — a
297/// cross-tenant data leak through the dedup cache. Unscoped executions
298/// keep their historical keys (empty tenant segment).
299fn idempotency_key(action: &Action, scope: Option<&crate::scope::RuntimeScope>) -> String {
300 let sorted: std::collections::BTreeMap<_, _> = action.parameters.iter().collect();
301 let params = serde_json::to_string(&sorted).unwrap_or_default();
302 let tenant = scope.and_then(|s| s.tenant_id.as_deref()).unwrap_or("");
303 format!(
304 "{}:{}:{}:{}",
305 tenant,
306 serde_json::to_string(&action.action_type).unwrap_or_default(),
307 action.tool.as_deref().unwrap_or(""),
308 params
309 )
310}
311
312/// Exact normal-serde proposal preimage plus its lowercase RFC 8785/JCS
313/// SHA-256 identity. The shared inference helper is the repository's settled
314/// canonicalizer; journal identity must not drift onto a second JSON sorter.
315fn proposal_journal_identity(proposal: &ActionProposal) -> Result<(Value, String), String> {
316 let preimage = serde_json::to_value(proposal)
317 .map_err(|error| format!("proposal serialization failed: {error}"))?;
318 let canonical = car_inference::catalog_identity::canonical_json(&preimage)?;
319 let digest = format!("{:x}", Sha256::digest(canonical.as_bytes()));
320 Ok((preimage, digest))
321}
322
323fn proposal_digest(proposal: &ActionProposal) -> Result<String, String> {
324 proposal_journal_identity(proposal).map(|(_, digest)| digest)
325}
326
327fn proposal_rejection_boundary_data(
328 proposal: &ActionProposal,
329 proposal_digest: Option<&str>,
330 reason: &str,
331) -> HashMap<String, Value> {
332 let mut data: HashMap<String, Value> = [
333 ("affected_actions".to_string(), Value::Array(Vec::new())),
334 (
335 "rolled_back_changes".to_string(),
336 Value::Object(serde_json::Map::new()),
337 ),
338 (
339 "changes_semantics".to_string(),
340 Value::from("proposal_rejected_no_state_commit"),
341 ),
342 ("attempted".to_string(), Value::from(false)),
343 ("stage".to_string(), Value::from("proposal_rejection")),
344 ("rejection_reason".to_string(), Value::from(reason)),
345 ]
346 .into();
347 if let Ok(preimage) = serde_json::to_value(proposal) {
348 data.insert("proposal".to_string(), preimage);
349 }
350 if let Some(digest) = proposal_digest {
351 data.insert("proposal_digest".to_string(), Value::from(digest));
352 }
353 data
354}
355
356fn proposal_lineage_entry(
357 proposal: &ActionProposal,
358 generation: u32,
359 status: ProposalLineageStatus,
360 rejection_reason: Option<String>,
361) -> ProposalLineageEntry {
362 ProposalLineageEntry {
363 generation,
364 proposal_id: proposal.id.clone(),
365 proposal_digest: proposal_digest(proposal).ok(),
366 status,
367 rejection_reason,
368 }
369}
370
371fn finalize_proposal_result(
372 mut result: ProposalResult,
373 original_proposal_id: &str,
374 lineage: &[ProposalLineageEntry],
375 accepted_proposal_preimages: &[AcceptedProposalPreimage],
376) -> ProposalResult {
377 result.original_proposal_id = original_proposal_id.to_string();
378 result.replan_lineage = lineage.to_vec();
379 result.accepted_proposal_preimages = accepted_proposal_preimages.to_vec();
380 result
381}
382
383/// Validate the proposal-local action identity boundary used by DAG execution,
384/// action receipts, results, and state-transition attribution. A retry reuses
385/// one admitted action id across attempts; two declared actions may not share
386/// an id because every downstream join would become ambiguous.
387pub fn validate_proposal_action_ids(proposal: &ActionProposal) -> Result<(), String> {
388 let mut seen = std::collections::HashSet::with_capacity(proposal.actions.len());
389 for action in &proposal.actions {
390 if !seen.insert(action.id.as_str()) {
391 return Err(format!("duplicate action id '{}'", action.id));
392 }
393 }
394 Ok(())
395}
396
397fn validate_proposal_retry_limits(proposal: &ActionProposal) -> Result<(), String> {
398 for action in &proposal.actions {
399 if action.failure_behavior == FailureBehavior::Retry
400 && action.max_retries.checked_add(1).is_none()
401 {
402 return Err(format!(
403 "action '{}' retry attempt count overflows u32",
404 action.id
405 ));
406 }
407 }
408 Ok(())
409}
410
411fn requires_integrity_rollback(result: &ActionResult) -> bool {
412 result.status == ActionStatus::Failed
413 && result.error.as_deref().is_some_and(|error| {
414 error.starts_with("tool output failed JCS/I-JSON validation after dispatch")
415 })
416}
417
418fn record_rollback_durability_error(
419 results: &mut [ActionResult],
420 prefix: &str,
421 error: &str,
422) -> String {
423 let detail = format!("{prefix}: {error}");
424 if let Some(result) = results
425 .iter_mut()
426 .find(|result| result.status != ActionStatus::Succeeded)
427 {
428 result.error = Some(match result.error.take() {
429 Some(existing) => format!("{existing}; {detail}"),
430 None => detail.clone(),
431 });
432 }
433 detail
434}
435
436fn rejected_result(action_id: &str, error: String) -> ActionResult {
437 ActionResult {
438 action_id: action_id.to_string(),
439 status: ActionStatus::Rejected,
440 output: None,
441 error: Some(error),
442 terminal: false,
443 state_changes: HashMap::new(),
444 rolled_back: false,
445 duration_ms: None,
446 timestamp: chrono::Utc::now(),
447 }
448}
449
450/// Whether the actions of one `build_dag` level run one after another, in level
451/// (index) order, rather than concurrently. A level runs sequentially when it
452/// holds one action or any action in it has `FailureBehavior::Abort` — the
453/// default — so an earlier action in such a level finishes before a later one
454/// is validated. The static-verification gate's cycle rule reads this too, so
455/// its notion of "finishes first" is the executor's by construction.
456pub(crate) fn level_runs_sequentially(level: &[usize], actions: &[Action]) -> bool {
457 level.len() == 1
458 || level
459 .iter()
460 .any(|&i| actions[i].failure_behavior == FailureBehavior::Abort)
461}
462
463/// The tenant namespace an action's own state keys live in when its proposal
464/// runs under `tenant` (car#3121). Dispatch routes `StateWrite`, `StateRead`
465/// and `Assertion` through `StateStore::scoped(tenant)`, while a `ToolCall`
466/// runs on the unscoped path: its handler reads and `state.set`s the shared
467/// store. So a tool call's preconditions, dependencies and assumptions name
468/// shared keys, and a state action's name its tenant's. Every pre-execution
469/// check (validator, admission gate, transactional pre-check, replan
470/// verification) resolves an action's keys through this, so it judges each
471/// action against the state that action executes against.
472pub(crate) fn action_state_tenant<'a>(action: &Action, tenant: Option<&'a str>) -> Option<&'a str> {
473 match action.action_type {
474 ActionType::ToolCall => None,
475 ActionType::StateWrite | ActionType::StateRead | ActionType::Assertion => tenant,
476 }
477}
478
479/// The store keys an action's declared writes (`expected_effects`, and a
480/// `state_write`'s key) can land on when its proposal runs under `tenant`.
481/// Each is the key in the action's own namespace ([`action_state_tenant`]);
482/// a tool call's declared effects are also its tenant's copies, because its
483/// handler can `state.set` the shared key while its callback `state_changes`
484/// (whose keys must equal `expected_effects`) land in the tenant's namespace.
485/// The static-verification gate and the transactional pre-check both read an
486/// action's writes through this, so they model the same keys (car#3133).
487/// Sorted and deduplicated.
488pub(crate) fn action_write_store_keys(action: &Action, tenant: Option<&str>) -> Vec<String> {
489 let own_tenant = action_state_tenant(action, tenant);
490 let state_write_key = (action.action_type == ActionType::StateWrite)
491 .then(|| action.parameters.get("key").and_then(|v| v.as_str()))
492 .flatten();
493 let mut keys: Vec<String> = state_write_key
494 .into_iter()
495 .chain(action.expected_effects.keys().map(String::as_str))
496 .flat_map(|key| {
497 let own = car_state::tenant_key(own_tenant, key);
498 let tenant_copy = (action.action_type == ActionType::ToolCall)
499 .then(|| car_state::tenant_key(tenant, key))
500 .filter(|k| *k != own);
501 std::iter::once(own).chain(tenant_copy)
502 })
503 .collect();
504 keys.sort_unstable();
505 keys.dedup();
506 keys
507}
508
509/// The state view an action's own keys are read through when its proposal
510/// runs under `tenant`: its tenant's namespace for a state action, and for a
511/// tool call the store under raw names restricted to the keys visible to
512/// `tenant` (shared keys and its own, never another tenant's;
513/// `StateStore::visible_to`). With no tenant, the whole store.
514fn action_state_view<'a>(
515 state: &'a StateStore,
516 action: &Action,
517 tenant: Option<&'a str>,
518) -> Box<dyn car_ir::precondition::StateView + Send + Sync + 'a> {
519 match action_state_tenant(action, tenant) {
520 Some(t) => Box::new(state.scoped(Some(t))),
521 None => Box::new(state.visible_to(tenant)),
522 }
523}
524
525/// `proposal` with every state key a tenant-namespaced action names
526/// rewritten to the store key it executes against (see
527/// [`action_state_tenant`]), so a whole-store check such as
528/// `car_verify::check_transaction` reads each action's keys in its own
529/// namespace. Borrowed unchanged when there is no tenant.
530///
531/// A tool call's keys stay shared, but its write set also gets the tenant's
532/// copies of its declared effects ([`action_write_store_keys`], car#3133).
533fn proposal_in_store_keys<'p>(
534 proposal: &'p ActionProposal,
535 tenant: Option<&str>,
536) -> std::borrow::Cow<'p, ActionProposal> {
537 if tenant.is_none_or(|t| t.is_empty()) {
538 return std::borrow::Cow::Borrowed(proposal);
539 }
540 let mut resolved = proposal.clone();
541 for action in &mut resolved.actions {
542 let Some(t) = action_state_tenant(action, tenant) else {
543 for k in action_write_store_keys(action, tenant) {
544 if !action.write_set.contains(&k) {
545 action.write_set.push(k);
546 }
547 }
548 continue;
549 };
550 let key = |k: &str| car_state::tenant_key(Some(t), k);
551 for k in action
552 .state_dependencies
553 .iter_mut()
554 .chain(&mut action.read_set)
555 .chain(&mut action.write_set)
556 {
557 *k = key(k);
558 }
559 for pre in &mut action.preconditions {
560 pre.key = key(&pre.key);
561 }
562 for assumption in &mut action.assumptions {
563 assumption.key = key(&assumption.key);
564 }
565 action.expected_effects = std::mem::take(&mut action.expected_effects)
566 .into_iter()
567 .map(|(k, v)| (key(&k), v))
568 .collect();
569 if let Some(Value::String(k)) = action.parameters.get_mut("key") {
570 *k = key(k);
571 }
572 }
573 std::borrow::Cow::Owned(resolved)
574}
575
576/// Capture only the state keys relevant to an action (state_dependencies + expected_effects).
577/// Returns an empty map if the action declares no relevant keys (e.g., tool calls
578/// that don't interact with state). `state` is the view the action executes
579/// against ([`action_state_tenant`]), so the values recorded are the ones the
580/// action saw.
581fn snapshot_relevant_keys(
582 state: &dyn car_ir::precondition::StateView,
583 action: &Action,
584) -> HashMap<String, Value> {
585 let mut keys: std::collections::HashSet<&str> = std::collections::HashSet::new();
586 for dep in &action.state_dependencies {
587 keys.insert(dep.as_str());
588 }
589 for key in action.expected_effects.keys() {
590 keys.insert(key.as_str());
591 }
592 // For state_write actions, capture the key being written
593 if action.action_type == ActionType::StateWrite {
594 if let Some(key) = action.parameters.get("key").and_then(|v| v.as_str()) {
595 keys.insert(key);
596 }
597 }
598
599 if keys.is_empty() {
600 // No declared state interaction — skip snapshot (empty map)
601 return HashMap::new();
602 }
603
604 keys.iter()
605 .filter_map(|&k| state.get_value(k).map(|v| (k.to_string(), v)))
606 .collect()
607}
608
609fn skipped_result(action_id: &str, reason: &str) -> ActionResult {
610 ActionResult {
611 action_id: action_id.to_string(),
612 status: ActionStatus::Skipped,
613 output: None,
614 error: Some(reason.to_string()),
615 terminal: false,
616 state_changes: HashMap::new(),
617 rolled_back: false,
618 duration_ms: None,
619 timestamp: chrono::Utc::now(),
620 }
621}
622
623fn insert_tool_event_provenance(
624 data: &mut HashMap<String, Value>,
625 action: &Action,
626 source: Option<car_ir::ToolSourceKind>,
627) {
628 if let Some(tool) = action.tool.as_deref() {
629 data.insert("tool".to_string(), Value::from(tool));
630 data.insert(
631 "tool_source".to_string(),
632 Value::from(
633 source
634 .unwrap_or(car_ir::ToolSourceKind::UserDefined)
635 .as_str(),
636 ),
637 );
638 }
639}
640
641/// SHA-256 over the RFC 8785/JCS rendering of only the action parameters.
642///
643/// The accepted proposal has already crossed `proposal_journal_identity`, so
644/// canonicalization cannot fail here without violating that admission
645/// invariant. Keeping only the digest on outcome events lets a later join find
646/// the raw parameters in `ProposalReceived` without copying them into every
647/// failure record.
648fn action_params_digest(action: &Action) -> String {
649 // Serialize the map directly: RFC 8785/JCS sorts object keys
650 // recursively at render time (see `canonical_json` and the
651 // `canonical_json_sorts_object_keys_recursively` test), so the map's
652 // iteration order does not affect the digest and cloning into a
653 // `Value::Object` would be pure per-action allocation overhead.
654 let canonical = car_inference::catalog_identity::canonical_json(&action.parameters)
655 .expect("accepted action parameters must remain RFC 8785/JCS canonicalizable");
656 format!("{:x}", Sha256::digest(canonical.as_bytes()))
657}
658
659fn insert_action_outcome_signal(
660 data: &mut HashMap<String, Value>,
661 action: &Action,
662 params_digest: &str,
663 error_class: Option<&str>,
664) {
665 data.insert(
666 "params_digest".to_string(),
667 Value::from(params_digest.to_string()),
668 );
669 data.insert(
670 "expected_effects".to_string(),
671 serde_json::to_value(&action.expected_effects)
672 .expect("action expected_effects must serialize as JSON"),
673 );
674 if let Some(error_class) = error_class {
675 data.insert("error_class".to_string(), Value::from(error_class));
676 }
677}
678
679/// Normalize execution failures into a stable, deliberately low-cardinality
680/// event-log vocabulary. Validation and policy checks that happen before
681/// dispatch remain `ActionRejected`; this maps failures reached after an
682/// action began executing.
683pub(crate) fn action_error_class(
684 action: &Action,
685 error: &str,
686 engine_timeout: bool,
687) -> &'static str {
688 // Classification is by substring over prose and is a published wire
689 // contract (docs/agent-ir-spec.md `error_class`). Each literal is pinned
690 // to its producer below; changing a producer's wording must update the
691 // matching literal (and `action_failure_error_class_mapping_is_stable`).
692 let error = error.to_ascii_lowercase();
693 // "callback timed out" — the engine deadline itself is typed via
694 // `engine_timeout`, so this literal only covers the detached-dispatch
695 // producer `tool '{tool}' callback timed out ({s}s)`
696 // (car-server-core/src/session.rs, `reap_detached_calls`/wait path).
697 // A tool faithfully relaying a *downstream* timeout in its own message
698 // also lands here; accepted drift, low-cardinality vocabulary.
699 if engine_timeout || error.contains("callback timed out") {
700 "timeout"
701 } else if error.starts_with("denied by policy:") || error.starts_with("rejected by policy:") {
702 // Exact producers return this prefix unwrapped from the tool's
703 // guard check: car-server-core/src/assistant/executor.rs and
704 // car-server-core/src/coder/shell_tool.rs
705 // (`format!("denied by policy: {reason}")`). An executor that
706 // *wraps* the denial (e.g. `tool 'x' failed: denied by policy: …`)
707 // deliberately falls through to `tool_error` — see the wrapped
708 // case in `action_failure_error_class_mapping_is_stable`.
709 "rejected_by_policy"
710 } else if error.contains("jcs/i-json validation")
711 || error.contains("output validation:")
712 || error.contains("invalid return json schema")
713 || error.contains("must return exact envelope {output,state_changes}")
714 || error.contains("state_changes must be an object")
715 || error.contains("state_changes keys do not match")
716 || error.contains("state_changes serialization failed")
717 {
718 "validation"
719 } else if action.tool.is_some() {
720 "tool_error"
721 } else {
722 "unknown"
723 }
724}
725
726fn action_outcome_data(
727 stage: &str,
728 reason: &str,
729 attempted: bool,
730 attempt: Option<u32>,
731) -> HashMap<String, Value> {
732 let mut data: HashMap<String, Value> = [
733 ("stage".to_string(), Value::from(stage)),
734 ("reason".to_string(), Value::from(reason)),
735 ("attempted".to_string(), Value::from(attempted)),
736 ]
737 .into();
738 if let Some(attempt) = attempt {
739 data.insert("attempt".to_string(), Value::from(attempt));
740 }
741 data
742}
743
744/// Prefix on the `error` field of an `ActionResult` that distinguishes
745/// "the user pulled the plug" from "earlier abort cascaded." The
746/// `ActionStatus` itself is `Skipped` in both cases (introducing a
747/// new variant ripples through every IR consumer + FFI binding); the
748/// prefix lets callers like the A2A bridge tell the cases apart
749/// without string-matching a magic literal.
750pub const CANCELED_PREFIX: &str = "canceled: ";
751
752/// Result for an action that didn't run because the proposal was
753/// cancelled mid-flight. Distinct from `Skipped` so callers can tell
754/// "didn't run because of an earlier abort" from "didn't run because
755/// the user pulled the plug."
756fn canceled_result(action_id: &str, reason: &str) -> ActionResult {
757 ActionResult {
758 action_id: action_id.to_string(),
759 status: ActionStatus::Skipped,
760 output: None,
761 error: Some(format!("{}{}", CANCELED_PREFIX, reason)),
762 terminal: false,
763 state_changes: HashMap::new(),
764 rolled_back: false,
765 duration_ms: None,
766 timestamp: chrono::Utc::now(),
767 }
768}
769
770/// Format a tool result for feeding back to a model.
771pub fn format_tool_result(result: &ActionResult) -> String {
772 match result.status {
773 ActionStatus::Succeeded => match &result.output {
774 Some(v) => serde_json::to_string(v).unwrap_or_else(|_| v.to_string()),
775 None => String::new(),
776 },
777 ActionStatus::Rejected => format!("[REJECTED] {}", result.error.as_deref().unwrap_or("")),
778 ActionStatus::Failed => format!("[FAILED] {}", result.error.as_deref().unwrap_or("")),
779 _ => format!(
780 "[{:?}] {}",
781 result.status,
782 result.error.as_deref().unwrap_or("")
783 ),
784 }
785}
786
787/// Names the shape [`format_tool_result_for_model`] gives a model, so a
788/// measurement can record what its model read. Change it whenever that shape
789/// changes: results measured under different formats are not comparable.
790pub const MODEL_OBSERVATION_FORMAT: &str = "text-v1";
791
792/// Fields that say how a call went; [`render_tool_output`] puts them first.
793const CONTROL_FIELDS: [&str; 6] = [
794 "truncated",
795 "error",
796 "ok",
797 "status",
798 "exit_code",
799 "timed_out",
800];
801
802/// A tool result's output as a model should read it: outcome fields first
803/// (`truncated`, `error`, `ok`, `status`, `exit_code`, `timed_out`), one line
804/// each; then other scalars as `key: value` lines; then each long or
805/// multi-line string (a file's content, a command's output) as a raw block
806/// headed `key (N bytes):`, and each array of records as `key (N records):`
807/// with one line per record. A cut at the budget loses the tail of the bulk,
808/// never the verdict.
809///
810/// Journals, receipts and events keep the JSON; only the model's copy changes.
811/// The model used to be handed the JSON too — so a file arrived as one line of
812/// `\n`, `\t` and `\"` escapes. On claude-haiku-4-5, 23% of native `edit_file`
813/// calls (42 of 182) missed with "old_text not found", against 1 of 60 for
814/// Claude Code's `Edit` on the same model and tasks; the misses carried escapes
815/// copied out of the JSON (`\"\"\"` for a docstring's `"""`). Claude Code's
816/// `Read` returns plain numbered lines; this is that shape.
817///
818/// The text reads back exactly with [`parse_rendered_output`], apart from the
819/// outcome fields (shown on one line) and a record list (read back as text):
820/// blocks carry their length, so no content line can be mistaken for a field,
821/// and a string that would read back as another JSON type (`"123"`, `"null"`)
822/// is written quoted.
823pub fn render_tool_output(v: &Value) -> String {
824 let map = match v {
825 Value::Object(map) => map,
826 Value::String(s) => return s.clone(),
827 other => return other.to_string(),
828 };
829 let mut head = Vec::new();
830 let mut lines = Vec::new();
831 let mut blocks = Vec::new();
832 // Sorted, so every build shows the same bytes: serde_json's map order
833 // depends on whether `preserve_order` is enabled anywhere in the graph.
834 let mut fields: Vec<_> = map.iter().collect();
835 fields.sort_by(|a, b| a.0.cmp(b.0));
836 for (key, value) in fields {
837 match value {
838 // Outcome fields lead, on one line each, whatever their size: a cut
839 // at the budget must never take the verdict with it.
840 _ if CONTROL_FIELDS.contains(&key.as_str()) => {
841 // Readable, not escaped: a traceback in `error` shows as one
842 // line with its breaks turned into spaces, keeping its start
843 // and its end (where the exception is) when it is long.
844 let line = match value {
845 Value::String(s) => scalar_text(&Value::String(s.replace('\n', " "))),
846 other => scalar_text(other),
847 };
848 head.push(format!(
849 "{}: {}",
850 key_text(key),
851 clip_keeping_ends(&line, CONTROL_FIELD_BYTES)
852 ));
853 }
854 Value::String(s) if s.contains('\n') || s.len() > 200 => {
855 blocks.push(format!("{} ({} bytes):\n{s}", key_text(key), s.len()))
856 }
857 Value::Array(items) if !items.is_empty() && items.iter().all(Value::is_object) => {
858 let records: Vec<String> = items.iter().map(render_record).collect();
859 blocks.push(format!(
860 "{} ({} records):\n{}",
861 key_text(key),
862 records.len(),
863 records.join("\n")
864 ));
865 }
866 other => lines.push(format!("{}: {}", key_text(key), scalar_text(other))),
867 }
868 }
869 head.extend(lines);
870 head.extend(blocks);
871 head.join("\n")
872}
873
874/// A scalar as it appears after `key: `. A string is written raw unless raw
875/// text would read back as something else — JSON of another type, a quoted
876/// string, or surrounding whitespace — in which case it is written as a JSON
877/// string. Everything else is compact JSON.
878fn scalar_text(value: &Value) -> String {
879 match value {
880 Value::String(s)
881 if s.trim() != s
882 || s.is_empty()
883 || s.contains('\n')
884 || s.ends_with("):")
885 || serde_json::from_str::<Value>(s).is_ok() =>
886 {
887 serde_json::to_string(s).unwrap_or_else(|_| s.clone())
888 }
889 Value::String(s) => s.clone(),
890 other => other.to_string(),
891 }
892}
893
894/// How much of an outcome field a model is shown on its one line.
895const CONTROL_FIELD_BYTES: usize = 500;
896
897/// A key as written before `: ` or ` (N bytes):` — raw unless raw text would
898/// not read back as this key, then as a JSON string.
899fn key_text(key: &str) -> String {
900 if key.is_empty()
901 || key.contains(char::is_whitespace)
902 || key.ends_with(':')
903 || key.contains(": ")
904 || key.contains(" (")
905 || key.contains('\n')
906 || key.starts_with('"')
907 || key.starts_with('[')
908 || key.starts_with('…')
909 {
910 serde_json::to_string(key).unwrap_or_else(|_| key.to_string())
911 } else {
912 key.to_string()
913 }
914}
915
916/// Read a key written by [`key_text`] off the front of `line`, returning it
917/// and what follows it.
918fn read_key(line: &str) -> Option<(String, &str)> {
919 if line.starts_with('"') {
920 let mut stream = serde_json::Deserializer::from_str(line).into_iter::<String>();
921 let key = stream.next()?.ok()?;
922 return Some((key, &line[stream.byte_offset()..]));
923 }
924 let end = line.find(": ").or_else(|| line.find(" ("))?;
925 let key = &line[..end];
926 // `key_text` quotes any key with whitespace or a leading `[`/`…`, so a
927 // raw one without them is a key; `[FAILED] tool 'x': …` is not.
928 (!key.is_empty()
929 && !key.contains(char::is_whitespace)
930 && !key.starts_with('[')
931 && !key.starts_with('…'))
932 .then(|| (key.to_string(), &line[end..]))
933}
934
935/// `s` in at most about `max` bytes, keeping its first third and its end,
936/// with the number of bytes dropped between them.
937fn clip_keeping_ends(s: &str, max: usize) -> String {
938 if s.len() <= max {
939 return s.to_string();
940 }
941 let mut head = max / 3;
942 while !s.is_char_boundary(head) {
943 head -= 1;
944 }
945 let mut tail = s.len() - (max - head);
946 while !s.is_char_boundary(tail) {
947 tail += 1;
948 }
949 format!("{}…[{} bytes]…{}", &s[..head], tail - head, &s[tail..])
950}
951
952/// A tool result's fields from whichever form it was stored in: the JSON a
953/// receipt holds, or the rendered text a model was shown (a transcript). Any
954/// JSON parses as itself; rendered text reads back through
955/// [`parse_rendered_output`]; `None` for anything else, such as a `[FAILED] …`
956/// line.
957pub fn parse_tool_output(text: &str) -> Option<Value> {
958 serde_json::from_str::<Value>(text)
959 .ok()
960 .or_else(|| parse_rendered_output(text))
961}
962
963/// Read [`render_tool_output`]'s text back into a JSON object — for consumers
964/// that need fields (an outcome, a receipt's structured redaction) when only
965/// the model's copy survives, such as a transcript replayed without its
966/// recorded `ok`. Exact for scalars and blocks; an outcome field reads back as
967/// the one line the model saw (breaks as spaces, long ones clipped), and a
968/// record list reads back as its text. A block cut short by a budget reads
969/// back as what is left of it, and parsing stops at the first line that is not
970/// a field, which is where a budget cut the text. `None` unless the first line
971/// is a field — a `[FAILED] …` line, a refusal or prose is not rendered output.
972pub fn parse_rendered_output(text: &str) -> Option<Value> {
973 fn block_header(line: &str) -> Option<(String, usize, bool)> {
974 let (key, rest) = read_key(line)?;
975 let count = rest.strip_prefix(" (")?.strip_suffix("):")?;
976 let (n, bytes) = if let Some(n) = count.strip_suffix(" bytes") {
977 (n, true)
978 } else {
979 (count.strip_suffix(" records")?, false)
980 };
981 Some((key, n.parse().ok()?, bytes))
982 }
983 let mut map = serde_json::Map::new();
984 let mut rest = text;
985 let mut seen_block = false;
986 while !rest.is_empty() {
987 let (line, after) = rest.split_once('\n').unwrap_or((rest, ""));
988 if let Some((key, n, bytes)) = block_header(line) {
989 let (value, remaining) = if bytes {
990 let mut end = n.min(after.len());
991 while !after.is_char_boundary(end) {
992 end -= 1;
993 }
994 (
995 &after[..end],
996 after[end..].strip_prefix('\n').unwrap_or(&after[end..]),
997 )
998 } else {
999 let mut end = 0;
1000 let mut taken = 0;
1001 while taken < n && end < after.len() {
1002 end = after[end..].find('\n').map_or(after.len(), |i| end + i + 1);
1003 taken += 1;
1004 }
1005 (after[..end].trim_end_matches('\n'), &after[end..])
1006 };
1007 // First wins: the renderer writes each key once, so a second is
1008 // text a cut spliced in, and must not replace an outcome field.
1009 map.entry(key)
1010 .or_insert_with(|| Value::String(value.to_string()));
1011 seen_block = true;
1012 rest = remaining;
1013 continue;
1014 }
1015 match read_key(line) {
1016 Some((key, raw)) if !seen_block && raw.starts_with(": ") => {
1017 let raw = &raw[2..];
1018 let value = serde_json::from_str::<Value>(raw)
1019 .unwrap_or_else(|_| Value::String(raw.to_string()));
1020 map.entry(key).or_insert(value);
1021 }
1022 // The first line decides what this is: a `[FAILED] …` line, a
1023 // refusal or prose is not rendered output and is not read as
1024 // fields. After a field, an unrecognised line is where a budget
1025 // cut the text (mid-line, or through a marker spliced into a
1026 // block): keep what came before it — above all the outcome
1027 // fields, which the renderer writes first.
1028 _ if map.is_empty() => return None,
1029 _ => break,
1030 }
1031 rest = after;
1032 }
1033 (!map.is_empty()).then_some(Value::Object(map))
1034}
1035
1036/// One record of an array field (a grep match, a directory entry) on one line,
1037/// its string values raw.
1038fn render_record(item: &Value) -> String {
1039 let Value::Object(fields) = item else {
1040 return item.to_string();
1041 };
1042 let mut fields: Vec<_> = fields.iter().collect();
1043 fields.sort_by(|a, b| a.0.cmp(b.0));
1044 fields
1045 .into_iter()
1046 .map(|(k, v)| {
1047 let k = k.replace('\n', "\\n");
1048 match v {
1049 Value::String(s) => format!("{k}={}", s.replace('\n', "\\n")),
1050 other => format!("{k}={other}"),
1051 }
1052 })
1053 .collect::<Vec<_>>()
1054 .join(" ")
1055}
1056
1057/// The phrase [`truncate_keeping_ends`] puts in its elision marker, for
1058/// readers that count cut observations.
1059pub const ELIDED_MARKER: &str = "bytes elided from the middle";
1060
1061/// Fit `s` in `max` bytes keeping its first third and its end, with a marker
1062/// saying how much of the middle was dropped. For command output: the verdict
1063/// a test run or build prints comes last, so a head-only cut loses exactly the
1064/// line the model needs.
1065pub fn truncate_keeping_ends(s: &str, max: usize) -> String {
1066 if s.len() <= max {
1067 return s.to_string();
1068 }
1069 if max < 512 {
1070 // Too small for a head, a tail and the marker: keep the end, the part
1071 // that carries the verdict.
1072 let mut start = s.len() - max;
1073 while !s.is_char_boundary(start) {
1074 start += 1;
1075 }
1076 return s[start..].to_string();
1077 }
1078 let budget = max.saturating_sub(128);
1079 let mut head = budget / 3;
1080 while !s.is_char_boundary(head) {
1081 head -= 1;
1082 }
1083 let mut tail = s.len() - (budget - head);
1084 while !s.is_char_boundary(tail) {
1085 tail += 1;
1086 }
1087 format!(
1088 "{}\n…[{} {ELIDED_MARKER} of {}; re-run with narrower output to see them]…\n{}",
1089 &s[..head],
1090 tail - head,
1091 s.len(),
1092 &s[tail..]
1093 )
1094}
1095
1096/// Whether a tool's result is command output, whose end matters most: the
1097/// tools every CAR agent loop truncates with [`truncate_keeping_ends`].
1098pub fn is_command_tool(name: &str) -> bool {
1099 matches!(name, "shell" | "run_command")
1100}
1101
1102/// [`format_tool_result`] as a model should read it: a successful result's
1103/// output goes through [`render_tool_output`] instead of being serialized as
1104/// JSON; failures keep their `[FAILED]` / `[REJECTED]` prefix.
1105pub fn format_tool_result_for_model(result: &ActionResult) -> String {
1106 match (&result.status, &result.output) {
1107 (ActionStatus::Succeeded, Some(v)) => render_tool_output(v),
1108 _ => format_tool_result(result),
1109 }
1110}
1111
1112/// Budget constraints for proposal execution.
1113#[derive(Debug, Clone)]
1114pub struct CostBudget {
1115 pub max_tool_calls: Option<u32>,
1116 pub max_duration_ms: Option<f64>,
1117 pub max_actions: Option<u32>,
1118}
1119
1120/// Common Agent Runtime — deterministic execution layer.
1121///
1122/// Lock ordering discipline (never hold multiple simultaneously, never hold sync locks across .await):
1123/// 1. capabilities (RwLock, read-only during execution)
1124/// 2. tools (RwLock, read-only during execution)
1125/// 3. policies (RwLock, read-only during execution)
1126/// 4. session_policies (RwLock to find the per-session engine, then read inner Arc)
1127/// 5. cost_budget (RwLock, read-only during execution)
1128/// 6. log (TokioMutex, acquired/released per event)
1129/// 7. tool_executor (TokioMutex, clone Arc and drop before await)
1130/// 8. idempotency_cache (TokioMutex, acquired/released per check)
1131///
1132/// StateStore uses parking_lot::Mutex (sync) — NEVER hold across .await points.
1133pub struct Runtime {
1134 pub state: Arc<StateStore>,
1135 pub tools: Arc<TokioRwLock<HashMap<String, ToolSchema>>>,
1136 pub policies: Arc<TokioRwLock<PolicyEngine>>,
1137 /// Per-session policy registries. Hosts that multiplex multiple
1138 /// concurrent agent sessions over a single `Runtime` (IDE-style
1139 /// frontends with per-project rules, multi-tenant servers) call
1140 /// [`Runtime::open_session`] to mint an id, then
1141 /// [`Runtime::register_policy_in_session`] to attach session-scoped
1142 /// rules. Validation under that session walks the global registry
1143 /// AND the session's — both must pass. Sessions can deny what
1144 /// global allows; sessions cannot allow what global denies.
1145 /// Closing a session drops its registry and any closures it holds.
1146 /// See `docs/proposals/per-session-policy-scoping.md`.
1147 pub session_policies: Arc<TokioRwLock<HashMap<String, Arc<TokioRwLock<PolicyEngine>>>>>,
1148 pub log: Arc<TokioMutex<EventLog>>,
1149 pub rate_limiter: Arc<RateLimiter>,
1150 pub result_cache: Arc<ResultCache>,
1151 /// Per-execution-session read ledgers backing the read-before-edit / staleness guard on
1152 /// the built-in file tools (H1/F4-remainder, audit 2026-07-06). Threaded into
1153 /// `agent_basics::execute_with_ledger` on the built-in fallback path so a raw
1154 /// Runtime session (no configured executor) still requires reading a file
1155 /// before editing or overwriting it. Configured executors
1156 /// (coder/assistant/bench) own their own ledger, so this one only governs the
1157 /// built-in fallback path.
1158 read_ledgers: crate::agent_basics::SessionReadLedgers,
1159 tool_executor: TokioMutex<Option<Arc<dyn ToolExecutor>>>,
1160 idempotency_cache: TokioMutex<HashMap<String, ActionResult>>,
1161 cost_budget: TokioRwLock<Option<CostBudget>>,
1162 capabilities: TokioRwLock<Option<CapabilitySet>>,
1163 inference_engine: Option<Arc<car_inference::InferenceEngine>>,
1164 /// Optional outbound transport backing the `messaging.send` built-in.
1165 /// `None` by default — a runtime with no sink refuses `messaging.send`
1166 /// outright rather than falling through to the host executor, and never
1167 /// advertises the tool. Attach one with
1168 /// [`Runtime::with_message_sink`]; see [`crate::messaging`] for why
1169 /// human-directed messaging belongs inside the governed chain at all.
1170 message_sink: Option<Arc<dyn crate::messaging::MessageSink>>,
1171 /// Optional memgine for skill learning (auto-distillation after execution).
1172 memgine: Option<Arc<TokioMutex<car_memgine::MemgineEngine>>>,
1173 /// Whether to auto-distill skills after each proposal execution.
1174 auto_distill: bool,
1175 /// Optional trajectory store for persisting execution traces.
1176 trajectory_store: Option<Arc<car_memgine::TrajectoryStore>>,
1177 /// Optional replan callback for failure recovery.
1178 replan_callback: TokioMutex<Option<Arc<dyn ReplanCallback>>>,
1179 /// Replan configuration.
1180 replan_config: TokioRwLock<ReplanConfig>,
1181 /// Pre-execution transactional conflict check mode (survey §4.3/§5.2.4).
1182 /// `Off` by default. When `Warn`/`Strict`, each proposal is checked
1183 /// against the versioned shared state before execution; `Strict`
1184 /// rejects on conflict. See [`Runtime::set_transaction_check_mode`].
1185 transaction_check: TokioRwLock<TransactionCheckMode>,
1186 /// The live harness operating config the Evolution Agent tunes (survey
1187 /// §3.5). `None` until one is installed via [`Runtime::set_harness_config`]
1188 /// — so default behavior is byte-identical to before (no cap on
1189 /// per-action retries, built-in backoff). Once installed, the runtime
1190 /// *reads* it: `max_retries` caps per-action retry budgets and
1191 /// `retry_backoff_ms` sets the inter-attempt delay; the setter also maps
1192 /// `planning_max_replans` onto the replan config. This is what makes an
1193 /// applied [`car_memgine::HarnessConfigPatch`] take effect.
1194 harness_config: TokioRwLock<Option<car_memgine::HarnessConfig>>,
1195 /// Canonical tool registry (optional — new code should use this).
1196 pub registry: Arc<crate::registry::ToolRegistry>,
1197 /// The environment the agent's side-effecting built-in tools act within.
1198 /// Defaults to [`crate::substrate::LocalSubstrate`] (host fs/process), which
1199 /// reproduces the historic `agent_basics` host behavior byte-for-byte.
1200 /// Bind a different environment (e.g. a VM via `McpSubstrate`) with
1201 /// [`Runtime::with_substrate`] / [`Runtime::set_substrate`]. `calculate`
1202 /// stays pure and never consults the substrate.
1203 substrate: TokioRwLock<Arc<dyn crate::substrate::Substrate>>,
1204 /// Refusal applied to the host paths the built-in file tools touch while
1205 /// the substrate is local. `None` by default, so a runtime that installs
1206 /// none behaves exactly as before. See [`Runtime::set_local_path_guard`].
1207 local_path_guard: TokioRwLock<Option<crate::agent_basics::LocalPathGuard>>,
1208 /// The folder the built-in `write_file` / `edit_file` must stay inside,
1209 /// when the session declared one (`agents.register_basics { root }`,
1210 /// car#3406). None (the default) is the long-standing behavior.
1211 basics_root: TokioRwLock<Option<std::path::PathBuf>>,
1212 /// Proposal-admission gates — the pre-execution safety seam (EPIC A /
1213 /// task A1). Each registered [`crate::admission::AdmissionGate`] runs
1214 /// during admission, before any action executes; a proposal that any
1215 /// gate blocks (or escalates to approval) is refused. Empty by default,
1216 /// so a runtime that registers no gates behaves exactly as before.
1217 /// Individual gates (information-flow, concurrency, blocking-policy)
1218 /// are layered on via [`Runtime::register_admission_gate`].
1219 admission_gates: TokioRwLock<Vec<Arc<dyn crate::admission::AdmissionGate>>>,
1220 /// Runtime taint provenance for the VIGIL intent gate
1221 /// ([`crate::taint::TaintLedger`]). Installed by
1222 /// [`Runtime::install_intent_gate`] and by nothing else: with no intent
1223 /// gate there is no ledger, so a runtime that never configures VIGIL
1224 /// pays no per-action cost and behaves exactly as before. When present,
1225 /// each successful action records which state keys its result wrote and
1226 /// whether that result was tainted, and the gate reads it back at
1227 /// admission to mark actions that READ a tainted key — the cross-
1228 /// proposal (replan) and trusted-tool-laundering cases the proposal-
1229 /// local dependency DAG cannot see.
1230 taint_ledger: TokioRwLock<Option<Arc<crate::taint::TaintLedger>>>,
1231 /// Durable human-in-the-loop approval ledger (EPIC A / A7). When set,
1232 /// an admission gate's `NeedsApproval` verdict is resolved against this
1233 /// ledger by fingerprint: a prior Approved decision admits the
1234 /// proposal, a Rejected decision blocks it, an unseen one stays pending
1235 /// (fail-closed). `None` by default — escalations fail closed with an
1236 /// explanatory reason until a ledger is installed.
1237 approval_ledger: TokioRwLock<Option<ApprovalLedger>>,
1238 /// Optional JSONL journal backing the idempotency cache (EPIC A / C3).
1239 /// `None` by default — the cache is in-memory and lost on restart. When
1240 /// set via [`Runtime::set_idempotency_cache_path`], idempotent results
1241 /// are persisted and reloaded so a crash-restart doesn't re-execute a
1242 /// completed idempotent action (avoiding duplicate side effects).
1243 idempotency_journal: TokioRwLock<Option<std::path::PathBuf>>,
1244 /// Detached tool invocations (C2): a `ToolCall` with a
1245 /// `streaming`/`long_running` invocation mode is registered here and
1246 /// its handle returned as the action's output while the DAG proceeds.
1247 /// Chunks are drained via [`Runtime::tool_poll`], cancelled via
1248 /// [`Runtime::tool_cancel`], and fanned out to
1249 /// [`Runtime::subscribe_tool_events`] subscribers.
1250 pub tool_handles: Arc<crate::tool_handles::ToolHandleRegistry>,
1251}
1252
1253/// A durable idempotency-cache record (C3). `result: None` is a tombstone
1254/// recorded when a cached entry is invalidated by a rollback.
1255#[derive(serde::Serialize, serde::Deserialize)]
1256struct IdempotencyEntry {
1257 key: String,
1258 result: Option<ActionResult>,
1259}
1260
1261impl Runtime {
1262 pub fn new() -> Self {
1263 Self {
1264 state: Arc::new(StateStore::new()),
1265 tools: Arc::new(TokioRwLock::new(HashMap::new())),
1266 policies: Arc::new(TokioRwLock::new(PolicyEngine::new())),
1267 session_policies: Arc::new(TokioRwLock::new(HashMap::new())),
1268 log: Arc::new(TokioMutex::new(EventLog::new())),
1269 rate_limiter: Arc::new(RateLimiter::new()),
1270 result_cache: Arc::new(ResultCache::new()),
1271 read_ledgers: crate::agent_basics::SessionReadLedgers::new(),
1272 tool_executor: TokioMutex::new(None),
1273 idempotency_cache: TokioMutex::new(HashMap::new()),
1274 cost_budget: TokioRwLock::new(None),
1275 capabilities: TokioRwLock::new(None),
1276 inference_engine: None,
1277 message_sink: None,
1278 memgine: None,
1279 auto_distill: false,
1280 trajectory_store: None,
1281 replan_callback: TokioMutex::new(None),
1282 replan_config: TokioRwLock::new(ReplanConfig::default()),
1283 transaction_check: TokioRwLock::new(TransactionCheckMode::Off),
1284 harness_config: TokioRwLock::new(None),
1285 registry: Arc::new(crate::registry::ToolRegistry::new()),
1286 substrate: TokioRwLock::new(Arc::new(crate::substrate::LocalSubstrate::new())),
1287 local_path_guard: TokioRwLock::new(None),
1288 basics_root: TokioRwLock::new(None),
1289 admission_gates: TokioRwLock::new(Vec::new()),
1290 taint_ledger: TokioRwLock::new(None),
1291 approval_ledger: TokioRwLock::new(None),
1292 idempotency_journal: TokioRwLock::new(None),
1293 tool_handles: Arc::new(crate::tool_handles::ToolHandleRegistry::new()),
1294 }
1295 }
1296
1297 /// Create a runtime with shared state, event log, and policies.
1298 /// Each runtime gets its own tool set, executor, and idempotency cache.
1299 pub fn with_shared(
1300 state: Arc<StateStore>,
1301 log: Arc<TokioMutex<EventLog>>,
1302 policies: Arc<TokioRwLock<PolicyEngine>>,
1303 ) -> Self {
1304 Self {
1305 state,
1306 tools: Arc::new(TokioRwLock::new(HashMap::new())),
1307 policies,
1308 // Session-policy registries are per-runtime — sharing them
1309 // across embedders that share global policies would defeat
1310 // the isolation point. Hosts that genuinely want shared
1311 // sessions should drive them through one shared Runtime.
1312 session_policies: Arc::new(TokioRwLock::new(HashMap::new())),
1313 log,
1314 rate_limiter: Arc::new(RateLimiter::new()),
1315 result_cache: Arc::new(ResultCache::new()),
1316 read_ledgers: crate::agent_basics::SessionReadLedgers::new(),
1317 tool_executor: TokioMutex::new(None),
1318 idempotency_cache: TokioMutex::new(HashMap::new()),
1319 cost_budget: TokioRwLock::new(None),
1320 capabilities: TokioRwLock::new(None),
1321 inference_engine: None,
1322 message_sink: None,
1323 memgine: None,
1324 auto_distill: false,
1325 trajectory_store: None,
1326 replan_callback: TokioMutex::new(None),
1327 replan_config: TokioRwLock::new(ReplanConfig::default()),
1328 transaction_check: TokioRwLock::new(TransactionCheckMode::Off),
1329 harness_config: TokioRwLock::new(None),
1330 registry: Arc::new(crate::registry::ToolRegistry::new()),
1331 substrate: TokioRwLock::new(Arc::new(crate::substrate::LocalSubstrate::new())),
1332 local_path_guard: TokioRwLock::new(None),
1333 basics_root: TokioRwLock::new(None),
1334 admission_gates: TokioRwLock::new(Vec::new()),
1335 taint_ledger: TokioRwLock::new(None),
1336 approval_ledger: TokioRwLock::new(None),
1337 idempotency_journal: TokioRwLock::new(None),
1338 tool_handles: Arc::new(crate::tool_handles::ToolHandleRegistry::new()),
1339 }
1340 }
1341
1342 // ─── Session policy lifecycle ───────────────────────────────────
1343 //
1344 // Per-session policy scoping. Policies registered against a
1345 // session apply to proposals executed under that session id (via
1346 // [`Self::execute_with_session`] / [`Self::execute_with_session_and_cancel`]).
1347 // Policies registered globally always apply, on top of any
1348 // session-scoped layer. See `docs/proposals/per-session-policy-scoping.md`.
1349
1350 /// Mint a new session id and pre-register an empty policy engine
1351 /// under it. Hosts call this once per concurrent agent context
1352 /// (an IDE project window, a multi-tenant client, etc.) and pair
1353 /// it with [`Self::close_session`] when the context ends.
1354 ///
1355 /// Returns the opaque id to pass to subsequent
1356 /// [`Self::register_policy_in_session`] / [`Self::execute_with_session`]
1357 /// calls. Ids are UUIDs so collisions across concurrent calls
1358 /// don't matter.
1359 pub async fn open_session(&self) -> String {
1360 let id = Uuid::new_v4().to_string();
1361 let mut sessions = self.session_policies.write().await;
1362 sessions.insert(id.clone(), Arc::new(TokioRwLock::new(PolicyEngine::new())));
1363 self.read_ledgers.ledger_for(Some(&id));
1364 id
1365 }
1366
1367 /// Drop the session and every policy scoped to it. Returns true
1368 /// if a session by that id existed; false if it didn't (already
1369 /// closed, never opened, etc.). Idempotent in effect — closing a
1370 /// missing session is a no-op the caller is free to ignore.
1371 pub async fn close_session(&self, session_id: &str) -> bool {
1372 let mut sessions = self.session_policies.write().await;
1373 let removed = sessions.remove(session_id).is_some();
1374 if removed {
1375 self.read_ledgers.remove(session_id);
1376 }
1377 removed
1378 }
1379
1380 /// Register a policy under a specific session id. The policy
1381 /// applies only when a proposal is executed under that session;
1382 /// proposals executed without a session (the default) only see
1383 /// global policies.
1384 ///
1385 /// Returns `Err(...)` if the session is unknown — callers either
1386 /// forgot to call [`Self::open_session`] or are using a
1387 /// stale/closed id.
1388 pub async fn register_policy_in_session(
1389 &self,
1390 session_id: &str,
1391 name: &str,
1392 check: car_policy::PolicyCheck,
1393 description: &str,
1394 ) -> Result<(), String> {
1395 let engine = {
1396 let sessions = self.session_policies.read().await;
1397 sessions
1398 .get(session_id)
1399 .cloned()
1400 .ok_or_else(|| format!("unknown session id '{session_id}'"))?
1401 };
1402 let mut engine = engine.write().await;
1403 engine.register(name, check, description);
1404 Ok(())
1405 }
1406
1407 /// [`Self::register_policy_in_session`] for a check that forbids `tool`
1408 /// outright, so `PolicyEngine::blanket_denied_tools` can read it back.
1409 /// Separate method rather than an extra parameter: the existing signature
1410 /// is public and crosses the bindings, and this is purely additive.
1411 pub async fn register_tool_deny_in_session(
1412 &self,
1413 session_id: &str,
1414 name: &str,
1415 tool: &str,
1416 check: car_policy::PolicyCheck,
1417 description: &str,
1418 ) -> Result<(), String> {
1419 let engine = {
1420 let sessions = self.session_policies.read().await;
1421 sessions
1422 .get(session_id)
1423 .cloned()
1424 .ok_or_else(|| format!("unknown session id '{session_id}'"))?
1425 };
1426 let mut engine = engine.write().await;
1427 engine.register_tool_deny(name, tool, check, description);
1428 Ok(())
1429 }
1430
1431 /// Remove a policy by name. `session_id` targets a session's policy set;
1432 /// `None` targets the global one.
1433 ///
1434 /// Returns how many policies were dropped, or `Err` if `session_id` names a
1435 /// session that doesn't exist. Session-scoped policies could always be
1436 /// dropped wholesale by [`Self::close_session`], but a *global* policy had
1437 /// no removal path at all — once registered it lived until the process
1438 /// exited, so a mistyped or over-broad rule could only be cleared by
1439 /// restarting the daemon (Parslee-ai/car#623).
1440 pub async fn unregister_policy(
1441 &self,
1442 name: &str,
1443 session_id: Option<&str>,
1444 ) -> Result<usize, String> {
1445 match session_id {
1446 Some(sid) => {
1447 let engine = {
1448 let sessions = self.session_policies.read().await;
1449 sessions
1450 .get(sid)
1451 .cloned()
1452 .ok_or_else(|| format!("unknown session id '{sid}'"))?
1453 };
1454 let mut engine = engine.write().await;
1455 Ok(engine.unregister(name))
1456 }
1457 None => {
1458 let mut engine = self.policies.write().await;
1459 Ok(engine.unregister(name))
1460 }
1461 }
1462 }
1463
1464 /// Registered policies as `(name, description)`. `session_id` lists a
1465 /// session's set; `None` lists the global one. Without this a client could
1466 /// register a policy but never ask what was in force, so a rejection could
1467 /// not be explained beyond its single message (Parslee-ai/car#623).
1468 pub async fn list_policies(
1469 &self,
1470 session_id: Option<&str>,
1471 ) -> Result<Vec<(String, String)>, String> {
1472 match session_id {
1473 Some(sid) => {
1474 let engine = {
1475 let sessions = self.session_policies.read().await;
1476 sessions
1477 .get(sid)
1478 .cloned()
1479 .ok_or_else(|| format!("unknown session id '{sid}'"))?
1480 };
1481 let engine = engine.read().await;
1482 Ok(engine.policy_details())
1483 }
1484 None => {
1485 let engine = self.policies.read().await;
1486 Ok(engine.policy_details())
1487 }
1488 }
1489 }
1490
1491 /// Load declarative deny rules from a project's `.car/policies/`
1492 /// directory and register them on the global policy engine (EPIC A /
1493 /// task A2).
1494 ///
1495 /// `car_dir` is a `.car` directory; this looks for
1496 /// `car_dir/policies/*.toml`. **The caller chooses the directory and
1497 /// there is no walk-up here** — this joins the path it is given, once.
1498 /// The two production callers pick differently: the daemon passes
1499 /// `$HOME/.car`, and the assistant passes its working directory's
1500 /// `.car` (`--dir`, else cwd). So a rule file at a repository root does
1501 /// not govern a `car do` run started from a subdirectory. Say which
1502 /// directory you mean at the call site rather than assuming discovery.
1503 ///
1504 /// A missing directory is not an error (returns 0). A malformed rule
1505 /// file *is* an error — a dropped security rule must surface loudly.
1506 /// Returns the number of rules registered.
1507 ///
1508 /// These rules are additive on top of any code-registered policies, and
1509 /// every one of them is a prohibition — though `allow_tool_param` states
1510 /// its prohibition as an allowlist, denying everything about its tool
1511 /// that it does not name. Once A9 makes policy violations blocking at
1512 /// admission, a matching action refuses the proposal.
1513 pub async fn load_project_policies(
1514 &self,
1515 car_dir: impl AsRef<std::path::Path>,
1516 ) -> Result<usize, car_policy::PolicyLoadError> {
1517 let dir = car_dir.as_ref().join("policies");
1518 let rules = car_policy::load_policy_dir(&dir)?;
1519 // `PolicyRules::len` counts every kind. Summing a hand-picked subset
1520 // here is what made this under-report once already — it covered the
1521 // three kinds that existed at the time and was not revisited when
1522 // more landed, so a file full of allowlists reported zero rules
1523 // loaded while `apply` registered all of them.
1524 let count = rules.len();
1525 let mut engine = self.policies.write().await;
1526 rules.apply(&mut engine);
1527 Ok(count)
1528 }
1529
1530 /// Load information-flow tool labels from a project's `.car` directory
1531 /// and register the information-flow admission gate (EPIC A / A3+A4).
1532 ///
1533 /// Reads `car_dir/tool-labels.json` (merged over built-in defaults) and
1534 /// registers an [`crate::flow::InformationFlowGate`] so every admitted
1535 /// proposal is checked for data exfiltration (blocked) and forbidden
1536 /// tool orderings (escalated to approval). A missing labels file is
1537 /// fine — the built-in defaults still mark the network tools as sinks.
1538 /// A malformed file is a loud error.
1539 pub async fn install_information_flow_gate(
1540 &self,
1541 car_dir: impl AsRef<std::path::Path>,
1542 ) -> Result<(), crate::flow::FlowLoadError> {
1543 let config = crate::flow::load_tool_labels(car_dir)?;
1544 let gate = Arc::new(crate::flow::InformationFlowGate::new(config));
1545 self.register_admission_gate(gate).await;
1546 Ok(())
1547 }
1548
1549 /// True if a session with this id is currently open. Mostly for
1550 /// tests and FFI surface validation — production code should
1551 /// trust the id it just opened.
1552 pub async fn session_exists(&self, session_id: &str) -> bool {
1553 self.session_policies.read().await.contains_key(session_id)
1554 }
1555
1556 /// Attach a local inference engine. Registers `infer`, `embed`, `classify`
1557 /// as built-in tools with real implementations.
1558 pub fn with_inference(mut self, engine: Arc<car_inference::InferenceEngine>) -> Self {
1559 self.inference_engine = Some(engine);
1560 // Register inference tool schemas (non-async init, use try_lock).
1561 // Keep the canonical registry populated too: execution-event provenance
1562 // is read from ToolEntry.source rather than inferred from a tool name.
1563 //
1564 // Contention contract: this runs at construction time on a private
1565 // runtime, so both try_locks are expected to succeed. If either fails
1566 // the two stores would disagree (validator reads `tools`, provenance
1567 // reads the registry first) — loud in debug, logged in release.
1568 if let Ok(mut tools) = self.tools.try_write() {
1569 for schema in car_inference::service::all_schemas() {
1570 let name = schema.name.clone();
1571 let entry = crate::registry::ToolEntry::builtin(schema);
1572 tools.insert(name.clone(), entry.schema.clone());
1573 if !self.registry.try_register(entry) {
1574 tracing::warn!(
1575 tool = %name,
1576 "with_inference: canonical registry contention; \
1577 schema map registered the inference tool but the \
1578 registry did not — provenance falls back to the map"
1579 );
1580 }
1581 }
1582 } else {
1583 debug_assert!(
1584 false,
1585 "with_inference must populate both stores; schema-map lock was contended \
1586 at construction time — inference tools stay unregistered"
1587 );
1588 tracing::warn!(
1589 "with_inference: schema map lock contended at construction; \
1590 inference tools were NOT registered"
1591 );
1592 }
1593 self
1594 }
1595
1596 /// Attach an outbound message sink, making `messaging.send` a real tool on
1597 /// this runtime.
1598 ///
1599 /// Registering the schema here — and ONLY here — is deliberate. A tool the
1600 /// runtime cannot execute must not be advertised to the model: without a
1601 /// sink the dispatch arm can only return an error, and a model that keeps
1602 /// seeing "message the human" in its tool list will keep trying to use it.
1603 /// So sink and schema arrive together, and neither exists alone.
1604 ///
1605 /// The entry is `AskUser` with side effects: reaching a human is
1606 /// irreversible, so the conservative default is the right one, and hosts
1607 /// that have their own consent model can re-register with a different
1608 /// permission. `category = "messaging"` groups it for the capability
1609 /// surfaces that filter by category.
1610 ///
1611 /// Follows [`Self::with_inference`]'s non-async registration pattern
1612 /// (`try_write` during construction, while the locks are uncontended) —
1613 /// making the builders async would break every `Runtime::new().with_…()`
1614 /// chain in the workspace for no gain.
1615 pub fn with_message_sink(mut self, sink: Arc<dyn crate::messaging::MessageSink>) -> Self {
1616 self.message_sink = Some(sink);
1617
1618 let entry = crate::registry::ToolEntry::builtin(car_ir::builtins::messaging_send())
1619 .with_permission(crate::registry::ToolPermission::AskUser)
1620 .with_side_effects(true)
1621 .with_category("messaging");
1622 let schema = entry.schema.clone();
1623
1624 // Honour the schema's declared rate limit, the way the async
1625 // `register_tool_schema` path does — otherwise the backstop on
1626 // outbound human messaging would silently not exist.
1627 if let Some(ref rl) = schema.rate_limit {
1628 self.rate_limiter.try_set_limit(
1629 &schema.name,
1630 RateLimit {
1631 max_calls: rl.max_calls,
1632 interval_secs: rl.interval_secs,
1633 },
1634 );
1635 }
1636 self.registry.try_register(entry);
1637 // The legacy schema map is what `validate_action` reads, so the
1638 // validator can only check `messaging.send` parameters once it lands
1639 // here.
1640 if let Ok(mut tools) = self.tools.try_write() {
1641 tools.insert(schema.name.clone(), schema);
1642 }
1643 self
1644 }
1645
1646 /// Attach a memgine for automatic skill learning after execution.
1647 /// When `auto_distill` is true, execution traces are automatically distilled
1648 /// into skills and domains are evolved when underperforming.
1649 pub fn with_learning(
1650 mut self,
1651 memgine: Arc<TokioMutex<car_memgine::MemgineEngine>>,
1652 auto_distill: bool,
1653 ) -> Self {
1654 self.memgine = Some(memgine);
1655 self.auto_distill = auto_distill;
1656 self
1657 }
1658
1659 /// Attach a memgine with auto-distillation enabled (recommended default).
1660 pub fn with_memgine(self, memgine: Arc<TokioMutex<car_memgine::MemgineEngine>>) -> Self {
1661 self.with_learning(memgine, true)
1662 }
1663
1664 /// Attach a trajectory store for persisting execution traces.
1665 pub fn with_trajectory_store(mut self, store: Arc<car_memgine::TrajectoryStore>) -> Self {
1666 self.trajectory_store = Some(store);
1667 self
1668 }
1669
1670 /// The attached trajectory store, if any.
1671 ///
1672 /// Writing traces was always the point of the store; reading them back is
1673 /// what makes them a feedback signal rather than an audit log.
1674 pub fn trajectory_store(&self) -> Option<&Arc<car_memgine::TrajectoryStore>> {
1675 self.trajectory_store.as_ref()
1676 }
1677
1678 /// Per-tool success rates observed over the last `window_days`, or `None`
1679 /// when no trajectory store is attached.
1680 ///
1681 /// Dispatch-conditional (see
1682 /// [`ToolFeedback::dispatched_from_trajectories`](car_planner::ToolFeedback::dispatched_from_trajectories))
1683 /// — rejected and skipped actions are excluded, because the tool never ran
1684 /// and a consumer that models rejection separately would otherwise count
1685 /// it twice.
1686 ///
1687 /// The window exists because a success rate is a claim about how a tool
1688 /// behaves *now*. An API that was broken for a week six months ago and has
1689 /// been fixed since should not still be dragging its own rate down, and an
1690 /// unbounded history makes the rate progressively less responsive to
1691 /// exactly the recent change an operator is trying to see. 30 days is the
1692 /// suggested default at the call sites.
1693 ///
1694 /// Reads from disk on every call — parsing is bounded by the window (day
1695 /// files outside it are skipped by filename, never opened), and the callers
1696 /// are interactive rather than hot-path. If that stops being true, cache
1697 /// here rather than at each call site.
1698 pub fn tool_feedback(&self, window_days: u32) -> Option<car_planner::ToolFeedback> {
1699 let store = self.trajectory_store.as_ref()?;
1700 let cutoff = chrono::Utc::now() - chrono::Duration::days(i64::from(window_days));
1701 let trajectories = store.load_since(cutoff);
1702 Some(car_planner::ToolFeedback::dispatched_from_trajectories(
1703 &trajectories,
1704 ))
1705 }
1706
1707 pub fn with_executor(self, executor: Arc<dyn ToolExecutor>) -> Self {
1708 // Use try_lock for non-async init context. Safe because we just created the mutex.
1709 if let Ok(mut guard) = self.tool_executor.try_lock() {
1710 *guard = Some(executor);
1711 }
1712 self
1713 }
1714
1715 /// Set a tool executor for the next execute() call.
1716 /// Used by NAPI bindings where executor varies per call.
1717 pub async fn set_executor(&self, executor: Arc<dyn ToolExecutor>) {
1718 *self.tool_executor.lock().await = Some(executor);
1719 }
1720
1721 /// Bind the execution substrate the side-effecting built-in tools
1722 /// (`read_file`/`write_file`/`edit_file`/`list_dir`/`find_files`/
1723 /// `grep_files`) act within (builder). Defaults to
1724 /// [`crate::substrate::LocalSubstrate`]; bind e.g.
1725 /// [`crate::substrate::McpSubstrate`] to make those tools hit a VM.
1726 /// `calculate` stays pure and ignores the substrate.
1727 pub fn with_substrate(self, substrate: Arc<dyn crate::substrate::Substrate>) -> Self {
1728 // Use try_write for non-async init context. Safe because we just created the lock.
1729 if let Ok(mut guard) = self.substrate.try_write() {
1730 *guard = substrate;
1731 }
1732 self
1733 }
1734
1735 /// Set the execution substrate at runtime.
1736 pub async fn set_substrate(&self, substrate: Arc<dyn crate::substrate::Substrate>) {
1737 {
1738 *self.substrate.write().await = substrate;
1739 }
1740 self.read_ledgers.clear();
1741 }
1742
1743 /// Clone the currently bound substrate.
1744 pub async fn substrate(&self) -> Arc<dyn crate::substrate::Substrate> {
1745 self.substrate.read().await.clone()
1746 }
1747
1748 pub fn with_event_log(mut self, log: EventLog) -> Self {
1749 self.log = Arc::new(TokioMutex::new(log));
1750 self
1751 }
1752
1753 /// Bind an event log the caller already holds a handle to.
1754 ///
1755 /// [`Self::with_event_log`] takes ownership, which leaves no way for
1756 /// anything outside the runtime to read what was recorded. That was fine
1757 /// while the log was observability-only, but a model-callable events
1758 /// surface has to run inside a `ToolExecutor` — and the executor is
1759 /// constructed *before* the runtime that would own it, so the two can only
1760 /// meet through a handle created first and shared into both
1761 /// (Parslee-ai/car#815).
1762 pub fn with_shared_event_log(mut self, log: Arc<TokioMutex<EventLog>>) -> Self {
1763 self.log = log;
1764 self
1765 }
1766
1767 /// The runtime's event log handle, for callers that need to read the record
1768 /// the runtime is writing.
1769 pub fn event_log_handle(&self) -> Arc<TokioMutex<EventLog>> {
1770 Arc::clone(&self.log)
1771 }
1772
1773 /// Attach a replan callback for failure recovery (builder).
1774 pub fn with_replan(self, callback: Arc<dyn ReplanCallback>, config: ReplanConfig) -> Self {
1775 if let Ok(mut guard) = self.replan_callback.try_lock() {
1776 *guard = Some(callback);
1777 }
1778 if let Ok(mut guard) = self.replan_config.try_write() {
1779 *guard = config;
1780 }
1781 self
1782 }
1783
1784 /// Set a replan callback at runtime.
1785 pub async fn set_replan_callback(&self, callback: Arc<dyn ReplanCallback>) {
1786 *self.replan_callback.lock().await = Some(callback);
1787 }
1788
1789 /// Set replan configuration at runtime.
1790 pub async fn set_replan_config(&self, config: ReplanConfig) {
1791 *self.replan_config.write().await = config;
1792 }
1793
1794 /// Set the pre-execution transactional conflict-check mode (survey
1795 /// §4.3/§5.2.4). `Off` (default) preserves prior behavior; `Warn`
1796 /// records conflicts as telemetry; `Strict` rejects a conflicting
1797 /// proposal before executing it.
1798 pub async fn set_transaction_check_mode(&self, mode: TransactionCheckMode) {
1799 *self.transaction_check.write().await = mode;
1800 }
1801
1802 /// Register a proposal-admission gate (EPIC A / task A1).
1803 ///
1804 /// Gates run during admission, before any action executes, in the
1805 /// order they were registered. Each is a verified pre-execution safety
1806 /// check — information-flow (A4), concurrency (A5), blocking-policy
1807 /// (A9) — that can block a proposal or escalate it to human approval.
1808 /// Registering no gates leaves behavior unchanged.
1809 pub async fn register_admission_gate(&self, gate: Arc<dyn crate::admission::AdmissionGate>) {
1810 self.admission_gates.write().await.push(gate);
1811 }
1812
1813 /// Register `gate`, **replacing** any already-registered gate answering
1814 /// to the same [`crate::admission::AdmissionGate::name`] instead of
1815 /// appending a second one. For gates that are *reconfigured* rather than
1816 /// layered — the VIGIL intent gate is reloaded whenever an operator
1817 /// re-reads `.car/intent.json` — appending is a bug, not a no-op: gate
1818 /// aggregation is fail-closed, so the stale gate's verdict still wins
1819 /// while it holds state (a [`crate::taint::TaintLedger`]) nothing writes
1820 /// to any more, and two gates with the same name emit two
1821 /// `AdmissionGateDecision` events under that name.
1822 ///
1823 /// The replaced gate keeps its **position**, so a reconfigure never
1824 /// silently reorders admission relative to the other gates. Any further
1825 /// gate carrying the same name is a stale duplicate from an earlier
1826 /// registration and is dropped, so exactly one gate answers to the name.
1827 async fn replace_admission_gate(&self, gate: Arc<dyn crate::admission::AdmissionGate>) {
1828 let name = gate.name().to_string();
1829 let mut gates = self.admission_gates.write().await;
1830 match gates.iter().position(|g| g.name() == name) {
1831 Some(i) => {
1832 gates[i] = gate;
1833 let mut idx = 0usize;
1834 gates.retain(|g| {
1835 let keep = idx == i || g.name() != name;
1836 idx += 1;
1837 keep
1838 });
1839 }
1840 None => gates.push(gate),
1841 }
1842 }
1843
1844 /// Remove all registered admission gates (primarily for tests and
1845 /// reconfiguration). After this, proposal admission reverts to the
1846 /// transactional pre-check only.
1847 pub async fn clear_admission_gates(&self) {
1848 self.admission_gates.write().await.clear();
1849 }
1850
1851 /// The number of currently-registered admission gates.
1852 pub async fn admission_gate_count(&self) -> usize {
1853 self.admission_gates.read().await.len()
1854 }
1855
1856 /// The `name()` of every registered admission gate, in registration order.
1857 ///
1858 /// Prefer this over [`Self::admission_gate_count`] when asserting that a
1859 /// *particular* gate is installed: a bare count couples the assertion to
1860 /// every other gate the runtime happens to register, so adding one breaks
1861 /// unrelated tests that only cared about their own.
1862 pub async fn admission_gate_names(&self) -> Vec<String> {
1863 self.admission_gates
1864 .read()
1865 .await
1866 .iter()
1867 .map(|g| g.name().to_string())
1868 .collect()
1869 }
1870
1871 /// Register the VIGIL intent gate (arXiv 2601.05755 — the live
1872 /// verify-before-commit call-site). Installs
1873 /// [`crate::intent_gate::IntentGate`] as an admission gate: a
1874 /// forbidden capability or a tool-stream-influenced out-of-intent
1875 /// action hard-rejects the proposal; untainted drift escalates to
1876 /// the durable approval flow (A7) by content-bound fingerprint.
1877 /// Replans are covered automatically (gates re-run on every
1878 /// replanned proposal).
1879 ///
1880 /// This is also where the runtime's [`crate::taint::TaintLedger`] is
1881 /// installed. From here on, every successful action records whether its
1882 /// result was tainted and which state keys it wrote, and the gate marks
1883 /// any incoming action that READS a tainted key as tool-stream-
1884 /// influenced. That closes the two holes the static `untrusted_tools`
1885 /// list leaves open: a trusted tool laundering attacker-controlled
1886 /// content out of state, and a replanned proposal that carries no
1887 /// dependency edge back to the poisoned action. Installing no intent
1888 /// gate installs no ledger, so nothing changes for a runtime that
1889 /// doesn't configure VIGIL.
1890 ///
1891 /// **Calling this again REPLACES the installed gate; it does not add a
1892 /// second one.** Reinstalling is a real operation — an operator editing
1893 /// `.car/intent.json` and re-running
1894 /// [`Runtime::install_intent_gate_from_project`] lands here — and each
1895 /// call builds a *fresh* ledger that becomes the only one the executor
1896 /// writes to. Appending would leave the previous gate registered while
1897 /// holding an orphaned ledger, and because gate aggregation is
1898 /// fail-closed that frozen gate's verdict would still win: every key
1899 /// tainted before the reload would stay tainted forever, no trusted
1900 /// overwrite could clear it, and legitimate out-of-intent work would
1901 /// hard-refuse with no approval path. So the runtime holds exactly one
1902 /// intent gate, and it is always the one bound to the live ledger. The
1903 /// replacement keeps the previous gate's position in the admission
1904 /// order, so a reload never silently reorders the other gates.
1905 pub async fn install_intent_gate(&self, config: crate::intent_gate::IntentGateConfig) {
1906 let ledger = Arc::new(crate::taint::TaintLedger::new(
1907 config.untrusted_tools.iter().cloned().collect(),
1908 ));
1909 *self.taint_ledger.write().await = Some(Arc::clone(&ledger));
1910 self.replace_admission_gate(Arc::new(crate::intent_gate::IntentGate::with_taint(
1911 config, ledger,
1912 )))
1913 .await;
1914 }
1915
1916 /// The runtime's taint provenance ledger, when an intent gate is
1917 /// installed (see [`Runtime::install_intent_gate`]). `None` otherwise —
1918 /// the ledger is strictly opt-in, alongside VIGIL.
1919 pub async fn taint_ledger(&self) -> Option<Arc<crate::taint::TaintLedger>> {
1920 self.taint_ledger.read().await.clone()
1921 }
1922
1923 /// Load `.car/intent.json` from `car_dir` and install the intent
1924 /// gate when present. Absent file → Ok(false) (opt-in, ungated);
1925 /// malformed file → loud error, never a silently-ungated session.
1926 pub async fn install_intent_gate_from_project(
1927 &self,
1928 car_dir: impl AsRef<std::path::Path>,
1929 ) -> Result<bool, crate::intent_gate::IntentLoadError> {
1930 match crate::intent_gate::load_intent_config(car_dir)? {
1931 Some(cfg) => {
1932 self.install_intent_gate(cfg).await;
1933 Ok(true)
1934 }
1935 None => Ok(false),
1936 }
1937 }
1938
1939 /// Register the skill deployment-tier ceiling gate (EPIC A / A8).
1940 ///
1941 /// Requires a memgine to be attached (skills live there). When a
1942 /// proposal names its driving skill in `context["skill"]`, the gate
1943 /// caps the proposal's actions at that skill's persisted
1944 /// `deployment_tier`, escalating an over-ceiling action to the durable
1945 /// approval flow (A7). Returns false if no memgine is attached.
1946 pub async fn install_skill_ceiling_gate(&self) -> bool {
1947 match &self.memgine {
1948 Some(mem) => {
1949 let gate = Arc::new(crate::skill_ceiling::SkillCeilingGate::new(mem.clone()));
1950 self.register_admission_gate(gate).await;
1951 true
1952 }
1953 None => false,
1954 }
1955 }
1956
1957 /// Drain buffered chunks + status for a detached tool invocation (C2).
1958 /// `None` for an unknown or fully-consumed handle. See
1959 /// [`crate::tool_handles::ToolHandleRegistry::poll`] for the
1960 /// consume-on-terminal contract.
1961 pub async fn tool_poll(&self, handle_id: &str) -> Option<crate::tool_handles::ToolPollResult> {
1962 self.tool_handles.poll(handle_id).await
1963 }
1964
1965 /// Request cancellation of a detached tool invocation (C2): fires the
1966 /// handle's cancel token (dropping the executor's chunk receiver) and
1967 /// seals its status as `cancelled` unless already terminal. Returns
1968 /// false for an unknown handle.
1969 pub async fn tool_cancel(&self, handle_id: &str) -> bool {
1970 self.tool_handles.cancel(handle_id).await
1971 }
1972
1973 /// Subscribe to the live [`car_ir::ToolStreamEvent`] fanout for all
1974 /// detached tool invocations on this runtime (C2). The WS layer
1975 /// forwards these as `tools.stream.event` notifications.
1976 pub fn subscribe_tool_events(
1977 &self,
1978 ) -> tokio::sync::broadcast::Receiver<car_ir::ToolStreamEvent> {
1979 self.tool_handles.subscribe()
1980 }
1981
1982 /// Enable tamper-evident hash chaining on the event log (EPIC A / A9).
1983 /// Every event appended from now on is linked to its predecessor by a
1984 /// content hash, so an after-the-fact edit to a chained event — or an
1985 /// interior deletion/reordering — is detectable via
1986 /// [`Runtime::verify_event_log_chain`]. Truncation at either end of the
1987 /// log (dropping a prefix or suffix wholesale) is NOT detectable — the
1988 /// chain has no anchored head hash and no trusted tail witness; that is
1989 /// out of scope until the chain head is anchored. Opt-in: existing logs
1990 /// stay byte-identical until enabled.
1991 pub async fn enable_event_log_hash_chaining(&self) {
1992 self.log.lock().await.enable_hash_chaining();
1993 }
1994
1995 /// Verify the event log's tamper-evidence chain (EPIC A / A9). Returns
1996 /// `Ok(n)` for `n` verified chained events, or `Err(index)` naming the
1997 /// first event whose hash/linkage doesn't match — the point of an
1998 /// interior edit, deletion, or reordering. Head/tail truncation is not
1999 /// detectable (no anchored head hash; the first chained event's
2000 /// `prev_hash` is taken on trust) — see `EventLog::verify_chain`.
2001 pub async fn verify_event_log_chain(&self) -> Result<usize, usize> {
2002 self.log.lock().await.verify_chain()
2003 }
2004
2005 /// Install a durable HITL approval ledger backed by a JSONL journal
2006 /// (EPIC A / A7). Loads any existing decisions so approvals survive a
2007 /// restart, then resolves future admission-gate `NeedsApproval`
2008 /// verdicts against it. The canonical daemon path is
2009 /// `~/.car/approvals.jsonl`.
2010 pub async fn set_approval_ledger_path(
2011 &self,
2012 path: impl Into<std::path::PathBuf>,
2013 ) -> std::io::Result<()> {
2014 let ledger = ApprovalLedger::with_journal(path.into())?;
2015 // Surface journal corruption (or a torn concurrent write) instead of
2016 // silently trusting a partial ledger (review A7 — the doc on
2017 // `skipped_on_load` promises callers surface it).
2018 if ledger.skipped_on_load() > 0 {
2019 tracing::warn!(
2020 skipped = ledger.skipped_on_load(),
2021 "approval ledger journal had unparseable lines skipped on load — \
2022 the ledger may be missing decisions"
2023 );
2024 }
2025 *self.approval_ledger.write().await = Some(ledger);
2026 Ok(())
2027 }
2028
2029 /// Use an in-memory approval ledger (no persistence) — primarily for
2030 /// tests and ephemeral runtimes.
2031 pub async fn set_approval_ledger_in_memory(&self) {
2032 *self.approval_ledger.write().await = Some(ApprovalLedger::new());
2033 }
2034
2035 /// Record a human approval for an admission fingerprint (EPIC A / A7).
2036 /// A subsequently re-submitted proposal whose escalation matches this
2037 /// fingerprint is admitted without asking again.
2038 pub async fn approve_admission(
2039 &self,
2040 fingerprint: &str,
2041 reviewer: &str,
2042 reason: &str,
2043 ) -> Result<(), String> {
2044 self.record_admission_decision(fingerprint, ApprovalDecision::Approved, reviewer, reason)
2045 .await
2046 }
2047
2048 /// Record a human rejection for an admission fingerprint (EPIC A / A7).
2049 /// A proposal whose escalation matches a rejected fingerprint is
2050 /// blocked outright.
2051 pub async fn reject_admission(
2052 &self,
2053 fingerprint: &str,
2054 reviewer: &str,
2055 reason: &str,
2056 ) -> Result<(), String> {
2057 self.record_admission_decision(fingerprint, ApprovalDecision::Rejected, reviewer, reason)
2058 .await
2059 }
2060
2061 async fn record_admission_decision(
2062 &self,
2063 fingerprint: &str,
2064 decision: ApprovalDecision,
2065 reviewer: &str,
2066 reason: &str,
2067 ) -> Result<(), String> {
2068 {
2069 let mut guard = self.approval_ledger.write().await;
2070 let ledger = guard
2071 .as_mut()
2072 .ok_or_else(|| "no approval ledger installed".to_string())?;
2073 ledger
2074 .record(ApprovalRecord {
2075 fingerprint: fingerprint.to_string(),
2076 // Admission escalations aren't tier-classified; record the
2077 // most restrictive tier so the decision reads as "elevated".
2078 required_tier: PermissionTier::FullAccess,
2079 decision,
2080 reviewer: reviewer.to_string(),
2081 reason: reason.to_string(),
2082 evidence: None,
2083 decided_at: chrono::Utc::now().to_rfc3339(),
2084 })
2085 // A journal write failure means the decision is NOT durable —
2086 // surface it instead of emitting a false ApprovalRecorded
2087 // audit event (review A7).
2088 .map_err(|e| format!("failed to persist approval decision: {e}"))?;
2089 }
2090 let approval = match decision {
2091 ApprovalDecision::Approved => "approved",
2092 ApprovalDecision::Rejected => "rejected",
2093 };
2094 let mut log = self.log.lock().await;
2095 log.append(
2096 EventKind::ApprovalRecorded,
2097 None,
2098 None,
2099 [
2100 ("fingerprint".to_string(), Value::from(fingerprint)),
2101 ("approval".to_string(), Value::from(approval)),
2102 ("reviewer".to_string(), Value::from(reviewer)),
2103 ("reason".to_string(), Value::from(reason)),
2104 ]
2105 .into(),
2106 );
2107 Ok(())
2108 }
2109
2110 /// Back the idempotency cache with a durable JSONL journal (EPIC A / C3).
2111 ///
2112 /// Loads any existing entries into the in-memory cache, then persists
2113 /// future idempotent results (and rollback invalidations, as
2114 /// tombstones) to the journal. After a crash-restart, re-submitting a
2115 /// completed idempotent action returns the cached result instead of
2116 /// re-executing it — preventing duplicate external side effects. The
2117 /// canonical daemon path is `~/.car/idempotency.jsonl`. Returns the
2118 /// number of live entries loaded.
2119 pub async fn set_idempotency_cache_path(
2120 &self,
2121 path: impl Into<std::path::PathBuf>,
2122 ) -> std::io::Result<usize> {
2123 let path = path.into();
2124 if let Some(parent) = path.parent() {
2125 let _ = std::fs::create_dir_all(parent);
2126 }
2127 // Replay the journal (last entry per key wins; a tombstone removes).
2128 let mut loaded: HashMap<String, ActionResult> = HashMap::new();
2129 if path.exists() {
2130 let content = std::fs::read_to_string(&path)?;
2131 for line in content.lines() {
2132 if line.trim().is_empty() {
2133 continue;
2134 }
2135 if let Ok(entry) = serde_json::from_str::<IdempotencyEntry>(line) {
2136 match entry.result {
2137 Some(r) => {
2138 loaded.insert(entry.key, r);
2139 }
2140 None => {
2141 loaded.remove(&entry.key);
2142 }
2143 }
2144 }
2145 }
2146 }
2147 {
2148 let mut cache = self.idempotency_cache.lock().await;
2149 for (k, v) in loaded.iter() {
2150 cache.entry(k.clone()).or_insert_with(|| v.clone());
2151 }
2152 }
2153 let count = loaded.len();
2154 *self.idempotency_journal.write().await = Some(path);
2155 Ok(count)
2156 }
2157
2158 /// Append an idempotency record to the journal if one is configured.
2159 /// `result = None` writes a tombstone (invalidation).
2160 async fn journal_idempotency(&self, key: &str, result: Option<&ActionResult>) {
2161 let path = {
2162 let guard = self.idempotency_journal.read().await;
2163 guard.clone()
2164 };
2165 let Some(path) = path else {
2166 return;
2167 };
2168 let entry = IdempotencyEntry {
2169 key: key.to_string(),
2170 result: result.cloned(),
2171 };
2172 if let Ok(mut line) = serde_json::to_string(&entry) {
2173 line.push('\n');
2174 use std::io::Write;
2175 // Persistence failure must be VISIBLE (linus review): a
2176 // silently-dropped journal write means an idempotent action
2177 // re-executes its side effects after a restart while the
2178 // operator believes it's covered. Fail-open (the in-memory
2179 // cache still dedups this process) but loudly.
2180 let write = std::fs::OpenOptions::new()
2181 .create(true)
2182 .append(true)
2183 .open(&path)
2184 .and_then(|mut f| f.write_all(line.as_bytes()));
2185 if let Err(e) = write {
2186 tracing::warn!(
2187 path = %path.display(),
2188 error = %e,
2189 "idempotency journal write failed — durable dedup is NOT covering this result"
2190 );
2191 }
2192 }
2193 }
2194
2195 /// Look up the current decision for an admission fingerprint, if any.
2196 pub async fn admission_decision(&self, fingerprint: &str) -> Option<ApprovalDecision> {
2197 let guard = self.approval_ledger.read().await;
2198 guard
2199 .as_ref()
2200 .and_then(|l| l.lookup(fingerprint).map(|r| r.decision))
2201 }
2202
2203 /// Verify a model's tool-use claims against the runtime's own execution
2204 /// receipts (EPIC A / A6 — arXiv 2603.10060). The runtime owns tool
2205 /// execution and logs it, so it holds unforgeable ground truth: this
2206 /// projects [`car_eventlog::tool_receipts::ToolReceipt`]s from the event
2207 /// log and cross-checks the supplied claims, catching a fabricated tool
2208 /// reference, a misstated result count, or a false "found nothing".
2209 ///
2210 /// `proposal_id` scopes the cross-check window to a single proposal's
2211 /// events (pass the proposal whose response the claims came from) — a
2212 /// claim is never judged against another run's receipts. The check is
2213 /// retention-coherent (review A6): when the log has trimmed events and
2214 /// the window can't be proven complete (unscoped, or the proposal's
2215 /// `ProposalReceived` marker — which precedes every receipt of that
2216 /// proposal — was itself evicted), a claim without a receipt comes back
2217 /// in `ReceiptReport::ungroundable` ("window evicted") instead of being
2218 /// mis-flagged `fabricated_tool_reference`.
2219 ///
2220 /// Returns the [`car_eventlog::tool_receipts::ReceiptReport`]; when it is not grounded, a
2221 /// `ToolReceiptHallucination` event is emitted so the caller's
2222 /// verdict→action loop (reject/flag the response) is auditable.
2223 /// Deterministic, zero-inference. Claims arrive structured — CAR's
2224 /// thesis is that intent is structured IR, so a caller extracts claims
2225 /// from the model's tool_calls / IR rather than regexing prose.
2226 pub async fn verify_tool_receipts(
2227 &self,
2228 claims: &[car_eventlog::tool_receipts::ToolClaim],
2229 proposal_id: Option<&str>,
2230 ) -> car_eventlog::tool_receipts::ReceiptReport {
2231 let (receipts, window_complete) = {
2232 let log = self.log.lock().await;
2233 let complete = if log.trimmed_events() == 0 {
2234 // Nothing was ever evicted — the window is complete whether
2235 // or not it is scoped.
2236 true
2237 } else {
2238 match proposal_id {
2239 // A proposal's window is complete iff its ProposalReceived
2240 // marker survived retention: every receipt of the proposal
2241 // was appended after it, so if the marker is retained, so
2242 // are the receipts.
2243 Some(pid) => log.events().iter().any(|e| {
2244 e.kind == EventKind::ProposalReceived
2245 && e.proposal_id.as_deref() == Some(pid)
2246 }),
2247 // Unscoped check over a trimmed log: unknowable.
2248 None => false,
2249 }
2250 };
2251 (
2252 car_eventlog::tool_receipts::receipts_from_events_scoped(log.events(), proposal_id),
2253 complete,
2254 )
2255 };
2256 let report = car_eventlog::tool_receipts::verify_tool_claims_windowed(
2257 claims,
2258 &receipts,
2259 window_complete,
2260 );
2261 if !report.grounded {
2262 let mut log = self.log.lock().await;
2263 log.append(
2264 EventKind::ToolReceiptHallucination,
2265 None,
2266 proposal_id,
2267 [
2268 (
2269 "count".to_string(),
2270 Value::from(report.hallucinations.len()),
2271 ),
2272 (
2273 "hallucinations".to_string(),
2274 serde_json::to_value(&report.hallucinations).unwrap_or_default(),
2275 ),
2276 ]
2277 .into(),
2278 );
2279 }
2280 report
2281 }
2282
2283 /// Run every registered admission gate against a proposal and fold
2284 /// their verdicts into a single [`crate::admission::AdmissionDecision`].
2285 ///
2286 /// Each gate's outcome is recorded as an `AdmissionGateDecision` event
2287 /// so a denial is attributable. Aggregation is conjunctive and
2288 /// fail-closed: the proposal is admitted only if every gate allowed it.
2289 /// Returns an admit decision immediately when no gates are registered
2290 /// (zero overhead on the common path).
2291 async fn run_admission_gates(
2292 &self,
2293 proposal: &ActionProposal,
2294 session_id: Option<&str>,
2295 scope: Option<&crate::scope::RuntimeScope>,
2296 ) -> crate::admission::AdmissionDecision {
2297 use crate::admission::{AdmissionDecision, GateContext};
2298
2299 let gates = self.admission_gates.read().await;
2300 if gates.is_empty() {
2301 return AdmissionDecision::admit();
2302 }
2303
2304 // One consistent snapshot for every gate this pass: the keys this
2305 // proposal may observe (all of them unscoped; shared plus its own
2306 // tenant's under a tenant, never another tenant's). A gate resolves
2307 // each action's keys in the namespace that action executes in
2308 // (car#3121).
2309 let tenant = scope.and_then(|s| s.tenant_id.as_deref());
2310 let (state, versions) = self.state.versioned_snapshot_visible_to(tenant);
2311 let ctx = GateContext {
2312 session_id,
2313 scope,
2314 state: &state,
2315 versions: &versions,
2316 };
2317
2318 let mut decision = AdmissionDecision::admit();
2319 for gate in gates.iter() {
2320 let outcome = gate.check(proposal, &ctx).await;
2321 // Audit every gate decision (allow included) so the trail shows
2322 // which gates ran, not just which objected.
2323 let mut props: HashMap<String, Value> = HashMap::new();
2324 props.insert("gate".to_string(), Value::from(gate.name()));
2325 // Pre-execution proposal admission (vs the multi-agent
2326 // commit barrier, which emits the same event kind with
2327 // phase:"commit_barrier").
2328 props.insert("phase".to_string(), Value::from("admission"));
2329 props.insert("decision".to_string(), Value::from(outcome.label()));
2330 match &outcome {
2331 crate::admission::GateOutcome::Allow => {}
2332 crate::admission::GateOutcome::Reject { blocked, reason } => {
2333 props.insert("reason".to_string(), Value::from(reason.clone()));
2334 props.insert(
2335 "blocked".to_string(),
2336 serde_json::to_value(blocked).unwrap_or_default(),
2337 );
2338 }
2339 crate::admission::GateOutcome::NeedsApproval {
2340 actions,
2341 fingerprint,
2342 reason,
2343 } => {
2344 props.insert("reason".to_string(), Value::from(reason.clone()));
2345 props.insert(
2346 "blocked".to_string(),
2347 serde_json::to_value(actions).unwrap_or_default(),
2348 );
2349 props.insert("fingerprint".to_string(), Value::from(fingerprint.clone()));
2350 }
2351 }
2352 {
2353 let mut log = self.log.lock().await;
2354 log.append(
2355 EventKind::AdmissionGateDecision,
2356 None,
2357 Some(&proposal.id),
2358 props,
2359 );
2360 }
2361 decision.absorb(gate.name(), outcome);
2362 }
2363 decision
2364 }
2365
2366 /// Install a harness operating config — the live end of the Evolution
2367 /// Agent loop (survey §3.5). After the meta-agent's `HarnessConfig::apply`
2368 /// produces a governed, regression-gated config, hand it here to take
2369 /// effect: `max_retries`/`retry_backoff_ms` drive the per-action retry
2370 /// loop, and `planning_max_replans` is mapped onto the replan budget.
2371 /// Every `HarnessConfig` knob is consumed here — none is aspirational.
2372 pub async fn set_harness_config(&self, cfg: car_memgine::HarnessConfig) {
2373 self.replan_config.write().await.max_replans = cfg.planning_max_replans;
2374 *self.harness_config.write().await = Some(cfg);
2375 }
2376
2377 /// The current harness operating config, if one has been installed.
2378 pub async fn harness_config(&self) -> Option<car_memgine::HarnessConfig> {
2379 self.harness_config.read().await.clone()
2380 }
2381
2382 /// Atomically read-modify-write the harness operating config under ONE
2383 /// write lock (installing the default first when none is set). The
2384 /// get→mutate→set alternative is a lost-update race when two requests
2385 /// mutate concurrently — the daemon's `evolution.run` harness-apply path
2386 /// uses this instead (kernel review S3). Keeps the same replan-budget
2387 /// propagation as [`Self::set_harness_config`].
2388 pub async fn update_harness_config<R>(
2389 &self,
2390 f: impl FnOnce(&mut car_memgine::HarnessConfig) -> R,
2391 ) -> R {
2392 let mut guard = self.harness_config.write().await;
2393 let cfg = guard.get_or_insert_with(car_memgine::HarnessConfig::default);
2394 let out = f(cfg);
2395 let max_replans = cfg.planning_max_replans;
2396 drop(guard);
2397 self.replan_config.write().await.max_replans = max_replans;
2398 out
2399 }
2400
2401 /// Run the pre-execution transactional check against the current
2402 /// versioned state, each action's keys resolved in the namespace it
2403 /// executes in (car#3121). Returns the conflicting action ids when the
2404 /// mode is `Strict` and conflicts exist (so the caller can reject those
2405 /// actions); always emits `TransactionConflict` telemetry for each
2406 /// conflict found. Empty/`None` means "proceed".
2407 async fn transaction_precheck(
2408 &self,
2409 proposal: &ActionProposal,
2410 scope: Option<&crate::scope::RuntimeScope>,
2411 ) -> std::collections::HashSet<String> {
2412 let mode = *self.transaction_check.read().await;
2413 if mode == TransactionCheckMode::Off {
2414 return std::collections::HashSet::new();
2415 }
2416 let tenant = scope.and_then(|s| s.tenant_id.as_deref());
2417 let resolved = proposal_in_store_keys(proposal, tenant);
2418 let (state, versions) = self.state.versioned_snapshot_visible_to(tenant);
2419 // Ordering comes from the proposal as written, which is what the
2420 // executor builds its DAG from: a tenant action depending on `k` runs
2421 // after a tool call declaring `k`, though their resolved keys differ.
2422 let report = car_verify::check_transaction_ordered_by(
2423 &resolved,
2424 &proposal.actions,
2425 &versions,
2426 Some(&state),
2427 );
2428 if report.consistent {
2429 return std::collections::HashSet::new();
2430 }
2431 let mut blocked = std::collections::HashSet::new();
2432 let mut log = self.log.lock().await;
2433 for c in &report.conflicts {
2434 for aid in &c.actions {
2435 blocked.insert(aid.clone());
2436 }
2437 log.append(
2438 EventKind::TransactionConflict,
2439 c.actions.first().map(|s| s.as_str()),
2440 Some(&proposal.id),
2441 [
2442 // Use the serde representation (snake_case: write_write
2443 // / read_write / stale_assumption) the .d.ts/.pyi/doc
2444 // contract is written against — NOT Debug, which would
2445 // emit "writewrite" (neo review #4).
2446 (
2447 "kind".to_string(),
2448 serde_json::to_value(c.kind).unwrap_or_default(),
2449 ),
2450 ("key".to_string(), Value::from(c.key.clone())),
2451 (
2452 "actions".to_string(),
2453 serde_json::to_value(&c.actions).unwrap_or_default(),
2454 ),
2455 (
2456 "explanation".to_string(),
2457 Value::from(c.explanation.clone()),
2458 ),
2459 ("resolution".to_string(), Value::from(c.resolution.clone())),
2460 ]
2461 .into(),
2462 );
2463 }
2464 drop(log);
2465 // Only Strict blocks execution; Warn records and proceeds.
2466 match mode {
2467 TransactionCheckMode::Strict => blocked,
2468 _ => std::collections::HashSet::new(),
2469 }
2470 }
2471
2472 /// Register a tool with just a name (backward compatible).
2473 pub async fn register_tool(&self, name: &str) {
2474 let schema = ToolSchema {
2475 name: name.to_string(),
2476 source: car_ir::ToolSourceKind::UserDefined,
2477 description: String::new(),
2478 parameters: serde_json::Value::Object(Default::default()),
2479 returns: None,
2480 idempotent: false,
2481 cache_ttl_secs: None,
2482 rate_limit: None,
2483 };
2484 self.register_tool_schema(schema).await;
2485 }
2486
2487 /// Register a tool with full schema.
2488 pub async fn register_tool_schema(&self, schema: ToolSchema) {
2489 // Auto-configure cache if schema specifies it
2490 if let Some(ttl) = schema.cache_ttl_secs {
2491 self.result_cache.enable_caching(&schema.name, ttl).await;
2492 }
2493 // Auto-configure rate limit if schema specifies it
2494 if let Some(ref rl) = schema.rate_limit {
2495 self.rate_limiter
2496 .set_limit(
2497 &schema.name,
2498 RateLimit {
2499 max_calls: rl.max_calls,
2500 interval_secs: rl.interval_secs,
2501 },
2502 )
2503 .await;
2504 }
2505 self.tools.write().await.insert(schema.name.clone(), schema);
2506 }
2507
2508 /// Register a tool via the canonical registry.
2509 /// This is the preferred way to register tools — it updates both the
2510 /// registry and the legacy tools HashMap for backward compatibility.
2511 pub async fn register_tool_entry(&self, mut entry: crate::registry::ToolEntry) {
2512 // The registry's source is authoritative for both event provenance and
2513 // the public tools.list/schema view, including literal-built entries.
2514 entry.schema.source = entry.source.kind();
2515 let schema = entry.schema.clone();
2516 self.registry.register(entry).await;
2517 self.register_tool_schema(schema).await;
2518 }
2519
2520 /// Remove a tool from both the canonical registry and the legacy
2521 /// `tools` schema map, so the model no longer sees it and the
2522 /// validator no longer accepts it. Used when a remote MCP connector
2523 /// tool is disabled or its connector is removed. Returns true if the
2524 /// tool was present in either store.
2525 pub async fn unregister_tool(&self, name: &str) -> bool {
2526 let removed_entry = self.registry.remove(name).await.is_some();
2527 let removed_schema = self.tools.write().await.remove(name).is_some();
2528 removed_entry || removed_schema
2529 }
2530
2531 /// Register CAR's built-in agent utility stdlib.
2532 ///
2533 /// This is an opt-in convenience layer for common local-file and text tools.
2534 /// Existing runtimes remain unchanged until this is called.
2535 pub async fn register_agent_basics(&self) {
2536 for entry in crate::agent_basics::entries() {
2537 self.register_tool_entry(entry).await;
2538 }
2539 }
2540
2541 /// Refuse the built-in file tools access to the host paths `guard` names.
2542 ///
2543 /// Applies on the built-in fallback path only, and only while the bound
2544 /// substrate is local: a configured executor that claims a tool owns its
2545 /// own checks, and a remote substrate owns its own filesystem. See
2546 /// [`crate::agent_basics::execute_with_ledger_guarded`] for what is checked.
2547 pub async fn set_local_path_guard(&self, guard: crate::agent_basics::LocalPathGuard) {
2548 *self.local_path_guard.write().await = Some(guard);
2549 }
2550
2551 /// Keep the built-in `write_file` / `edit_file` inside `root` (car#3406):
2552 /// a write that resolves outside it, symlinks followed, is refused naming
2553 /// the root. Applies on the built-in path while the substrate is local.
2554 pub async fn set_basics_root(&self, root: std::path::PathBuf) {
2555 *self.basics_root.write().await = Some(root);
2556 }
2557
2558 /// Get all registered tool schemas (for model prompt generation).
2559 pub async fn tool_schemas(&self) -> Vec<ToolSchema> {
2560 self.tools.read().await.values().cloned().collect()
2561 }
2562
2563 /// Set a cost budget that limits proposal execution.
2564 pub async fn set_cost_budget(&self, budget: CostBudget) {
2565 *self.cost_budget.write().await = Some(budget);
2566 }
2567
2568 /// Set per-agent capability permissions that restrict tools, state keys, and action count.
2569 pub async fn set_capabilities(&self, caps: CapabilitySet) {
2570 *self.capabilities.write().await = Some(caps);
2571 }
2572
2573 /// Set a per-tool rate limit (token bucket).
2574 ///
2575 /// `max_calls` tokens are available per `interval_secs` window.
2576 /// When the bucket is empty, `dispatch()` applies backpressure by
2577 /// waiting until a token refills.
2578 pub async fn set_rate_limit(&self, tool: &str, max_calls: u32, interval_secs: f64) {
2579 self.rate_limiter
2580 .set_limit(
2581 tool,
2582 RateLimit {
2583 max_calls,
2584 interval_secs,
2585 },
2586 )
2587 .await;
2588 }
2589
2590 /// Enable cross-proposal result caching for a tool with a TTL in seconds.
2591 pub async fn enable_tool_cache(&self, tool: &str, ttl_secs: u64) {
2592 self.result_cache.enable_caching(tool, ttl_secs).await;
2593 }
2594
2595 /// Execute a proposal with automatic replanning on failure.
2596 ///
2597 /// If a `ReplanCallback` is registered and `max_replans > 0`, the runtime
2598 /// will catch abort failures, roll back state, ask the model for an
2599 /// alternative proposal via the callback, and re-execute. This transforms
2600 /// "execute-and-hope" into "execute-and-recover."
2601 ///
2602 /// If no callback is registered or `max_replans == 0`, behaves identically
2603 /// to a single `execute_inner()` call (zero overhead, fully backward compatible).
2604 #[instrument(
2605 name = "proposal.execute",
2606 skip_all,
2607 fields(
2608 proposal_id = %proposal.id,
2609 action_count = proposal.actions.len(),
2610 )
2611 )]
2612 pub async fn execute(&self, proposal: &ActionProposal) -> ProposalResult {
2613 // Forward to the cancel-aware variant with a never-cancelled
2614 // token. Existing callers see no behaviour change.
2615 let token = tokio_util::sync::CancellationToken::new();
2616 self.execute_with_cancel(proposal, &token).await
2617 }
2618
2619 /// Execute a proposal scoped to a specific session id.
2620 ///
2621 /// Validation walks the global policy registry plus the session's
2622 /// own registry — both must pass for an action to run. Session
2623 /// policies can deny what global allows; they cannot allow what
2624 /// global denies (validation is conjunctive).
2625 ///
2626 /// Returns the same [`ProposalResult`] shape as [`Self::execute`].
2627 /// Errors with an action-level rejection if the session id is
2628 /// unknown — callers should check via [`Self::session_exists`] or
2629 /// trust an id they minted via [`Self::open_session`].
2630 pub async fn execute_with_session(
2631 &self,
2632 proposal: &ActionProposal,
2633 session_id: &str,
2634 ) -> ProposalResult {
2635 let token = tokio_util::sync::CancellationToken::new();
2636 self.execute_with_session_and_cancel(proposal, session_id, &token)
2637 .await
2638 }
2639
2640 /// Combined session-scoped + cancellable execute. The session id
2641 /// is passed verbatim to the per-action policy check; the cancel
2642 /// token behaves identically to [`Self::execute_with_cancel`].
2643 pub async fn execute_with_session_and_cancel(
2644 &self,
2645 proposal: &ActionProposal,
2646 session_id: &str,
2647 cancel: &tokio_util::sync::CancellationToken,
2648 ) -> ProposalResult {
2649 self.execute_with_optional_session(proposal, Some(session_id), None, None, cancel)
2650 .await
2651 }
2652
2653 /// Execute an active-run proposal while requiring every accepted replan
2654 /// to retain the authenticated proposal id already claimed by the server.
2655 pub async fn execute_with_session_and_stable_replan_id(
2656 &self,
2657 proposal: &ActionProposal,
2658 session_id: &str,
2659 ) -> ProposalResult {
2660 let token = tokio_util::sync::CancellationToken::new();
2661 self.execute_with_optional_session(
2662 proposal,
2663 Some(session_id),
2664 None,
2665 Some(&proposal.id),
2666 &token,
2667 )
2668 .await
2669 }
2670
2671 /// Execute a proposal with cooperative cancellation.
2672 ///
2673 /// The runtime checks `token.is_cancelled()` at each DAG level
2674 /// boundary. When set, every action that hadn't yet started runs
2675 /// is reported as `Skipped` with `error = "canceled: ..."` so
2676 /// callers can distinguish "user pulled the plug" from "earlier
2677 /// abort cascaded." Actions already in flight continue to
2678 /// completion — tool calls dispatched to user-provided executors
2679 /// can't be safely interrupted from the engine.
2680 ///
2681 /// The CAR A2A bridge uses this so `tasks/cancel` produces a
2682 /// `ProposalResult` with clean partial state rather than relying
2683 /// on `JoinHandle::abort` to interrupt mid-await (which leaves
2684 /// no record of which actions actually ran).
2685 ///
2686 /// **FFI exposure:** this method is intentionally not surfaced
2687 /// through the NAPI / PyO3 / `car-server-core` JSON-RPC bindings.
2688 /// Those consumers (Node, Python, WebSocket) don't currently
2689 /// expose long-running async-task surfaces that need
2690 /// cancellation; the bridge is the lone consumer. When a binding
2691 /// gains a long-running task surface, the path is clear: add a
2692 /// per-binding token registry keyed by some caller-provided id,
2693 /// expose `cancelExecution(id)` / `cancel_execution(id)` /
2694 /// `proposal.cancel { id }`, and have the runtime call
2695 /// `execute_with_cancel` with the matching token. Skipping that
2696 /// today avoids speculative API surface that bloats bindings
2697 /// without a consumer.
2698 pub async fn execute_with_cancel(
2699 &self,
2700 proposal: &ActionProposal,
2701 cancel: &tokio_util::sync::CancellationToken,
2702 ) -> ProposalResult {
2703 self.execute_with_optional_session(proposal, None, None, None, cancel)
2704 .await
2705 }
2706
2707 pub async fn execute_with_stable_replan_id(&self, proposal: &ActionProposal) -> ProposalResult {
2708 let token = tokio_util::sync::CancellationToken::new();
2709 self.execute_with_optional_session(proposal, None, None, Some(&proposal.id), &token)
2710 .await
2711 }
2712
2713 /// Execute a proposal with an attached [`RuntimeScope`](crate::scope::RuntimeScope)
2714 /// (Parslee-ai/car#187 phase 3).
2715 ///
2716 /// Same contract as [`Self::execute`] plus a per-execution
2717 /// identity surface — typically built by the car-a2a dispatcher
2718 /// from the verified `Identity` and cooperative `a2a_caller`
2719 /// metadata on the inbound `ActionProposal`. The scope is
2720 /// recorded on the event log so downstream audit / log analysis
2721 /// can see which caller / tenant issued each action.
2722 ///
2723 /// **What this enforces today**: scope is captured + logged.
2724 /// Memgine queries and state-store ops still hit global
2725 /// namespaces — those follow-ups are tracked under #187.
2726 /// Tool / policy code that needs per-tenant behaviour right now
2727 /// should keep reading `proposal.context["a2a_caller_verified"]`
2728 /// directly (the phase 1 / 2 surface).
2729 pub async fn execute_scoped(
2730 &self,
2731 proposal: &ActionProposal,
2732 scope: &crate::scope::RuntimeScope,
2733 ) -> ProposalResult {
2734 let token = tokio_util::sync::CancellationToken::new();
2735 self.execute_scoped_with_cancel(proposal, scope, &token)
2736 .await
2737 }
2738
2739 /// Combined scoped + cancellable execute. Mirrors the shape of
2740 /// [`Self::execute_with_session_and_cancel`] for symmetry — both
2741 /// add a side-channel (session id / scope) on top of the
2742 /// cancellable form.
2743 pub async fn execute_scoped_with_cancel(
2744 &self,
2745 proposal: &ActionProposal,
2746 scope: &crate::scope::RuntimeScope,
2747 cancel: &tokio_util::sync::CancellationToken,
2748 ) -> ProposalResult {
2749 self.execute_with_optional_session(proposal, None, Some(scope), None, cancel)
2750 .await
2751 }
2752
2753 pub async fn execute_scoped_with_stable_replan_id(
2754 &self,
2755 proposal: &ActionProposal,
2756 scope: &crate::scope::RuntimeScope,
2757 ) -> ProposalResult {
2758 let token = tokio_util::sync::CancellationToken::new();
2759 self.execute_with_optional_session(proposal, None, Some(scope), Some(&proposal.id), &token)
2760 .await
2761 }
2762
2763 /// Internal entry point that backs both
2764 /// [`Self::execute_with_cancel`] (no session) and
2765 /// [`Self::execute_with_session_and_cancel`]. Holds the replan
2766 /// loop and threads `session_id` into the per-action validation
2767 /// path so session-scoped policies stack on top of global ones.
2768 async fn execute_with_optional_session(
2769 &self,
2770 proposal: &ActionProposal,
2771 session_id: Option<&str>,
2772 scope: Option<&crate::scope::RuntimeScope>,
2773 required_replan_proposal_id: Option<&str>,
2774 cancel: &tokio_util::sync::CancellationToken,
2775 ) -> ProposalResult {
2776 // The shared StateStore has one proposal transaction at a time. The
2777 // guard spans validation, replanning, and rollback so distinct Runtime
2778 // facades over the same store cannot interleave state snapshots.
2779 let _proposal_execution = self.state.lock_proposal_execution().await;
2780 self.execute_with_optional_session_already_guarded(
2781 proposal,
2782 session_id,
2783 scope,
2784 required_replan_proposal_id,
2785 cancel,
2786 )
2787 .await
2788 }
2789
2790 /// Execute while the caller retains this StateStore's proposal-execution
2791 /// guard. `plan_and_execute` uses this path so one guard covers its initial
2792 /// snapshot, ranking, every candidate, and the final commit/rollback.
2793 async fn execute_with_optional_session_already_guarded(
2794 &self,
2795 proposal: &ActionProposal,
2796 session_id: Option<&str>,
2797 scope: Option<&crate::scope::RuntimeScope>,
2798 required_replan_proposal_id: Option<&str>,
2799 cancel: &tokio_util::sync::CancellationToken,
2800 ) -> ProposalResult {
2801 // Establish canonical identity before any other rejection so an
2802 // undigested lineage entry can only mean the proposal was genuinely
2803 // impossible to represent as JCS/I-JSON. No journal row is emitted.
2804 let original_proposal_digest = match proposal_digest(proposal) {
2805 Ok(digest) => digest,
2806 Err(error) => {
2807 self.log.lock().await.append(
2808 EventKind::StateRollback,
2809 None,
2810 Some(&proposal.id),
2811 proposal_rejection_boundary_data(proposal, None, &error),
2812 );
2813 let result = ProposalResult::for_proposal(
2814 proposal,
2815 proposal
2816 .actions
2817 .iter()
2818 .map(|action| rejected_result(&action.id, error.clone()))
2819 .collect(),
2820 CostSummary::default(),
2821 );
2822 return finalize_proposal_result(
2823 result,
2824 &proposal.id,
2825 &[ProposalLineageEntry {
2826 generation: 0,
2827 proposal_id: proposal.id.clone(),
2828 proposal_digest: None,
2829 status: ProposalLineageStatus::Rejected,
2830 rejection_reason: Some(error),
2831 }],
2832 &[],
2833 );
2834 }
2835 };
2836
2837 // Proposal-local action ids are the join key for DAG scheduling,
2838 // result rows, receipts, and StateTransition attribution. Reject a
2839 // duplicate before scope/admission/transaction logging or any state
2840 // mutation. Retried attempts remain valid reuse of one admitted id.
2841 if let Err(error) = validate_proposal_action_ids(proposal) {
2842 self.log.lock().await.append(
2843 EventKind::StateRollback,
2844 None,
2845 Some(&proposal.id),
2846 proposal_rejection_boundary_data(proposal, Some(&original_proposal_digest), &error),
2847 );
2848 let result = ProposalResult::for_proposal(
2849 proposal,
2850 proposal
2851 .actions
2852 .iter()
2853 .map(|action| rejected_result(&action.id, error.clone()))
2854 .collect(),
2855 CostSummary::default(),
2856 );
2857 return finalize_proposal_result(
2858 result,
2859 &proposal.id,
2860 &[proposal_lineage_entry(
2861 proposal,
2862 0,
2863 ProposalLineageStatus::Rejected,
2864 Some(error),
2865 )],
2866 &[],
2867 );
2868 }
2869 if let Err(error) = validate_proposal_retry_limits(proposal) {
2870 self.log.lock().await.append(
2871 EventKind::StateRollback,
2872 None,
2873 Some(&proposal.id),
2874 proposal_rejection_boundary_data(proposal, Some(&original_proposal_digest), &error),
2875 );
2876 let result = ProposalResult::for_proposal(
2877 proposal,
2878 proposal
2879 .actions
2880 .iter()
2881 .map(|action| rejected_result(&action.id, error.clone()))
2882 .collect(),
2883 CostSummary::default(),
2884 );
2885 return finalize_proposal_result(
2886 result,
2887 &proposal.id,
2888 &[proposal_lineage_entry(
2889 proposal,
2890 0,
2891 ProposalLineageStatus::Rejected,
2892 Some(error),
2893 )],
2894 &[],
2895 );
2896 }
2897 let config = self.replan_config.read().await.clone();
2898 let mut current_proposal = proposal.clone();
2899 let mut attempt: u32 = 0;
2900 let mut lineage = vec![proposal_lineage_entry(
2901 proposal,
2902 0,
2903 ProposalLineageStatus::Accepted,
2904 None,
2905 )];
2906
2907 // Phase 3 foundation (Parslee-ai/car#187): record the scope
2908 // on the event log so audit / log analysis can correlate
2909 // actions to the caller / tenant that triggered them. Only
2910 // logged when at least one identity field is set — keeps
2911 // the existing in-process call sites free of noise.
2912 if let Some(s) = scope {
2913 if !s.is_unscoped() {
2914 let mut props: HashMap<String, Value> = HashMap::new();
2915 if let Some(cid) = &s.caller_id {
2916 props.insert("caller_id".to_string(), Value::from(cid.as_str()));
2917 }
2918 if let Some(tid) = &s.tenant_id {
2919 props.insert("tenant_id".to_string(), Value::from(tid.as_str()));
2920 }
2921 if !s.claims.is_empty() {
2922 if let Ok(claims_json) = serde_json::to_value(&s.claims) {
2923 props.insert("claims".to_string(), claims_json);
2924 }
2925 }
2926 let mut log = self.log.lock().await;
2927 log.append(EventKind::SessionScope, None, Some(&proposal.id), props);
2928 }
2929 }
2930
2931 // Pre-execution transactional conflict check (survey §4.3/§5.2.4).
2932 // Off by default; Warn records conflicts; Strict rejects the whole
2933 // proposal before any action runs, since a transactional conflict is
2934 // a property of the action *set* against current state, not an
2935 // isolated action. This is an advisory planning-time gate, not a
2936 // substitute for per-action execution-time validation. Replanned
2937 // proposals are re-checked inside the loop (at the replan quality
2938 // gate below). A pre-execution rejection here deliberately produces
2939 // no execution trajectory — nothing ran; the emitted
2940 // `TransactionConflict` events are the audit record.
2941 let blocked = self.transaction_precheck(¤t_proposal, scope).await;
2942 if !blocked.is_empty() {
2943 let reason = "transactional conflict with current shared state (strict mode); \
2944 see TransactionConflict events for details and resolution"
2945 .to_string();
2946 let results = current_proposal
2947 .actions
2948 .iter()
2949 .map(|a| rejected_result(&a.id, reason.clone()))
2950 .collect();
2951 self.log.lock().await.append(
2952 EventKind::StateRollback,
2953 None,
2954 Some(¤t_proposal.id),
2955 proposal_rejection_boundary_data(
2956 ¤t_proposal,
2957 Some(&original_proposal_digest),
2958 &reason,
2959 ),
2960 );
2961 lineage[0].status = ProposalLineageStatus::Rejected;
2962 lineage[0].rejection_reason = Some(reason);
2963 return finalize_proposal_result(
2964 ProposalResult::for_proposal(proposal, results, Default::default()),
2965 &proposal.id,
2966 &lineage,
2967 &[],
2968 );
2969 }
2970
2971 // Pre-execution admission gates (EPIC A / task A1 — the safety
2972 // seam). Runs every registered AdmissionGate against the proposal
2973 // before any action executes. Aggregation is conjunctive and
2974 // fail-closed: a proposal any gate blocks (or escalates to
2975 // approval) does not run. Like the transactional pre-check above, a
2976 // rejection here produces no execution trajectory — the emitted
2977 // AdmissionGateDecision events are the audit record. No gates
2978 // registered → zero overhead, identical behavior.
2979 let admission = self
2980 .run_admission_gates(¤t_proposal, session_id, scope)
2981 .await;
2982 if !admission.admitted {
2983 let gate = admission.deciding_gate.as_deref().unwrap_or("admission");
2984 let base_reason = admission
2985 .reason
2986 .clone()
2987 .unwrap_or_else(|| "blocked by admission gate".to_string());
2988 // Resolve approval escalations against the durable ledger (A7).
2989 // A hard `Reject` from ANY gate is never overridable — the
2990 // ledger is not consulted at all in that case (an old approval
2991 // for one gate's escalation must not steamroll another gate's
2992 // deny). Otherwise EVERY escalation must resolve to Approved,
2993 // each by its own fingerprint: one Rejected fingerprint blocks,
2994 // one unseen fingerprint stays pending (fail-closed).
2995 let mut approved = false;
2996 let reason = if admission.hard_rejected {
2997 format!("{base_reason} (gate: {gate})")
2998 } else if admission.needs_approval() {
2999 let mut blocking_reason = None;
3000 for esc in &admission.escalations {
3001 match self.admission_decision(&esc.fingerprint).await {
3002 Some(ApprovalDecision::Approved) => continue,
3003 Some(ApprovalDecision::Rejected) => {
3004 blocking_reason = Some(format!(
3005 "rejected by operator (gate: {}; fingerprint: {})",
3006 esc.gate, esc.fingerprint
3007 ));
3008 break;
3009 }
3010 None => {
3011 blocking_reason = Some(format!(
3012 "requires human approval (gate: {}; {}); \
3013 approve fingerprint '{}' then re-run",
3014 esc.gate, esc.reason, esc.fingerprint
3015 ));
3016 break;
3017 }
3018 }
3019 }
3020 match blocking_reason {
3021 None => {
3022 approved = true;
3023 // Audit every durable approval taking effect.
3024 let mut log = self.log.lock().await;
3025 for esc in &admission.escalations {
3026 log.append(
3027 EventKind::ApprovalRecorded,
3028 None,
3029 Some(&proposal.id),
3030 [
3031 (
3032 "fingerprint".to_string(),
3033 Value::from(esc.fingerprint.as_str()),
3034 ),
3035 ("gate".to_string(), Value::from(esc.gate.as_str())),
3036 ("approval".to_string(), Value::from("approved")),
3037 ("applied".to_string(), Value::from(true)),
3038 ]
3039 .into(),
3040 );
3041 }
3042 String::new()
3043 }
3044 Some(r) => r,
3045 }
3046 } else {
3047 format!("{base_reason} (gate: {gate})")
3048 };
3049 if approved {
3050 // Escalation cleared by a durable approval — proceed to
3051 // execution as if admitted.
3052 } else {
3053 let results = current_proposal
3054 .actions
3055 .iter()
3056 .map(|a| {
3057 // Name the specific reason on the offending actions; a
3058 // generic note on the rest (the whole proposal is held,
3059 // since a safety hazard is a property of the set).
3060 if admission.blocked.is_empty() || admission.blocked.contains(&a.id) {
3061 rejected_result(&a.id, reason.clone())
3062 } else {
3063 rejected_result(
3064 &a.id,
3065 format!("proposal blocked by admission gate: {gate}"),
3066 )
3067 }
3068 })
3069 .collect();
3070 self.log.lock().await.append(
3071 EventKind::StateRollback,
3072 None,
3073 Some(¤t_proposal.id),
3074 proposal_rejection_boundary_data(
3075 ¤t_proposal,
3076 Some(&original_proposal_digest),
3077 &reason,
3078 ),
3079 );
3080 lineage[0].status = ProposalLineageStatus::Rejected;
3081 lineage[0].rejection_reason = Some(reason.clone());
3082 let result = finalize_proposal_result(
3083 ProposalResult::for_proposal(proposal, results, Default::default()),
3084 &proposal.id,
3085 &lineage,
3086 &[],
3087 );
3088 // Record the trajectory even though nothing dispatched. This
3089 // return is *before* the execution loop's persist calls, so
3090 // without this an admission rejection is invisible to the
3091 // trajectory store — and the per-tool success rates
3092 // `verify.monte_carlo` derives from it would silently skew
3093 // optimistic, counting only calls that got far enough to run.
3094 // The state map is empty because no action mutated anything.
3095 if let Some(err) = self.persist_trajectory(
3096 proposal,
3097 ¤t_proposal,
3098 &result,
3099 car_memgine::TrajectoryOutcome::Failed,
3100 0,
3101 &HashMap::new(),
3102 ) {
3103 tracing::warn!(
3104 error = %err,
3105 proposal_id = %proposal.id,
3106 "failed to persist trajectory for admission-rejected proposal"
3107 );
3108 }
3109 return finalize_proposal_result(result, &proposal.id, &lineage, &[]);
3110 }
3111 }
3112
3113 let mut accepted_proposal_preimages = vec![AcceptedProposalPreimage {
3114 generation: 0,
3115 proposal_digest: original_proposal_digest,
3116 proposal: proposal.clone(),
3117 }];
3118
3119 loop {
3120 let (result, state_before_map) = self
3121 .execute_inner_with_cancel(¤t_proposal, Some(cancel), session_id, scope)
3122 .await;
3123
3124 // Statuses that trigger rollback + replan: always runtime Failed,
3125 // and (opt-in) validator/policy/capability Rejected.
3126 let replan_triggers = |s: &ActionStatus| {
3127 *s == ActionStatus::Failed
3128 || (config.replan_on_rejected && *s == ActionStatus::Rejected)
3129 };
3130
3131 // A terminal tool failure ends this engine execution. It may not
3132 // enter the retry loop above or the proposal-replan loop here;
3133 // daemon-owned session halting is layered on top by the caller.
3134 let terminal_failure = result.results.iter().any(|result| result.terminal);
3135 // Check if we aborted
3136 let aborted = result.results.iter().any(|r| replan_triggers(&r.status));
3137 let rollback_durability_failed = result.results.iter().any(|result| {
3138 result.error.as_deref().is_some_and(|error| {
3139 error.contains(ROLLBACK_DURABILITY_ERROR)
3140 || error.contains(ROLLBACK_DURABILITY_UNKNOWN)
3141 })
3142 });
3143 if !aborted
3144 || terminal_failure
3145 || attempt >= config.max_replans
3146 || rollback_durability_failed
3147 {
3148 if aborted && !terminal_failure && attempt > 0 && !rollback_durability_failed {
3149 // Exhausted all replan attempts
3150 let mut log = self.log.lock().await;
3151 log.append(
3152 EventKind::ReplanExhausted,
3153 None,
3154 Some(&proposal.id),
3155 [("attempts".to_string(), Value::from(attempt))].into(),
3156 );
3157 }
3158
3159 // Persist trajectory
3160 let outcome = if !aborted {
3161 if attempt > 0 {
3162 car_memgine::TrajectoryOutcome::ReplanSuccess
3163 } else {
3164 car_memgine::TrajectoryOutcome::Success
3165 }
3166 } else if attempt > 0 && !terminal_failure {
3167 car_memgine::TrajectoryOutcome::ReplanExhausted
3168 } else {
3169 car_memgine::TrajectoryOutcome::Failed
3170 };
3171 if let Some(err) = self.persist_trajectory(
3172 proposal,
3173 ¤t_proposal,
3174 &result,
3175 outcome,
3176 attempt,
3177 &state_before_map,
3178 ) {
3179 let mut log = self.log.lock().await;
3180 log.append(
3181 EventKind::ActionFailed,
3182 None,
3183 Some(&proposal.id),
3184 [(
3185 "trajectory_persist_error".to_string(),
3186 Value::from(err.as_str()),
3187 )]
3188 .into(),
3189 );
3190 }
3191
3192 return finalize_proposal_result(
3193 result,
3194 &proposal.id,
3195 &lineage,
3196 &accepted_proposal_preimages,
3197 );
3198 }
3199
3200 // Get replan callback (clone Arc, drop lock immediately)
3201 let callback = {
3202 let guard = self.replan_callback.lock().await;
3203 guard.clone()
3204 };
3205 let Some(callback) = callback else {
3206 // No callback registered — persist trajectory and return
3207 if let Some(err) = self.persist_trajectory(
3208 proposal,
3209 ¤t_proposal,
3210 &result,
3211 car_memgine::TrajectoryOutcome::Failed,
3212 attempt,
3213 &state_before_map,
3214 ) {
3215 let mut log = self.log.lock().await;
3216 log.append(
3217 EventKind::ActionFailed,
3218 None,
3219 Some(&proposal.id),
3220 [(
3221 "trajectory_persist_error".to_string(),
3222 Value::from(err.as_str()),
3223 )]
3224 .into(),
3225 );
3226 }
3227 return finalize_proposal_result(
3228 result,
3229 &proposal.id,
3230 &lineage,
3231 &accepted_proposal_preimages,
3232 );
3233 };
3234
3235 // Build the stable failure context once. Candidate rejection is a
3236 // planning-only loop below: it consumes a replan generation but
3237 // never returns to `execute_inner_with_cancel`, so a failed
3238 // proposal with external effects cannot be dispatched twice.
3239 let failed_actions: Vec<FailedActionSummary> = result
3240 .results
3241 .iter()
3242 .filter(|r| replan_triggers(&r.status) && !r.rolled_back)
3243 .map(|r| {
3244 let action = current_proposal
3245 .actions
3246 .iter()
3247 .find(|a| a.id == r.action_id);
3248 FailedActionSummary {
3249 action_id: r.action_id.clone(),
3250 tool: action.and_then(|a| a.tool.clone()),
3251 error: r.error.clone().unwrap_or_default(),
3252 parameters: action.map(|a| a.parameters.clone()).unwrap_or_default(),
3253 }
3254 })
3255 .collect();
3256
3257 let completed_action_ids: Vec<String> = result
3258 .results
3259 .iter()
3260 .filter(|r| r.rolled_back)
3261 .map(|r| r.action_id.clone())
3262 .collect();
3263
3264 'replan_candidates: loop {
3265 if attempt >= config.max_replans {
3266 let mut log = self.log.lock().await;
3267 log.append(
3268 EventKind::ReplanExhausted,
3269 None,
3270 Some(&proposal.id),
3271 [("attempts".to_string(), Value::from(attempt))].into(),
3272 );
3273 drop(log);
3274 if let Some(err) = self.persist_trajectory(
3275 proposal,
3276 ¤t_proposal,
3277 &result,
3278 car_memgine::TrajectoryOutcome::ReplanExhausted,
3279 attempt,
3280 &state_before_map,
3281 ) {
3282 self.log.lock().await.append(
3283 EventKind::ActionFailed,
3284 None,
3285 Some(&proposal.id),
3286 [("trajectory_persist_error".to_string(), Value::from(err))].into(),
3287 );
3288 }
3289 return finalize_proposal_result(
3290 result,
3291 &proposal.id,
3292 &lineage,
3293 &accepted_proposal_preimages,
3294 );
3295 }
3296
3297 let generation = attempt + 1;
3298 let ctx = ReplanContext {
3299 proposal_id: proposal.id.clone(),
3300 attempt: generation,
3301 failed_actions: failed_actions.clone(),
3302 completed_action_ids: completed_action_ids.clone(),
3303 // Under raw names as before, minus other tenants' keys: a
3304 // tenant proposal's replanner sees the shared keys its tool
3305 // calls run against and its own `tenant:<id>:` keys
3306 // (car#3121). Unscoped: the whole store.
3307 state_snapshot: self
3308 .state
3309 .versioned_snapshot_visible_to(scope.and_then(|s| s.tenant_id.as_deref()))
3310 .0,
3311 replans_remaining: config.max_replans.saturating_sub(generation),
3312 original_source: proposal.source.clone(),
3313 original_action_count: proposal.actions.len(),
3314 original_context: proposal.context.clone(),
3315 };
3316
3317 // Backoff delay between replan attempts
3318 if config.delay_ms > 0 {
3319 tokio::time::sleep(Duration::from_millis(config.delay_ms)).await;
3320 }
3321
3322 // Log replan attempt
3323 {
3324 let mut log = self.log.lock().await;
3325 log.append(
3326 EventKind::ReplanAttempted,
3327 None,
3328 Some(&proposal.id),
3329 [
3330 ("attempt".to_string(), Value::from(generation)),
3331 (
3332 "failed_count".to_string(),
3333 Value::from(ctx.failed_actions.len()),
3334 ),
3335 ]
3336 .into(),
3337 );
3338 // Deep-telemetry breadcrumbs (§3.5.1): record the fork the
3339 // harness took (replan vs accept vs abandon) and the
3340 // approach it discarded, so failure-mode diagnosis can see
3341 // the path *not* taken, not just the path taken.
3342 let rejected_ids: Vec<String> = ctx
3343 .failed_actions
3344 .iter()
3345 .map(|f| f.action_id.clone())
3346 .collect();
3347 log.append(
3348 EventKind::BranchDecision,
3349 None,
3350 Some(&proposal.id),
3351 [
3352 ("branch".to_string(), Value::from("replan")),
3353 (
3354 "reason".to_string(),
3355 Value::from("actions failed; requesting a revised plan"),
3356 ),
3357 ("attempt".to_string(), Value::from(generation)),
3358 ]
3359 .into(),
3360 );
3361 log.append(
3362 EventKind::AlternativeRejected,
3363 None,
3364 Some(&proposal.id),
3365 [
3366 (
3367 "alternative".to_string(),
3368 serde_json::to_value(&rejected_ids).unwrap_or_default(),
3369 ),
3370 (
3371 "reason".to_string(),
3372 Value::from("plan superseded by replan after action failure"),
3373 ),
3374 ]
3375 .into(),
3376 );
3377 }
3378
3379 // Call the model for a new plan
3380 match callback.replan(&ctx).await {
3381 Ok(new_proposal) => {
3382 attempt = generation;
3383 let (new_proposal_preimage, new_proposal_digest) =
3384 match proposal_journal_identity(&new_proposal) {
3385 Ok(identity) => identity,
3386 Err(error) => {
3387 lineage.push(ProposalLineageEntry {
3388 generation,
3389 proposal_id: new_proposal.id.clone(),
3390 proposal_digest: None,
3391 status: ProposalLineageStatus::Rejected,
3392 rejection_reason: Some(error.clone()),
3393 });
3394 self.log.lock().await.append(
3395 EventKind::ReplanRejected,
3396 None,
3397 Some(&proposal.id),
3398 [
3399 ("errors".to_string(), Value::from(error)),
3400 ("attempt".to_string(), Value::from(generation)),
3401 ]
3402 .into(),
3403 );
3404 continue 'replan_candidates;
3405 }
3406 };
3407 if required_replan_proposal_id
3408 .is_some_and(|required| new_proposal.id != required)
3409 {
3410 let reason = format!(
3411 "replan proposal id `{}` does not retain authenticated proposal id `{}`",
3412 new_proposal.id,
3413 required_replan_proposal_id.expect("checked above")
3414 );
3415 lineage.push(ProposalLineageEntry {
3416 generation,
3417 proposal_id: new_proposal.id.clone(),
3418 proposal_digest: Some(new_proposal_digest),
3419 status: ProposalLineageStatus::Rejected,
3420 rejection_reason: Some(reason.clone()),
3421 });
3422 self.log.lock().await.append(
3423 EventKind::ReplanRejected,
3424 None,
3425 Some(&proposal.id),
3426 [
3427 ("errors".to_string(), Value::from(reason)),
3428 ("attempt".to_string(), Value::from(generation)),
3429 ]
3430 .into(),
3431 );
3432 continue 'replan_candidates;
3433 }
3434 if let Err(error) = validate_proposal_action_ids(&new_proposal) {
3435 lineage.push(ProposalLineageEntry {
3436 generation,
3437 proposal_id: new_proposal.id.clone(),
3438 proposal_digest: Some(new_proposal_digest),
3439 status: ProposalLineageStatus::Rejected,
3440 rejection_reason: Some(error.clone()),
3441 });
3442 let mut log = self.log.lock().await;
3443 log.append(
3444 EventKind::ReplanRejected,
3445 None,
3446 Some(&proposal.id),
3447 [
3448 (
3449 "errors".to_string(),
3450 Value::from(format!("invalid proposal: {error}")),
3451 ),
3452 ("attempt".to_string(), Value::from(generation)),
3453 ]
3454 .into(),
3455 );
3456 continue 'replan_candidates;
3457 }
3458
3459 if let Err(error) = validate_proposal_retry_limits(&new_proposal) {
3460 lineage.push(ProposalLineageEntry {
3461 generation,
3462 proposal_id: new_proposal.id.clone(),
3463 proposal_digest: Some(new_proposal_digest),
3464 status: ProposalLineageStatus::Rejected,
3465 rejection_reason: Some(error.clone()),
3466 });
3467 self.log.lock().await.append(
3468 EventKind::ReplanRejected,
3469 None,
3470 Some(&proposal.id),
3471 [
3472 ("errors".to_string(), Value::from(error)),
3473 ("attempt".to_string(), Value::from(generation)),
3474 ]
3475 .into(),
3476 );
3477 continue 'replan_candidates;
3478 }
3479
3480 // Quality gate: verify replan proposal before executing
3481 if config.verify_before_execute {
3482 // The keys this proposal may observe, each action's
3483 // keys resolved in its own namespace, like
3484 // admission (car#3121).
3485 let (current_state, _) = self.state.versioned_snapshot_visible_to(
3486 scope.and_then(|s| s.tenant_id.as_deref()),
3487 );
3488 // Verify against the full registered schemas
3489 // (not just names) so a replan with a bad
3490 // parameter type / missing required field is
3491 // rejected here, per register_tool_schema's
3492 // contract (car-releases#56). The read guard is
3493 // held across the synchronous verify call.
3494 // Same helper the admission gate uses, so the two
3495 // verification points cannot disagree about what is
3496 // fatal. They previously did: this path blocked on the
3497 // loop heuristic and on state-dependent findings that
3498 // the gate treats as advisory, and it passed the tool
3499 // map through unconditionally — so an embedder with no
3500 // registered schemas burned its whole replan budget on
3501 // "unregistered tool" for every call.
3502 let blocking = {
3503 let tools_guard = self.tools.read().await;
3504 crate::verify_gate::blocking_errors(
3505 &new_proposal,
3506 Some(¤t_state),
3507 scope.and_then(|s| s.tenant_id.as_deref()),
3508 &tools_guard,
3509 100,
3510 )
3511 };
3512 if !blocking.is_empty() {
3513 let error_msgs: Vec<String> =
3514 blocking.iter().map(|i| i.message.clone()).collect();
3515 let reason = error_msgs.join("; ");
3516 lineage.push(ProposalLineageEntry {
3517 generation,
3518 proposal_id: new_proposal.id.clone(),
3519 proposal_digest: Some(new_proposal_digest),
3520 status: ProposalLineageStatus::Rejected,
3521 rejection_reason: Some(reason.clone()),
3522 });
3523 let mut log = self.log.lock().await;
3524 log.append(
3525 EventKind::ReplanRejected,
3526 None,
3527 Some(&proposal.id),
3528 [
3529 ("errors".to_string(), Value::from(reason)),
3530 ("attempt".to_string(), Value::from(generation)),
3531 ]
3532 .into(),
3533 );
3534 // Don't execute a broken replan — count as failed attempt
3535 continue 'replan_candidates;
3536 }
3537 }
3538
3539 // Transactional re-check of the replan against the now-
3540 // mutated shared state (neo review #1): a replan runs
3541 // after earlier actions changed state, so it's exactly
3542 // where a fresh stale-assumption/write conflict appears.
3543 // A Strict conflict rejects this replan attempt via the
3544 // same bounded ReplanRejected path (capped by
3545 // max_replans), never an infinite loop.
3546 let replan_blocked = self.transaction_precheck(&new_proposal, scope).await;
3547 if !replan_blocked.is_empty() {
3548 let reason = "transactional conflict with current state".to_string();
3549 lineage.push(ProposalLineageEntry {
3550 generation,
3551 proposal_id: new_proposal.id.clone(),
3552 proposal_digest: Some(new_proposal_digest),
3553 status: ProposalLineageStatus::Rejected,
3554 rejection_reason: Some(reason.clone()),
3555 });
3556 let mut log = self.log.lock().await;
3557 log.append(
3558 EventKind::ReplanRejected,
3559 None,
3560 Some(&proposal.id),
3561 [
3562 ("errors".to_string(), Value::from(reason)),
3563 ("attempt".to_string(), Value::from(generation)),
3564 ]
3565 .into(),
3566 );
3567 drop(log);
3568 continue 'replan_candidates;
3569 }
3570
3571 // Admission gates re-run on EVERY replanned proposal
3572 // (linus review C-2): the replan callback is exactly
3573 // where injected/adversarial content reshapes a plan,
3574 // so a proposal that was clean at first admission must
3575 // not smuggle a hazardous replan past the gates. Same
3576 // fail-closed contract as first admission — a hard
3577 // reject or any unapproved escalation rejects this
3578 // replan attempt (bounded by max_replans).
3579 let replan_admission = self
3580 .run_admission_gates(&new_proposal, session_id, scope)
3581 .await;
3582 if !replan_admission.admitted {
3583 let mut cleared = !replan_admission.hard_rejected
3584 && replan_admission.needs_approval();
3585 if cleared {
3586 for esc in &replan_admission.escalations {
3587 if self.admission_decision(&esc.fingerprint).await
3588 != Some(ApprovalDecision::Approved)
3589 {
3590 cleared = false;
3591 break;
3592 }
3593 }
3594 }
3595 if !cleared {
3596 let gate = replan_admission
3597 .deciding_gate
3598 .as_deref()
3599 .unwrap_or("admission");
3600 let why = replan_admission
3601 .reason
3602 .clone()
3603 .unwrap_or_else(|| "blocked by admission gate".to_string());
3604 let reason =
3605 format!("admission gate blocked replan (gate: {gate}; {why})");
3606 lineage.push(ProposalLineageEntry {
3607 generation,
3608 proposal_id: new_proposal.id.clone(),
3609 proposal_digest: Some(new_proposal_digest),
3610 status: ProposalLineageStatus::Rejected,
3611 rejection_reason: Some(reason.clone()),
3612 });
3613 let mut log = self.log.lock().await;
3614 log.append(
3615 EventKind::ReplanRejected,
3616 None,
3617 Some(&proposal.id),
3618 [
3619 ("errors".to_string(), Value::from(reason)),
3620 ("attempt".to_string(), Value::from(generation)),
3621 ]
3622 .into(),
3623 );
3624 drop(log);
3625 continue 'replan_candidates;
3626 }
3627 }
3628
3629 lineage.push(ProposalLineageEntry {
3630 generation,
3631 proposal_id: new_proposal.id.clone(),
3632 proposal_digest: Some(new_proposal_digest.clone()),
3633 status: ProposalLineageStatus::Accepted,
3634 rejection_reason: None,
3635 });
3636 accepted_proposal_preimages.push(AcceptedProposalPreimage {
3637 generation,
3638 proposal_digest: new_proposal_digest.clone(),
3639 proposal: new_proposal.clone(),
3640 });
3641
3642 // Log accepted proposal
3643 {
3644 let mut log = self.log.lock().await;
3645 log.append(
3646 EventKind::ReplanProposalReceived,
3647 None,
3648 Some(&proposal.id),
3649 [
3650 ("attempt".to_string(), Value::from(generation)),
3651 (
3652 "proposal_id".to_string(),
3653 Value::from(new_proposal.id.as_str()),
3654 ),
3655 (
3656 "proposal_digest".to_string(),
3657 Value::from(new_proposal_digest),
3658 ),
3659 ("proposal".to_string(), new_proposal_preimage),
3660 (
3661 "new_action_count".to_string(),
3662 Value::from(new_proposal.actions.len()),
3663 ),
3664 ]
3665 .into(),
3666 );
3667 }
3668 current_proposal = new_proposal;
3669 break 'replan_candidates;
3670 }
3671 Err(e) => {
3672 // Replan callback itself failed — log and return original failure
3673 let mut log = self.log.lock().await;
3674 log.append(
3675 EventKind::ReplanExhausted,
3676 None,
3677 Some(&proposal.id),
3678 [
3679 ("reason".to_string(), Value::from("callback_error")),
3680 ("error".to_string(), Value::from(e.as_str())),
3681 ("attempt".to_string(), Value::from(generation)),
3682 ]
3683 .into(),
3684 );
3685 if let Some(err) = self.persist_trajectory(
3686 proposal,
3687 ¤t_proposal,
3688 &result,
3689 car_memgine::TrajectoryOutcome::Failed,
3690 attempt,
3691 &state_before_map,
3692 ) {
3693 log.append(
3694 EventKind::ActionFailed,
3695 None,
3696 Some(&proposal.id),
3697 [(
3698 "trajectory_persist_error".to_string(),
3699 Value::from(err.as_str()),
3700 )]
3701 .into(),
3702 );
3703 }
3704 return finalize_proposal_result(
3705 result,
3706 &proposal.id,
3707 &lineage,
3708 &accepted_proposal_preimages,
3709 );
3710 }
3711 }
3712 }
3713 }
3714 }
3715
3716 /// Persist a trajectory to the store if configured.
3717 fn persist_trajectory(
3718 &self,
3719 proposal: &ActionProposal,
3720 current_proposal: &ActionProposal,
3721 result: &ProposalResult,
3722 outcome: car_memgine::TrajectoryOutcome,
3723 attempt: u32,
3724 state_before_map: &HashMap<String, HashMap<String, Value>>,
3725 ) -> Option<String> {
3726 let store = self.trajectory_store.as_ref()?;
3727
3728 let trace_events: Vec<car_memgine::TraceEvent> = result
3729 .results
3730 .iter()
3731 .map(|r| {
3732 let kind = match r.status {
3733 ActionStatus::Succeeded => "action_succeeded",
3734 ActionStatus::Failed => "action_failed",
3735 ActionStatus::Rejected => "action_rejected",
3736 ActionStatus::Skipped => "action_skipped",
3737 _ => "unknown",
3738 };
3739 let tool = current_proposal
3740 .actions
3741 .iter()
3742 .find(|a| a.id == r.action_id)
3743 .and_then(|a| a.tool.clone());
3744 let reward = match r.status {
3745 ActionStatus::Succeeded => Some(1.0),
3746 ActionStatus::Failed => Some(0.0),
3747 ActionStatus::Rejected => Some(0.0),
3748 ActionStatus::Skipped => None,
3749 _ => None,
3750 };
3751 car_memgine::TraceEvent {
3752 kind: kind.to_string(),
3753 action_id: Some(r.action_id.clone()),
3754 tool,
3755 data: r
3756 .error
3757 .as_ref()
3758 .map(|e| serde_json::json!({"error": e}))
3759 .unwrap_or(serde_json::json!({})),
3760 duration_ms: r.duration_ms,
3761 state_before: state_before_map.get(&r.action_id).cloned(),
3762 state_after: if !r.state_changes.is_empty() {
3763 Some(r.state_changes.clone())
3764 } else {
3765 None
3766 },
3767 reward,
3768 }
3769 })
3770 .collect();
3771
3772 let trajectory = car_memgine::Trajectory {
3773 proposal_id: proposal.id.clone(),
3774 source: proposal.source.clone(),
3775 action_count: current_proposal.actions.len(),
3776 events: trace_events,
3777 outcome,
3778 timestamp: chrono::Utc::now(),
3779 duration_ms: result.cost.total_duration_ms,
3780 replan_attempts: attempt,
3781 };
3782
3783 match store.append(&trajectory) {
3784 Ok(()) => None,
3785 Err(e) => Some(e.to_string()),
3786 }
3787 }
3788
3789 async fn rollback_failed_plan_candidate(
3790 &self,
3791 proposal: &ActionProposal,
3792 result: &mut ProposalResult,
3793 pre_plan_snapshot: &HashMap<String, Value>,
3794 pre_plan_transitions: usize,
3795 pre_plan_idempotency_keys: &std::collections::HashSet<String>,
3796 ) -> Result<(), String> {
3797 let executed_proposal = result.final_proposal.as_ref().unwrap_or(proposal);
3798 let executed_proposal_digest = result
3799 .accepted_proposal_preimages
3800 .last()
3801 .map(|accepted| accepted.proposal_digest.clone())
3802 .unwrap_or_else(|| {
3803 proposal_digest(executed_proposal)
3804 .expect("an executed planning candidate already passed proposal identity")
3805 });
3806 let observed_changes_by_action: std::collections::BTreeMap<String, HashMap<String, Value>> =
3807 result
3808 .results
3809 .iter()
3810 .filter(|action_result| {
3811 action_result.status == ActionStatus::Succeeded
3812 && !action_result.state_changes.is_empty()
3813 })
3814 .map(|action_result| {
3815 (
3816 action_result.action_id.clone(),
3817 action_result.state_changes.clone(),
3818 )
3819 })
3820 .collect();
3821 let affected_actions: Vec<String> = observed_changes_by_action.keys().cloned().collect();
3822
3823 // The planning transaction owns the StateStore proposal guard, so this
3824 // exact snapshot/count restore cannot erase another proposal's commit.
3825 let durability = match self
3826 .state
3827 .restore(pre_plan_snapshot.clone(), pre_plan_transitions)
3828 {
3829 Ok(durability) => durability,
3830 Err(error) => {
3831 let detail = record_rollback_durability_error(
3832 &mut result.results,
3833 ROLLBACK_DURABILITY_ERROR,
3834 &error.to_string(),
3835 );
3836 self.log.lock().await.append(
3837 EventKind::ActionFailed,
3838 None,
3839 Some(&proposal.id),
3840 [
3841 ("error".to_string(), Value::from(detail.clone())),
3842 ("attempted".to_string(), Value::from(true)),
3843 ("publication_succeeded".to_string(), Value::from(false)),
3844 ("rollback_succeeded".to_string(), Value::from(false)),
3845 ("stage".to_string(), Value::from("plan_fallback_rollback")),
3846 ]
3847 .into(),
3848 );
3849 return Err(detail);
3850 }
3851 };
3852 let durability_error = match &durability {
3853 RestoreDurability::Durable => None,
3854 RestoreDurability::DurabilityUnknown { error } => Some(error.clone()),
3855 };
3856
3857 {
3858 let mut log = self.log.lock().await;
3859 log.append(
3860 EventKind::StateSnapshot,
3861 None,
3862 Some(&proposal.id),
3863 [(
3864 "state".to_string(),
3865 serde_json::to_value(pre_plan_snapshot).unwrap_or_default(),
3866 )]
3867 .into(),
3868 );
3869 log.append(
3870 EventKind::StateRollback,
3871 None,
3872 Some(&proposal.id),
3873 [
3874 (
3875 "rolled_back_to".to_string(),
3876 Value::from("pre-plan snapshot"),
3877 ),
3878 (
3879 "affected_actions".to_string(),
3880 serde_json::to_value(&affected_actions).unwrap_or_default(),
3881 ),
3882 (
3883 "rolled_back_changes".to_string(),
3884 serde_json::to_value(&observed_changes_by_action).unwrap_or_default(),
3885 ),
3886 (
3887 "changes_semantics".to_string(),
3888 Value::from("rolled_back_failed_planning_candidate"),
3889 ),
3890 (
3891 "proposal_digest".to_string(),
3892 Value::from(executed_proposal_digest),
3893 ),
3894 ("attempted".to_string(), Value::from(true)),
3895 (
3896 "durability_unknown".to_string(),
3897 Value::from(durability_error.is_some()),
3898 ),
3899 ("publication_succeeded".to_string(), Value::from(true)),
3900 ("rollback_succeeded".to_string(), Value::from(true)),
3901 ("stage".to_string(), Value::from("plan_fallback_rollback")),
3902 ]
3903 .into(),
3904 );
3905 }
3906
3907 // Only invalidate entries introduced by this planning transaction.
3908 // Pre-existing keys may have been read by a candidate, but their
3909 // committed result predates the pre-plan snapshot and must survive.
3910 let mut invalidated = std::collections::BTreeSet::new();
3911 {
3912 let mut cache = self.idempotency_cache.lock().await;
3913 for action_result in &result.results {
3914 if action_result.status != ActionStatus::Succeeded {
3915 continue;
3916 }
3917 let Some(action) = executed_proposal
3918 .actions
3919 .iter()
3920 .find(|action| action.id == action_result.action_id)
3921 else {
3922 continue;
3923 };
3924 if !action.idempotent || action.invocation_mode.is_detached() {
3925 continue;
3926 }
3927 let key = idempotency_key(action, None);
3928 if !pre_plan_idempotency_keys.contains(&key) && cache.remove(&key).is_some() {
3929 invalidated.insert(key);
3930 }
3931 }
3932 }
3933 for key in invalidated {
3934 self.journal_idempotency(&key, None).await;
3935 }
3936
3937 // Same reasoning as the abort path above: a rejected planning
3938 // candidate loses its state effects, not the record of what ran. This
3939 // result is observable — `plan_and_execute` returns it directly on a
3940 // rollback durability error, and otherwise keeps it as
3941 // `first_failure` (car#1157). `rolled_back` is the machine-readable
3942 // fact; the warning remains descriptive and must never drive logic.
3943 for action_result in &mut result.results {
3944 if action_result.status == ActionStatus::Succeeded {
3945 action_result.rolled_back = true;
3946 action_result.error = Some(PLAN_FALLBACK_ROLLBACK_WARNING.to_string());
3947 action_result.state_changes.clear();
3948 }
3949 }
3950 if let Some(error) = durability_error {
3951 return Err(record_rollback_durability_error(
3952 &mut result.results,
3953 ROLLBACK_DURABILITY_UNKNOWN,
3954 &error,
3955 ));
3956 }
3957 Ok(())
3958 }
3959
3960 /// Score N candidate proposals, execute the best valid one, fall back to
3961 /// next-best on failure. Combines car-planner scoring with engine execution.
3962 ///
3963 /// Returns the result from whichever proposal was executed (best or fallback).
3964 /// If all candidates fail verification, returns an error result for the first.
3965 pub async fn plan_and_execute(
3966 &self,
3967 candidates: &[ActionProposal],
3968 planner_config: Option<car_planner::PlannerConfig>,
3969 feedback: Option<&car_planner::ToolFeedback>,
3970 ) -> ProposalResult {
3971 if candidates.is_empty() {
3972 return ProposalResult::new("empty", vec![], car_ir::CostSummary::default());
3973 }
3974
3975 // One transaction guard covers ranking's state snapshot, every
3976 // candidate attempt and rollback, and the accepted candidate's commit.
3977 // Candidate execution calls the already-guarded path to avoid recursive
3978 // acquisition of the non-reentrant mutex.
3979 let _proposal_execution = self.state.lock_proposal_execution().await;
3980
3981 // Score all candidates
3982 let planner = car_planner::Planner::new(planner_config.unwrap_or_default());
3983 let tools_guard = self.tools.read().await;
3984 let tool_names: std::collections::HashSet<String> = tools_guard.keys().cloned().collect();
3985 drop(tools_guard);
3986
3987 let pre_plan_snapshot = self.state.snapshot();
3988 let pre_plan_transitions = self.state.transition_count();
3989 let pre_plan_idempotency_keys: std::collections::HashSet<String> = self
3990 .idempotency_cache
3991 .lock()
3992 .await
3993 .keys()
3994 .cloned()
3995 .collect();
3996 let ranked = planner.rank_with_feedback(
3997 candidates,
3998 Some(&pre_plan_snapshot),
3999 Some(&tool_names),
4000 feedback,
4001 );
4002
4003 // Try each valid candidate in score order
4004 let mut first_failure: Option<ProposalResult> = None;
4005 for scored in &ranked {
4006 if !scored.valid {
4007 continue;
4008 }
4009
4010 let proposal = &candidates[scored.index];
4011 let cancel = tokio_util::sync::CancellationToken::new();
4012 let mut result = self
4013 .execute_with_optional_session_already_guarded(proposal, None, None, None, &cancel)
4014 .await;
4015
4016 if result.all_succeeded() {
4017 return result;
4018 }
4019
4020 if self
4021 .rollback_failed_plan_candidate(
4022 proposal,
4023 &mut result,
4024 &pre_plan_snapshot,
4025 pre_plan_transitions,
4026 &pre_plan_idempotency_keys,
4027 )
4028 .await
4029 .is_err()
4030 {
4031 return result;
4032 }
4033
4034 tracing::info!(
4035 proposal_id = %proposal.id,
4036 score = scored.score,
4037 "plan_and_execute: proposal failed, trying next candidate"
4038 );
4039
4040 if first_failure.is_none() {
4041 first_failure = Some(result);
4042 }
4043 }
4044
4045 // Return the first failure result (don't re-execute — avoids duplicate side effects)
4046 first_failure.unwrap_or_else(|| {
4047 ProposalResult::for_proposal(&candidates[0], vec![], car_ir::CostSummary::default())
4048 })
4049 }
4050
4051 /// Execute a single proposal through the runtime loop (no replanning).
4052 /// Returns (result, state_before_map) where state_before_map has per-action snapshots.
4053 ///
4054 /// `session_id`, when `Some`, scopes per-action policy validation
4055 /// to the named session in addition to global policies. Both
4056 /// layers must pass for an action to run; the session layer cannot
4057 /// loosen what global denies.
4058 async fn execute_inner_with_cancel(
4059 &self,
4060 proposal: &ActionProposal,
4061 cancel: Option<&tokio_util::sync::CancellationToken>,
4062 session_id: Option<&str>,
4063 scope: Option<&crate::scope::RuntimeScope>,
4064 ) -> (ProposalResult, HashMap<String, HashMap<String, Value>>) {
4065 // Identity is established before CAR admits the proposal or emits any
4066 // action transition. In particular, an I-JSON-incompatible number may
4067 // not produce a journal row whose mandatory JCS digest is missing.
4068 let (proposal_preimage, proposal_digest) = match proposal_journal_identity(proposal) {
4069 Ok(identity) => identity,
4070 Err(error) => {
4071 let results = proposal
4072 .actions
4073 .iter()
4074 .map(|action| rejected_result(&action.id, error.clone()))
4075 .collect();
4076 self.log.lock().await.append(
4077 EventKind::StateRollback,
4078 None,
4079 Some(&proposal.id),
4080 proposal_rejection_boundary_data(proposal, None, &error),
4081 );
4082 return (
4083 ProposalResult::for_proposal(proposal, results, CostSummary::default()),
4084 HashMap::new(),
4085 );
4086 }
4087 };
4088
4089 // Whether validator/policy/capability rejections should count toward
4090 // the abort-and-replan path (default false). Read once up front so the
4091 // per-action loop doesn't take the replan_config lock repeatedly (and
4092 // never inside join_all). Independent lock from log/tools/policies/
4093 // capabilities, so no lock-ordering conflict.
4094 let replan_on_rejected = self.replan_config.read().await.replan_on_rejected;
4095
4096 // Generate trace_id for this proposal execution
4097 let trace_id = Uuid::new_v4().to_string();
4098
4099 // Begin root span for proposal execution
4100 let root_span_id = {
4101 let mut log = self.log.lock().await;
4102 log.begin_span(
4103 "proposal.execute",
4104 &trace_id,
4105 None,
4106 [("proposal_id".to_string(), Value::from(proposal.id.as_str()))].into(),
4107 )
4108 };
4109
4110 // Log proposal received
4111 {
4112 let mut log = self.log.lock().await;
4113 log.append(
4114 EventKind::ProposalReceived,
4115 None,
4116 Some(&proposal.id),
4117 [
4118 ("source".to_string(), Value::from(proposal.source.as_str())),
4119 (
4120 "action_count".to_string(),
4121 Value::from(proposal.actions.len()),
4122 ),
4123 ("proposal".to_string(), proposal_preimage.clone()),
4124 (
4125 "proposal_digest".to_string(),
4126 Value::from(proposal_digest.clone()),
4127 ),
4128 ]
4129 .into(),
4130 );
4131 }
4132
4133 // Capability check: max_actions budget for entire proposal
4134 {
4135 let caps = self.capabilities.read().await;
4136 if let Some(ref cap) = *caps {
4137 if !cap.actions_within_budget(proposal.actions.len() as u32) {
4138 let reason = format!(
4139 "capability denied: proposal has {} actions, max allowed is {:?}",
4140 proposal.actions.len(),
4141 cap.max_actions
4142 );
4143 let mut action_results = Vec::new();
4144 let mut log = self.log.lock().await;
4145 for action in &proposal.actions {
4146 log.append(
4147 EventKind::ActionRejected,
4148 Some(&action.id),
4149 Some(&proposal.id),
4150 action_outcome_data("proposal_capability", &reason, false, None),
4151 );
4152 action_results.push(rejected_result(&action.id, reason.clone()));
4153 }
4154 log.append(
4155 EventKind::StateRollback,
4156 None,
4157 Some(&proposal.id),
4158 proposal_rejection_boundary_data(proposal, Some(&proposal_digest), &reason),
4159 );
4160 drop(log);
4161 return (
4162 ProposalResult::for_proposal(
4163 proposal,
4164 action_results,
4165 CostSummary::default(),
4166 ),
4167 HashMap::new(),
4168 );
4169 }
4170 }
4171 }
4172
4173 // Snapshot for rollback. When the proposal carries a tenant scope,
4174 // snapshot only that tenant's namespace so a rollback can't clobber
4175 // concurrent tenants' state (EPIC E / E2). No tenant → unchanged
4176 // full snapshot.
4177 let rollback_tenant: Option<&str> = scope.and_then(|s| s.tenant_id.as_deref());
4178 let snapshot = match rollback_tenant {
4179 Some(_) => self.state.snapshot_scoped(rollback_tenant),
4180 None => self.state.snapshot(),
4181 };
4182 let transition_count = self.state.transition_count();
4183
4184 let mut results: Vec<ActionResult> = Vec::new();
4185 // Per-action state snapshots captured before execution (for TraceEvent.state_before).
4186 let mut state_before_map: HashMap<String, HashMap<String, Value>> = HashMap::new();
4187 let mut aborted = false;
4188 let mut budget_exceeded = false;
4189 let mut total_retries: u32 = 0;
4190
4191 // Running cost counters for budget enforcement
4192 let mut running_tool_calls: u32 = 0;
4193 let mut running_actions: u32 = 0;
4194 let mut running_duration_ms: f64 = 0.0;
4195
4196 // Snapshot the budget once
4197 let budget = self.cost_budget.read().await.clone();
4198
4199 // Build DAG
4200 let levels = build_dag(&proposal.actions);
4201
4202 let mut canceled = false;
4203 for level in &levels {
4204 // Cooperative cancellation check at the level boundary.
4205 // Actions already in flight aren't interrupted (we can't
4206 // safely cancel a tool call dispatched to a user-provided
4207 // executor), but every action that hadn't started runs
4208 // is recorded as canceled with a clear reason.
4209 if !canceled {
4210 if let Some(token) = cancel {
4211 if token.is_cancelled() {
4212 canceled = true;
4213 }
4214 }
4215 }
4216 if canceled {
4217 for &idx in level {
4218 let action = &proposal.actions[idx];
4219 let reason = format!("{CANCELED_PREFIX}cancellation requested by caller");
4220 self.log.lock().await.append(
4221 EventKind::ActionSkipped,
4222 Some(&action.id),
4223 Some(&proposal.id),
4224 action_outcome_data("cancellation", &reason, false, None),
4225 );
4226 results.push(canceled_result(
4227 &action.id,
4228 "cancellation requested by caller",
4229 ));
4230 }
4231 continue;
4232 }
4233 if aborted || budget_exceeded {
4234 let skip_reason = if budget_exceeded {
4235 "cost budget exceeded"
4236 } else {
4237 "skipped due to earlier abort"
4238 };
4239 for &idx in level {
4240 let action = &proposal.actions[idx];
4241 let stage = if budget_exceeded {
4242 "cost_budget"
4243 } else {
4244 "dependency_abort"
4245 };
4246 self.log.lock().await.append(
4247 EventKind::ActionSkipped,
4248 Some(&action.id),
4249 Some(&proposal.id),
4250 action_outcome_data(stage, skip_reason, false, None),
4251 );
4252 results.push(skipped_result(&action.id, skip_reason));
4253 }
4254 continue;
4255 }
4256
4257 if level_runs_sequentially(level, &proposal.actions) {
4258 // Sequential execution
4259 for &idx in level {
4260 if aborted || budget_exceeded {
4261 let skip_reason = if budget_exceeded {
4262 "cost budget exceeded"
4263 } else {
4264 "skipped due to abort"
4265 };
4266 let action = &proposal.actions[idx];
4267 let stage = if budget_exceeded {
4268 "cost_budget"
4269 } else {
4270 "dependency_abort"
4271 };
4272 self.log.lock().await.append(
4273 EventKind::ActionSkipped,
4274 Some(&action.id),
4275 Some(&proposal.id),
4276 action_outcome_data(stage, skip_reason, false, None),
4277 );
4278 results.push(skipped_result(&action.id, skip_reason));
4279 continue;
4280 }
4281
4282 // Budget check before execution
4283 if let Some(ref b) = budget {
4284 if let Some(max) = b.max_actions {
4285 if running_actions >= max {
4286 budget_exceeded = true;
4287 self.log.lock().await.append(
4288 EventKind::ActionSkipped,
4289 Some(&proposal.actions[idx].id),
4290 Some(&proposal.id),
4291 action_outcome_data(
4292 "cost_budget",
4293 "cost budget exceeded",
4294 false,
4295 None,
4296 ),
4297 );
4298 results.push(skipped_result(
4299 &proposal.actions[idx].id,
4300 "cost budget exceeded",
4301 ));
4302 continue;
4303 }
4304 }
4305 if let Some(max) = b.max_tool_calls {
4306 if proposal.actions[idx].action_type == ActionType::ToolCall
4307 && running_tool_calls >= max
4308 {
4309 budget_exceeded = true;
4310 self.log.lock().await.append(
4311 EventKind::ActionSkipped,
4312 Some(&proposal.actions[idx].id),
4313 Some(&proposal.id),
4314 action_outcome_data(
4315 "cost_budget",
4316 "cost budget exceeded",
4317 false,
4318 None,
4319 ),
4320 );
4321 results.push(skipped_result(
4322 &proposal.actions[idx].id,
4323 "cost budget exceeded",
4324 ));
4325 continue;
4326 }
4327 }
4328 if let Some(max) = b.max_duration_ms {
4329 if running_duration_ms >= max {
4330 budget_exceeded = true;
4331 self.log.lock().await.append(
4332 EventKind::ActionSkipped,
4333 Some(&proposal.actions[idx].id),
4334 Some(&proposal.id),
4335 action_outcome_data(
4336 "cost_budget",
4337 "cost budget exceeded",
4338 false,
4339 None,
4340 ),
4341 );
4342 results.push(skipped_result(
4343 &proposal.actions[idx].id,
4344 "cost budget exceeded",
4345 ));
4346 continue;
4347 }
4348 }
4349 }
4350
4351 state_before_map.insert(
4352 proposal.actions[idx].id.clone(),
4353 snapshot_relevant_keys(
4354 &*action_state_view(
4355 &self.state,
4356 &proposal.actions[idx],
4357 rollback_tenant,
4358 ),
4359 &proposal.actions[idx],
4360 ),
4361 );
4362 let (ar, action_retries) = self
4363 .process_action(
4364 &proposal.actions[idx],
4365 &proposal.id,
4366 &trace_id,
4367 &root_span_id,
4368 session_id,
4369 scope,
4370 )
4371 .await;
4372 total_retries += action_retries;
4373
4374 // Update running counters
4375 if ar.status == ActionStatus::Succeeded
4376 && proposal.actions[idx].action_type == ActionType::ToolCall
4377 {
4378 running_tool_calls += 1;
4379 }
4380 if ar.status != ActionStatus::Skipped {
4381 running_actions += 1;
4382 }
4383 if let Some(d) = ar.duration_ms {
4384 running_duration_ms += d;
4385 }
4386
4387 if ar.terminal
4388 || requires_integrity_rollback(&ar)
4389 || ((ar.status == ActionStatus::Failed
4390 || (replan_on_rejected && ar.status == ActionStatus::Rejected))
4391 && proposal.actions[idx].failure_behavior == FailureBehavior::Abort)
4392 {
4393 aborted = true;
4394 }
4395 results.push(ar);
4396 }
4397 } else {
4398 // Concurrent execution via futures::join_all
4399 // Snapshot only relevant keys per action (all see same pre-level state)
4400 for &idx in level {
4401 state_before_map.insert(
4402 proposal.actions[idx].id.clone(),
4403 snapshot_relevant_keys(
4404 &*action_state_view(
4405 &self.state,
4406 &proposal.actions[idx],
4407 rollback_tenant,
4408 ),
4409 &proposal.actions[idx],
4410 ),
4411 );
4412 }
4413 let futs: Vec<_> = level
4414 .iter()
4415 .map(|&idx| {
4416 self.process_action(
4417 &proposal.actions[idx],
4418 &proposal.id,
4419 &trace_id,
4420 &root_span_id,
4421 session_id,
4422 scope,
4423 )
4424 })
4425 .collect();
4426 let level_results = futures::future::join_all(futs).await;
4427
4428 for (i, (ar, action_retries)) in level_results.into_iter().enumerate() {
4429 let idx = level[i];
4430 total_retries += action_retries;
4431 if ar.status == ActionStatus::Succeeded
4432 && proposal.actions[idx].action_type == ActionType::ToolCall
4433 {
4434 running_tool_calls += 1;
4435 }
4436 if ar.status != ActionStatus::Skipped {
4437 running_actions += 1;
4438 }
4439 if let Some(d) = ar.duration_ms {
4440 running_duration_ms += d;
4441 }
4442 if ar.terminal || requires_integrity_rollback(&ar) {
4443 aborted = true;
4444 }
4445 results.push(ar);
4446 }
4447 }
4448 }
4449
4450 let observed_changes_by_action: std::collections::BTreeMap<String, HashMap<String, Value>> =
4451 proposal
4452 .actions
4453 .iter()
4454 .filter_map(|action| {
4455 results
4456 .iter()
4457 .find(|result| {
4458 result.action_id == action.id
4459 && result.status == ActionStatus::Succeeded
4460 && !result.state_changes.is_empty()
4461 })
4462 .map(|result| (action.id.clone(), result.state_changes.clone()))
4463 })
4464 .collect();
4465 let affected_actions: Vec<String> = observed_changes_by_action.keys().cloned().collect();
4466
4467 // Handle rollback. Per-action StateChanged rows are provisional until
4468 // this proposal-level transaction boundary is durably recorded.
4469 if aborted {
4470 // Tenant-scoped rollback when the proposal is tenant-scoped
4471 // (EPIC E / E2), else the full restore.
4472 let rollback = match rollback_tenant {
4473 Some(_) => {
4474 self.state
4475 .restore_scoped(rollback_tenant, snapshot.clone(), transition_count)
4476 }
4477 None => self.state.restore(snapshot.clone(), transition_count),
4478 };
4479
4480 let rollback_durability = match rollback {
4481 Ok(durability) => Some(durability),
4482 Err(error) => {
4483 let detail = record_rollback_durability_error(
4484 &mut results,
4485 ROLLBACK_DURABILITY_ERROR,
4486 &error.to_string(),
4487 );
4488 self.log.lock().await.append(
4489 EventKind::ActionFailed,
4490 None,
4491 Some(&proposal.id),
4492 [
4493 ("error".to_string(), Value::from(detail)),
4494 ("attempted".to_string(), Value::from(true)),
4495 ("publication_succeeded".to_string(), Value::from(false)),
4496 ("in_memory_state_preserved".to_string(), Value::from(true)),
4497 (
4498 "idempotency_entries_preserved".to_string(),
4499 Value::from(true),
4500 ),
4501 ("rollback_succeeded".to_string(), Value::from(false)),
4502 ("stage".to_string(), Value::from("proposal_rollback")),
4503 ]
4504 .into(),
4505 );
4506 None
4507 }
4508 };
4509
4510 if let Some(durability) = rollback_durability {
4511 let durability_error = match &durability {
4512 RestoreDurability::Durable => None,
4513 RestoreDurability::DurabilityUnknown { error } => Some(error.clone()),
4514 };
4515 if let Some(error) = &durability_error {
4516 record_rollback_durability_error(
4517 &mut results,
4518 ROLLBACK_DURABILITY_UNKNOWN,
4519 error,
4520 );
4521 }
4522 let mut log = self.log.lock().await;
4523 log.append(
4524 EventKind::StateSnapshot,
4525 None,
4526 Some(&proposal.id),
4527 [(
4528 "state".to_string(),
4529 serde_json::to_value(&snapshot).unwrap_or_default(),
4530 )]
4531 .into(),
4532 );
4533 log.append(
4534 EventKind::StateRollback,
4535 None,
4536 Some(&proposal.id),
4537 [
4538 (
4539 "rolled_back_to".to_string(),
4540 Value::from("pre-proposal snapshot"),
4541 ),
4542 (
4543 "affected_actions".to_string(),
4544 serde_json::to_value(&affected_actions).unwrap_or_default(),
4545 ),
4546 (
4547 "rolled_back_changes".to_string(),
4548 serde_json::to_value(&observed_changes_by_action).unwrap_or_default(),
4549 ),
4550 (
4551 "changes_semantics".to_string(),
4552 Value::from("rolled_back_provisional_state_mutations"),
4553 ),
4554 (
4555 "proposal_digest".to_string(),
4556 Value::from(proposal_digest.clone()),
4557 ),
4558 ("attempted".to_string(), Value::from(true)),
4559 (
4560 "durability_unknown".to_string(),
4561 Value::from(durability_error.is_some()),
4562 ),
4563 ("publication_succeeded".to_string(), Value::from(true)),
4564 ("rollback_succeeded".to_string(), Value::from(true)),
4565 ("stage".to_string(), Value::from("proposal_rollback")),
4566 ]
4567 .into(),
4568 );
4569 drop(log);
4570
4571 // Clear idempotency cache for rolled-back actions only after
4572 // the state journal has durably accepted the rollback.
4573 let mut invalidated: Vec<String> = Vec::new();
4574 {
4575 let mut cache = self.idempotency_cache.lock().await;
4576 for r in &results {
4577 if r.status == ActionStatus::Succeeded {
4578 for action in &proposal.actions {
4579 if action.id == r.action_id && action.idempotent {
4580 let key = idempotency_key(action, scope);
4581 cache.remove(&key);
4582 invalidated.push(key);
4583 }
4584 }
4585 }
4586 }
4587 }
4588 // Record tombstones so the durable journal (C3) doesn't
4589 // resurrect a rolled-back result on reload.
4590 for key in &invalidated {
4591 self.journal_idempotency(key, None).await;
4592 }
4593 // The proposal did not commit, so no action's state changes
4594 // survive. Each action's STATUS still records whether that
4595 // action executed; `rolled_back` independently records whether
4596 // that successful execution preceded this rollback. Folding
4597 // the second fact into ActionStatus erased which action
4598 // aborted the proposal and marked `failed` precisely the
4599 // actions whose external effects DID happen and may remain.
4600 // The warning is human-readable context, never control flow
4601 // (car#1157).
4602 for result in &mut results {
4603 if result.status == ActionStatus::Succeeded {
4604 result.rolled_back = true;
4605 result.error = Some(ROLLBACK_WARNING.to_string());
4606 result.state_changes.clear();
4607 }
4608 }
4609 }
4610 } else {
4611 self.log.lock().await.append(
4612 EventKind::StateCommitted,
4613 None,
4614 Some(&proposal.id),
4615 [
4616 (
4617 "affected_actions".to_string(),
4618 serde_json::to_value(&affected_actions).unwrap_or_default(),
4619 ),
4620 (
4621 "committed_changes".to_string(),
4622 serde_json::to_value(&observed_changes_by_action).unwrap_or_default(),
4623 ),
4624 (
4625 "changes_semantics".to_string(),
4626 Value::from("committed_provisional_state_mutations"),
4627 ),
4628 ("proposal_digest".to_string(), Value::from(proposal_digest)),
4629 ("attempted".to_string(), Value::from(true)),
4630 ("stage".to_string(), Value::from("proposal_commit")),
4631 ]
4632 .into(),
4633 );
4634 }
4635
4636 // Sort results to match original action order
4637 let action_order: HashMap<String, usize> = proposal
4638 .actions
4639 .iter()
4640 .enumerate()
4641 .map(|(i, a)| (a.id.clone(), i))
4642 .collect();
4643 results.sort_by_key(|r| {
4644 action_order
4645 .get(&r.action_id)
4646 .copied()
4647 .unwrap_or(usize::MAX)
4648 });
4649
4650 // Compute cost summary from results
4651 let mut cost = CostSummary::default();
4652 for r in &results {
4653 let action = action_order
4654 .get(&r.action_id)
4655 .and_then(|&i| proposal.actions.get(i));
4656 match r.status {
4657 ActionStatus::Succeeded => {
4658 cost.actions_executed += 1;
4659 if let Some(a) = action {
4660 if a.action_type == ActionType::ToolCall {
4661 cost.tool_calls += 1;
4662 }
4663 }
4664 }
4665 // A failed action ran — the tool was invoked and errored — so it
4666 // belongs in `actions_executed`. A rejected one was blocked by
4667 // the validator or a policy and never started; counting it as
4668 // executed reported work that never happened (car#624).
4669 ActionStatus::Failed => {
4670 cost.actions_executed += 1;
4671 }
4672 ActionStatus::Rejected => {
4673 cost.actions_rejected += 1;
4674 }
4675 ActionStatus::Skipped => {
4676 cost.actions_skipped += 1;
4677 }
4678 _ => {}
4679 }
4680 if let Some(d) = r.duration_ms {
4681 cost.total_duration_ms += d;
4682 }
4683 }
4684
4685 // Set retries from inline counter
4686 cost.retries = total_retries;
4687
4688 // End root span — Ok if no abort, Error if aborted
4689 {
4690 let span_status = if aborted {
4691 SpanStatus::Error
4692 } else {
4693 SpanStatus::Ok
4694 };
4695 let mut log = self.log.lock().await;
4696 log.end_span(&root_span_id, span_status);
4697 }
4698
4699 let proposal_result = ProposalResult::for_proposal(proposal, results, cost);
4700
4701 // Post-execution: auto-distill skills from this execution trace
4702 if self.auto_distill {
4703 if let Some(ref memgine) = self.memgine {
4704 // Convert results to TraceEvents for distillation
4705 let trace_events: Vec<car_memgine::TraceEvent> = proposal_result
4706 .results
4707 .iter()
4708 .map(|r| {
4709 let kind = match r.status {
4710 ActionStatus::Succeeded => "action_succeeded",
4711 ActionStatus::Failed => "action_failed",
4712 ActionStatus::Rejected => "action_rejected",
4713 ActionStatus::Skipped => "action_skipped",
4714 _ => "unknown",
4715 };
4716 // Find the matching action to get the tool name
4717 let tool = proposal
4718 .actions
4719 .iter()
4720 .find(|a| a.id == r.action_id)
4721 .and_then(|a| a.tool.clone());
4722 let mut data = serde_json::Map::new();
4723 if let Some(ref e) = r.error {
4724 data.insert("error".into(), Value::from(e.as_str()));
4725 }
4726 if let Some(ref o) = r.output {
4727 data.insert("output".into(), o.clone());
4728 }
4729 car_memgine::TraceEvent {
4730 kind: kind.to_string(),
4731 action_id: Some(r.action_id.clone()),
4732 tool,
4733 data: Value::Object(data),
4734 duration_ms: r.duration_ms,
4735 reward: match r.status {
4736 ActionStatus::Succeeded => Some(1.0),
4737 ActionStatus::Failed | ActionStatus::Rejected => Some(0.0),
4738 _ => None,
4739 },
4740 ..Default::default()
4741 }
4742 })
4743 .collect();
4744
4745 let mut engine = memgine.lock().await;
4746 let skills = engine.distill_skills(&trace_events).await;
4747 if !skills.is_empty() {
4748 let count = skills.len();
4749 // Validation-gated ingest (SkillOpt-inspired). Distilled
4750 // skills are NOT trusted straight into the active pool; each
4751 // enters as a PROVISIONAL candidate on trial and must prove
4752 // itself (or beat its incumbent) before the promotion gate in
4753 // consolidate() makes it Active. Tenant is threaded through so
4754 // candidates are stamped for the calling tenant's namespace.
4755 let tenant = scope.and_then(|s| s.tenant_id.as_deref());
4756 let provisional = engine.ingest_provisional_candidates(&skills, tenant);
4757
4758 // Log the distillation event
4759 let mut log = self.log.lock().await;
4760 log.append(
4761 EventKind::SkillDistilled,
4762 None,
4763 Some(&proposal_result.proposal_id),
4764 [
4765 ("skills_count".to_string(), Value::from(count)),
4766 ("provisional_ingested".to_string(), Value::from(provisional)),
4767 (
4768 "skill_names".to_string(),
4769 Value::from(
4770 skills.iter().map(|s| s.name.as_str()).collect::<Vec<_>>(),
4771 ),
4772 ),
4773 ]
4774 .into(),
4775 );
4776
4777 // Check if any domains need evolution
4778 let threshold = engine.evolution_threshold();
4779 let domains = engine.domains_needing_evolution(threshold);
4780 for domain in &domains {
4781 // Collect failed events for this domain
4782 let failed: Vec<car_memgine::TraceEvent> = trace_events
4783 .iter()
4784 .filter(|e| {
4785 matches!(e.kind.as_str(), "action_failed" | "action_rejected")
4786 })
4787 .cloned()
4788 .collect();
4789 if !failed.is_empty() {
4790 let evolved = engine.evolve_skills(&failed, domain).await;
4791 if !evolved.is_empty() {
4792 log.append(
4793 EventKind::EvolutionTriggered,
4794 None,
4795 Some(&proposal_result.proposal_id),
4796 [
4797 ("domain".to_string(), Value::from(domain.as_str())),
4798 ("new_skills".to_string(), Value::from(evolved.len())),
4799 ]
4800 .into(),
4801 );
4802 }
4803 }
4804 }
4805 }
4806 }
4807 }
4808
4809 (proposal_result, state_before_map)
4810 }
4811
4812 /// Process a single action: capability → validate → policy → idempotency → execute.
4813 /// (Idempotency dedup runs AFTER the deny checks — review C3: a durable
4814 /// cached result must never outlive a tool's revocation.)
4815 /// Returns (ActionResult, retries_count).
4816 async fn process_action(
4817 &self,
4818 action: &Action,
4819 proposal_id: &str,
4820 trace_id: &str,
4821 parent_span_id: &str,
4822 session_id: Option<&str>,
4823 scope: Option<&crate::scope::RuntimeScope>,
4824 ) -> (ActionResult, u32) {
4825 // Derive action type name for span naming
4826 let action_type_name = serde_json::to_string(&action.action_type)
4827 .unwrap_or_default()
4828 .trim_matches('"')
4829 .to_string();
4830 let span_name = format!("action.{}", action_type_name);
4831
4832 // Begin child span for this action
4833 let action_span_id = {
4834 let mut attrs: HashMap<String, Value> = HashMap::new();
4835 attrs.insert("action_id".to_string(), Value::from(action.id.as_str()));
4836 if let Some(ref tool) = action.tool {
4837 attrs.insert("tool".to_string(), Value::from(tool.as_str()));
4838 }
4839 let mut log = self.log.lock().await;
4840 log.begin_span(&span_name, trace_id, Some(parent_span_id), attrs)
4841 };
4842
4843 // Execute the action pipeline and capture result
4844 let (result, retries) = self
4845 .process_action_inner(action, proposal_id, session_id, scope)
4846 .await;
4847
4848 // End action span based on result status
4849 let span_status = match result.status {
4850 ActionStatus::Succeeded => SpanStatus::Ok,
4851 ActionStatus::Failed | ActionStatus::Rejected => SpanStatus::Error,
4852 _ => SpanStatus::Unset,
4853 };
4854 {
4855 let mut log = self.log.lock().await;
4856 log.end_span(&action_span_id, span_status);
4857 }
4858
4859 (result, retries)
4860 }
4861
4862 /// Inner action processing: idempotency -> validate -> policy -> execute.
4863 /// Returns (ActionResult, retries_count).
4864 #[instrument(
4865 name = "action.process",
4866 skip_all,
4867 fields(
4868 action_id = %action.id,
4869 action_type = ?action.action_type,
4870 tool = action.tool.as_deref().unwrap_or("none"),
4871 )
4872 )]
4873 async fn process_action_inner(
4874 &self,
4875 action: &Action,
4876 proposal_id: &str,
4877 session_id: Option<&str>,
4878 scope: Option<&crate::scope::RuntimeScope>,
4879 ) -> (ActionResult, u32) {
4880 // Capability check
4881 {
4882 let caps = self.capabilities.read().await;
4883 if let Some(ref cap) = *caps {
4884 // Check tool capability for ToolCall actions
4885 if action.action_type == ActionType::ToolCall {
4886 if let Some(ref tool_name) = action.tool {
4887 if !cap.tool_allowed(tool_name) {
4888 let reason =
4889 format!("capability denied: tool '{}' not allowed", tool_name);
4890 let mut log = self.log.lock().await;
4891 log.append(
4892 EventKind::ActionRejected,
4893 Some(&action.id),
4894 Some(proposal_id),
4895 action_outcome_data("capability", &reason, false, None),
4896 );
4897 return (rejected_result(&action.id, reason), 0);
4898 }
4899 }
4900 }
4901
4902 // Check state key capability for StateWrite/StateRead actions
4903 if action.action_type == ActionType::StateWrite
4904 || action.action_type == ActionType::StateRead
4905 {
4906 if let Some(key) = action.parameters.get("key").and_then(|v| v.as_str()) {
4907 if !cap.state_key_allowed(key) {
4908 let reason =
4909 format!("capability denied: state key '{}' not allowed", key);
4910 let mut log = self.log.lock().await;
4911 log.append(
4912 EventKind::ActionRejected,
4913 Some(&action.id),
4914 Some(proposal_id),
4915 action_outcome_data("capability", &reason, false, None),
4916 );
4917 return (rejected_result(&action.id, reason), 0);
4918 }
4919 }
4920 }
4921 }
4922 }
4923
4924 // Validate
4925 let tools = self.tools.read().await;
4926 // Against the namespace the action executes in (car#3121): a state
4927 // action's preconditions and dependencies name keys in its tenant's
4928 // view, a tool call's name shared keys, and never another tenant's.
4929 let view = action_state_view(
4930 &self.state,
4931 action,
4932 scope.and_then(|s| s.tenant_id.as_deref()),
4933 );
4934 let validation = validate_action(action, &*view, &tools);
4935 drop(tools);
4936
4937 if !validation.valid() {
4938 let error = validation
4939 .errors
4940 .iter()
4941 .map(|e| e.reason.as_str())
4942 .collect::<Vec<_>>()
4943 .join("; ");
4944 let mut log = self.log.lock().await;
4945 log.append(
4946 EventKind::ActionRejected,
4947 Some(&action.id),
4948 Some(proposal_id),
4949 action_outcome_data("validation", &error, false, None),
4950 );
4951 return (rejected_result(&action.id, error), 0);
4952 }
4953
4954 // Policy check — global registry plus, when the proposal is
4955 // executed under a session, that session's registry. Both
4956 // layers must pass for the action to proceed; session
4957 // policies are additive deny rules — they can deny what
4958 // global allows but cannot allow what global denies.
4959 {
4960 let mut violations = {
4961 let policies = self.policies.read().await;
4962 policies.check(action, &self.state)
4963 };
4964 if let Some(sid) = session_id {
4965 // Snapshot the per-session engine handle out from under
4966 // the outer registry lock so the inner check holds
4967 // only the engine's own RwLock — preserves the
4968 // documented lock-ordering discipline.
4969 let session_engine = {
4970 let sessions = self.session_policies.read().await;
4971 sessions.get(sid).cloned()
4972 };
4973 if let Some(engine) = session_engine {
4974 let engine = engine.read().await;
4975 violations.extend(engine.check(action, &self.state));
4976 } else {
4977 let error = format!(
4978 "unknown session id '{sid}' — open one via Runtime::open_session before executing under a session"
4979 );
4980 // Unknown session id — refuse the action rather
4981 // than silently fall back to global-only. A
4982 // proposal submitted under a closed session
4983 // shouldn't run with looser rules than the caller
4984 // intended.
4985 let mut log = self.log.lock().await;
4986 log.append(
4987 EventKind::PolicyViolation,
4988 Some(&action.id),
4989 Some(proposal_id),
4990 action_outcome_data("policy", &error, false, None),
4991 );
4992 return (rejected_result(&action.id, error), 0);
4993 }
4994 }
4995 if !violations.is_empty() {
4996 let error = violations
4997 .iter()
4998 .map(|v| format!("policy '{}': {}", v.policy_name, v.reason))
4999 .collect::<Vec<_>>()
5000 .join("; ");
5001 let mut log = self.log.lock().await;
5002 log.append(
5003 EventKind::PolicyViolation,
5004 Some(&action.id),
5005 Some(proposal_id),
5006 action_outcome_data("policy", &error, false, None),
5007 );
5008 return (rejected_result(&action.id, error), 0);
5009 }
5010 }
5011
5012 // Idempotency check — deliberately AFTER capability + policy
5013 // (linus review C3): with the durable journal, a cached result
5014 // consulted first would let a tool that is denied TODAY serve
5015 // yesterday's cached result after every restart, forever. Deny
5016 // rules must win over dedup.
5017 if action.idempotent && !action.invocation_mode.is_detached() {
5018 let key = idempotency_key(action, scope);
5019 let cache = self.idempotency_cache.lock().await;
5020 if let Some(cached) = cache.get(&key) {
5021 let mut log = self.log.lock().await;
5022 log.append(
5023 EventKind::ActionDeduplicated,
5024 Some(&action.id),
5025 Some(proposal_id),
5026 [
5027 (
5028 "cached_action_id".to_string(),
5029 Value::from(cached.action_id.as_str()),
5030 ),
5031 ("attempted".to_string(), Value::from(false)),
5032 ("stage".to_string(), Value::from("idempotency_cache")),
5033 ]
5034 .into(),
5035 );
5036 return (
5037 ActionResult {
5038 action_id: action.id.clone(),
5039 status: cached.status.clone(),
5040 output: cached.output.clone(),
5041 error: cached.error.clone(),
5042 terminal: cached.terminal,
5043 // The cached action committed these mutations in an
5044 // earlier proposal. This proposal did not observe or
5045 // apply them, so its actual-change evidence is empty.
5046 state_changes: HashMap::new(),
5047 rolled_back: false,
5048 duration_ms: Some(0.0),
5049 timestamp: chrono::Utc::now(),
5050 },
5051 0,
5052 );
5053 }
5054 }
5055
5056 // Validated
5057 {
5058 let mut log = self.log.lock().await;
5059 log.append(
5060 EventKind::ActionValidated,
5061 Some(&action.id),
5062 Some(proposal_id),
5063 HashMap::new(),
5064 );
5065 }
5066
5067 // Execute with retry
5068 let (result, retries) = self
5069 .execute_with_retry(action, proposal_id, session_id, scope)
5070 .await;
5071
5072 // Cache idempotent results. Detached invocations are exempt
5073 // (linus review D4): their Succeeded output is a live tool
5074 // HANDLE, exactly as uncacheable here as in the result cache —
5075 // deduping would return a stale handle instead of starting the
5076 // tool, and journaling would replay a handle id that doesn't
5077 // exist in a fresh registry after restart.
5078 if action.idempotent
5079 && !action.invocation_mode.is_detached()
5080 && result.status == ActionStatus::Succeeded
5081 {
5082 let key = idempotency_key(action, scope);
5083 {
5084 let mut cache = self.idempotency_cache.lock().await;
5085 cache.insert(key.clone(), result.clone());
5086 }
5087 // Persist to the durable journal (C3) so the result survives a
5088 // restart and the action isn't re-executed.
5089 self.journal_idempotency(&key, Some(&result)).await;
5090 }
5091
5092 tracing::info!(
5093 status = ?result.status,
5094 duration_ms = result.duration_ms,
5095 "action completed"
5096 );
5097
5098 (result, retries)
5099 }
5100
5101 /// Execute with retry logic and timeout.
5102 /// Returns (ActionResult, retries_count).
5103 async fn execute_with_retry(
5104 &self,
5105 action: &Action,
5106 proposal_id: &str,
5107 session_id: Option<&str>,
5108 scope: Option<&crate::scope::RuntimeScope>,
5109 ) -> (ActionResult, u32) {
5110 // The harness config (Evolution Agent, §3.5), when installed, caps
5111 // the per-action retry budget and sets the inter-attempt backoff —
5112 // this is where an applied retry-config mutation actually takes
5113 // effect. `None` (the default) preserves prior behavior exactly.
5114 let hc = self.harness_config.read().await.clone();
5115 let retry_cap = hc.as_ref().map(|c| c.max_retries).unwrap_or(u32::MAX);
5116 let backoff_override = hc.as_ref().map(|c| c.retry_backoff_ms).filter(|&v| v > 0);
5117 let max_attempts = if action.failure_behavior == FailureBehavior::Retry {
5118 match action.max_retries.min(retry_cap).checked_add(1) {
5119 Some(max_attempts) => max_attempts,
5120 None => {
5121 let reason =
5122 format!("action '{}' retry attempt count overflows u32", action.id);
5123 self.log.lock().await.append(
5124 EventKind::ActionRejected,
5125 Some(&action.id),
5126 Some(proposal_id),
5127 action_outcome_data("retry_admission", &reason, false, None),
5128 );
5129 return (rejected_result(&action.id, reason), 0);
5130 }
5131 }
5132 } else {
5133 1
5134 };
5135
5136 let mut last_error: Option<String> = None;
5137 let mut retries: u32 = 0;
5138 let tool_source = self.tool_source_for_action(action).await;
5139 let params_digest = action_params_digest(action);
5140
5141 for attempt in 0..max_attempts {
5142 if attempt > 0 {
5143 retries += 1;
5144 // Base delay from the harness config when installed (an
5145 // applied retry-config mutation), else the built-in default.
5146 let base = backoff_override.unwrap_or(RETRY_BASE_DELAY_MS);
5147 let delay = base * RETRY_BACKOFF_FACTOR.pow(attempt - 1);
5148 tokio::time::sleep(Duration::from_millis(delay)).await;
5149 let mut log = self.log.lock().await;
5150 log.append(
5151 EventKind::ActionRetrying,
5152 Some(&action.id),
5153 Some(proposal_id),
5154 [("attempt".to_string(), Value::from(attempt + 1))].into(),
5155 );
5156 }
5157
5158 {
5159 let mut data: HashMap<String, Value> =
5160 [("attempt".to_string(), Value::from(attempt + 1))].into();
5161 insert_tool_event_provenance(&mut data, action, tool_source);
5162 let mut log = self.log.lock().await;
5163 log.append(
5164 EventKind::ActionExecuting,
5165 Some(&action.id),
5166 Some(proposal_id),
5167 data,
5168 );
5169 }
5170
5171 let start = std::time::Instant::now();
5172 let transitions_before = self.state.transition_count();
5173
5174 // Execute with optional timeout. Keep the timeout bit typed until
5175 // event classification rather than trying to recover it from the
5176 // human-readable error later.
5177 let (exec_result, engine_timeout) = if let Some(timeout_ms) = action.timeout_ms {
5178 match timeout(
5179 Duration::from_millis(timeout_ms),
5180 self.dispatch(action, session_id, scope, attempt + 1),
5181 )
5182 .await
5183 {
5184 Ok(result) => (result, false),
5185 Err(_) => (
5186 Err(ToolFailure::ordinary(format!(
5187 "action timed out after {}ms",
5188 timeout_ms
5189 ))),
5190 true,
5191 ),
5192 }
5193 } else {
5194 (
5195 self.dispatch(action, session_id, scope, attempt + 1).await,
5196 false,
5197 )
5198 };
5199
5200 let duration_ms = start.elapsed().as_secs_f64() * 1000.0;
5201
5202 match exec_result {
5203 Ok(output) => {
5204 if let Err(error) = car_inference::catalog_identity::canonical_json(&output) {
5205 let reason = format!(
5206 "tool output failed JCS/I-JSON validation after dispatch; external effect may have occurred and was not undone: {error}"
5207 );
5208 let mut data = action_outcome_data(
5209 "output_validation",
5210 &reason,
5211 true,
5212 Some(attempt + 1),
5213 );
5214 data.insert(
5215 "external_effect_status".to_string(),
5216 Value::from("may_have_occurred_not_undone"),
5217 );
5218 insert_tool_event_provenance(&mut data, action, tool_source);
5219 insert_action_outcome_signal(
5220 &mut data,
5221 action,
5222 ¶ms_digest,
5223 Some("validation"),
5224 );
5225 if action.tool.is_some() {
5226 data.insert("ok".to_string(), Value::from(false));
5227 }
5228 self.log.lock().await.append(
5229 EventKind::ActionFailed,
5230 Some(&action.id),
5231 Some(proposal_id),
5232 data,
5233 );
5234 return (
5235 ActionResult {
5236 action_id: action.id.clone(),
5237 status: ActionStatus::Failed,
5238 output: None,
5239 error: Some(reason),
5240 terminal: false,
5241 state_changes: HashMap::new(),
5242 rolled_back: false,
5243 duration_ms: Some(duration_ms),
5244 timestamp: chrono::Utc::now(),
5245 },
5246 retries,
5247 );
5248 }
5249
5250 // Snapshot the changes made by dispatch before applying
5251 // any result accounting. Deletion and set-to-null are
5252 // intentionally tagged differently on the authenticated
5253 // wire; a bare JSON null cannot represent both.
5254 let runtime_state_changes: HashMap<String, Value> = self
5255 .state
5256 .transitions_since(transitions_before)
5257 .into_iter()
5258 // Actions at the same DAG level execute concurrently,
5259 // so the shared transition tail can contain a sibling's
5260 // writes. StateTransition.action_id is assigned at the
5261 // mutation site and is the authoritative attribution
5262 // boundary for this per-action journal field.
5263 .filter(|transition| transition.action_id == action.id)
5264 .map(|transition| {
5265 (
5266 transition.key,
5267 StateMutation::from_new_value(transition.new_value).encode(),
5268 )
5269 })
5270 .collect();
5271
5272 // Declared expected effects are plan assertions only.
5273 // They must never be applied as if the runtime observed
5274 // them, nor merged into committed result evidence.
5275 let tenant_for_effects = scope.and_then(|s| s.tenant_id.as_deref());
5276 let state_changes = runtime_state_changes.clone();
5277
5278 // Record this result's provenance for the VIGIL intent
5279 // gate (`crate::taint`) — the only point in the system
5280 // that knows which SPECIFIC results were tainted. Only
5281 // on success: a failed action commits no effects, so it
5282 // changes no taint. No intent gate ⇒ no ledger ⇒ one
5283 // `Option` read and nothing else.
5284 let ledger = self.taint_ledger.read().await.clone();
5285 if let Some(ledger) = ledger {
5286 ledger
5287 .record_result(
5288 tenant_for_effects,
5289 action,
5290 state_changes.keys().cloned(),
5291 )
5292 .await;
5293 }
5294
5295 let mut log = self.log.lock().await;
5296 // Record the tool name + result count so the tool-receipt
5297 // verifier (A6) can project ground-truth receipts from the
5298 // log and catch tool-use hallucinations.
5299 let mut succ_data: HashMap<String, Value> = HashMap::new();
5300 succ_data.insert("attempt".to_string(), Value::from(attempt + 1));
5301 succ_data.insert("attempted".to_string(), Value::from(true));
5302 succ_data.insert("stage".to_string(), Value::from("dispatch"));
5303 insert_tool_event_provenance(&mut succ_data, action, tool_source);
5304 insert_action_outcome_signal(&mut succ_data, action, ¶ms_digest, None);
5305 if action.tool.is_some() {
5306 succ_data.insert("ok".to_string(), Value::from(true));
5307 if let Value::Array(arr) = &output {
5308 succ_data
5309 .insert("result_count".to_string(), Value::from(arr.len() as u64));
5310 }
5311 }
5312 // Record duration through the standardized metric path
5313 // (§3.5.1) so it feeds `EventLog::metrics_totals` via the
5314 // same contract as inference token metrics — one path,
5315 // not a coincidentally-matching `"duration_ms"` string.
5316 log.append_metered(
5317 EventKind::ActionSucceeded,
5318 Some(&action.id),
5319 Some(proposal_id),
5320 succ_data,
5321 car_eventlog::Metrics::latency(duration_ms),
5322 );
5323
5324 if !action.expected_effects.is_empty() || !runtime_state_changes.is_empty() {
5325 log.append(
5326 EventKind::StateChanged,
5327 Some(&action.id),
5328 Some(proposal_id),
5329 [
5330 (
5331 "declared_expected_effects".to_string(),
5332 serde_json::to_value(&action.expected_effects)
5333 .unwrap_or_default(),
5334 ),
5335 (
5336 "runtime_state_mutations".to_string(),
5337 serde_json::to_value(&runtime_state_changes)
5338 .unwrap_or_default(),
5339 ),
5340 (
5341 "changes".to_string(),
5342 serde_json::to_value(&state_changes).unwrap_or_default(),
5343 ),
5344 (
5345 "changes_semantics".to_string(),
5346 Value::from("provisional_state_mutations"),
5347 ),
5348 ("attempt".to_string(), Value::from(attempt + 1)),
5349 ("attempted".to_string(), Value::from(true)),
5350 ("stage".to_string(), Value::from("state_effect")),
5351 ]
5352 .into(),
5353 );
5354 }
5355
5356 return (
5357 ActionResult {
5358 action_id: action.id.clone(),
5359 status: ActionStatus::Succeeded,
5360 output: Some(output),
5361 error: None,
5362 terminal: false,
5363 state_changes,
5364 rolled_back: false,
5365 duration_ms: Some(duration_ms),
5366 timestamp: chrono::Utc::now(),
5367 },
5368 retries,
5369 );
5370 }
5371 Err(failure) => {
5372 let terminal = match failure.classification {
5373 ToolFailureClassification::Ordinary => false,
5374 ToolFailureClassification::Terminal => true,
5375 };
5376 let error = failure.message;
5377 last_error = Some(error.clone());
5378 let mut log = self.log.lock().await;
5379 let mut fail_data: HashMap<String, Value> = [
5380 ("error".to_string(), Value::from(error.as_str())),
5381 ("attempt".to_string(), Value::from(attempt + 1)),
5382 ("attempted".to_string(), Value::from(true)),
5383 ("stage".to_string(), Value::from("dispatch")),
5384 ]
5385 .into();
5386 // Tool name + ok=false so the receipt verifier (A6) records
5387 // that the tool *executed* (a failed call still ran).
5388 insert_tool_event_provenance(&mut fail_data, action, tool_source);
5389 insert_action_outcome_signal(
5390 &mut fail_data,
5391 action,
5392 ¶ms_digest,
5393 Some(action_error_class(action, &error, engine_timeout)),
5394 );
5395 if action.tool.is_some() {
5396 fail_data.insert("ok".to_string(), Value::from(false));
5397 }
5398 if terminal {
5399 fail_data.insert("terminal".to_string(), Value::from(true));
5400 }
5401 log.append(
5402 EventKind::ActionFailed,
5403 Some(&action.id),
5404 Some(proposal_id),
5405 fail_data,
5406 );
5407 drop(log);
5408
5409 if terminal {
5410 return (
5411 ActionResult {
5412 action_id: action.id.clone(),
5413 status: ActionStatus::Failed,
5414 output: None,
5415 error: Some(error),
5416 terminal: true,
5417 rolled_back: false,
5418 state_changes: HashMap::new(),
5419 duration_ms: Some(duration_ms),
5420 timestamp: chrono::Utc::now(),
5421 },
5422 retries,
5423 );
5424 }
5425 }
5426 }
5427 }
5428
5429 // All attempts exhausted
5430 if action.failure_behavior == FailureBehavior::Skip {
5431 let reason = last_error.as_deref().unwrap_or("all attempts exhausted");
5432 self.log.lock().await.append(
5433 EventKind::ActionSkipped,
5434 Some(&action.id),
5435 Some(proposal_id),
5436 action_outcome_data("failure_behavior", reason, true, Some(max_attempts)),
5437 );
5438 return (skipped_result(&action.id, reason), retries);
5439 }
5440
5441 (
5442 ActionResult {
5443 action_id: action.id.clone(),
5444 status: ActionStatus::Failed,
5445 output: None,
5446 error: last_error,
5447 terminal: false,
5448 state_changes: HashMap::new(),
5449 rolled_back: false,
5450 duration_ms: None,
5451 timestamp: chrono::Utc::now(),
5452 },
5453 retries,
5454 )
5455 }
5456
5457 /// Resolve the stable source category for an action's tool. Canonical
5458 /// ToolEntry provenance wins; the ToolSchema fallback keeps legacy direct
5459 /// registrations observable, and the final default covers old callers that
5460 /// supplied a tool field on a non-tool action.
5461 async fn tool_source_for_action(&self, action: &Action) -> Option<car_ir::ToolSourceKind> {
5462 let tool = action.tool.as_deref()?;
5463 if let Some(entry) = self.registry.get(tool).await {
5464 return Some(entry.source.kind());
5465 }
5466 Some(
5467 self.tools
5468 .read()
5469 .await
5470 .get(tool)
5471 .map(|schema| schema.source)
5472 .unwrap_or(car_ir::ToolSourceKind::UserDefined),
5473 )
5474 }
5475
5476 /// Dispatch an action to the appropriate handler.
5477 ///
5478 /// `scope` is the per-execution caller / tenant surface
5479 /// (Parslee-ai/car#187 phase 3). When the scope carries a
5480 /// tenant id, state R/W operations (`StateWrite`, `StateRead`,
5481 /// `Assertion`) route through `StateStore::scoped(tenant_id)`
5482 /// so distinct tenants can't see each other's keys. Unscoped
5483 /// proposals get the legacy flat-namespace behaviour
5484 /// automatically.
5485 async fn dispatch(
5486 &self,
5487 action: &Action,
5488 session_id: Option<&str>,
5489 scope: Option<&crate::scope::RuntimeScope>,
5490 attempt: u32,
5491 ) -> Result<Value, ToolFailure> {
5492 match action.action_type {
5493 ActionType::ToolCall => {
5494 let tool_name = action.tool.as_deref().ok_or("tool_call has no tool")?;
5495 let params = Value::Object(
5496 action
5497 .parameters
5498 .iter()
5499 .map(|(k, v)| (k.clone(), v.clone()))
5500 .collect(),
5501 );
5502
5503 // Detached invocation modes (C2): start the tool via the
5504 // configured executor's streaming entry point, register a
5505 // handle, and return immediately — the DAG must not block
5506 // on a streaming/long-running tool. Deliberately BEFORE
5507 // the result cache (a handle is a live invocation, never a
5508 // cacheable value) but still behind the rate limiter.
5509 // Built-ins don't stream; a detached call requires a
5510 // configured executor that implements execute_stream.
5511 if action.invocation_mode.is_detached() {
5512 self.rate_limiter.acquire(tool_name).await;
5513 let configured = {
5514 let guard = self.tool_executor.lock().await;
5515 guard.as_ref().cloned()
5516 };
5517 let executor = configured.ok_or_else(|| {
5518 format!("tool '{tool_name}': detached invocation requires a tool executor")
5519 })?;
5520 let rx = executor
5521 .execute_stream(tool_name, ¶ms, &action.id)
5522 .await?;
5523 let (handle, cancel) = self.tool_handles.register(tool_name, &action.id).await;
5524 crate::tool_handles::spawn_drain(
5525 self.tool_handles.clone(),
5526 handle.id.clone(),
5527 rx,
5528 cancel,
5529 );
5530 return Ok(serde_json::json!({
5531 "tool_handle": handle.id,
5532 "status": "running",
5533 }));
5534 }
5535
5536 // Check cross-proposal result cache.
5537 if let Some(cached) = self.result_cache.get(tool_name, ¶ms).await {
5538 return Ok(cached);
5539 }
5540
5541 // Apply rate limit backpressure before executing.
5542 self.rate_limiter.acquire(tool_name).await;
5543
5544 // Try built-in inference tools first when the inference engine is available.
5545 if matches!(
5546 tool_name,
5547 "infer" | "infer.grounded" | "embed" | "classify" | "transcribe" | "synthesize"
5548 ) {
5549 if let Some(ref engine) = self.inference_engine {
5550 // For "infer.grounded" or "infer" with memgine available,
5551 // build context from memory and attach it to the request.
5552 let params = {
5553 let should_ground =
5554 tool_name == "infer.grounded" || tool_name == "infer";
5555 if should_ground {
5556 if let Some(ref memgine) = self.memgine {
5557 if let Some(prompt) =
5558 params.get("prompt").and_then(|v| v.as_str())
5559 {
5560 let ctx = {
5561 let mut m = memgine.lock().await;
5562 m.build_context(prompt)
5563 };
5564 if !ctx.is_empty() {
5565 let mut p = params.clone();
5566 if let Some(obj) = p.as_object_mut() {
5567 obj.insert("context".to_string(), Value::from(ctx));
5568 }
5569 p
5570 } else {
5571 params
5572 }
5573 } else {
5574 params
5575 }
5576 } else {
5577 params
5578 }
5579 } else {
5580 params
5581 }
5582 };
5583
5584 // Route "infer.grounded" to "infer" for the service layer
5585 let effective_tool = if tool_name == "infer.grounded" {
5586 "infer"
5587 } else {
5588 tool_name
5589 };
5590 let result =
5591 car_inference::service::execute_tool(engine, effective_tool, ¶ms)
5592 .await
5593 .map_err(|e| e.to_string());
5594
5595 if let Ok(ref value) = result {
5596 self.result_cache
5597 .put(tool_name, ¶ms, value.clone())
5598 .await;
5599 }
5600
5601 return result.map_err(ToolFailure::from);
5602 }
5603 }
5604
5605 // Built-in memory consolidation tool.
5606 if tool_name == "memory.consolidate" {
5607 if let Some(ref memgine) = self.memgine {
5608 let report = {
5609 let mut m = memgine.lock().await;
5610 m.consolidate().await
5611 };
5612 // Log the consolidation event
5613 {
5614 let mut log = self.log.lock().await;
5615 log.append(
5616 EventKind::Consolidated,
5617 None,
5618 None,
5619 [
5620 (
5621 "expired_pruned".to_string(),
5622 Value::from(report.expired_pruned),
5623 ),
5624 (
5625 "superseded_gc".to_string(),
5626 Value::from(report.superseded_gc),
5627 ),
5628 (
5629 "stale_embeddings_removed".to_string(),
5630 Value::from(report.stale_embeddings_removed),
5631 ),
5632 (
5633 "nodes_embedded".to_string(),
5634 Value::from(report.nodes_embedded),
5635 ),
5636 (
5637 "domains_evolved".to_string(),
5638 Value::from(report.domains_evolved.clone()),
5639 ),
5640 ("total_nodes".to_string(), Value::from(report.total_nodes)),
5641 ("total_edges".to_string(), Value::from(report.total_edges)),
5642 ]
5643 .into(),
5644 );
5645 // Per-candidate gate telemetry (SkillOpt-inspired):
5646 // one event per promotion/rejection so the skill
5647 // bank's evolution is auditable.
5648 for key in &report.candidates_promoted {
5649 log.append(
5650 EventKind::CandidatePromoted,
5651 None,
5652 None,
5653 [("candidate".to_string(), Value::from(key.as_str()))].into(),
5654 );
5655 }
5656 for key in &report.candidates_rejected {
5657 log.append(
5658 EventKind::CandidateRejected,
5659 None,
5660 None,
5661 [("candidate".to_string(), Value::from(key.as_str()))].into(),
5662 );
5663 }
5664 }
5665 return Ok(serde_json::to_value(&report).unwrap_or(Value::Null));
5666 } else {
5667 return Err(
5668 "memory.consolidate requires memgine (attach with with_learning)"
5669 .into(),
5670 );
5671 }
5672 }
5673
5674 // Built-in outbound human messaging. Sits here, alongside the
5675 // other built-ins — downstream of `validate_action` and the
5676 // policy check in the caller, and downstream of the result
5677 // cache read and `rate_limiter.acquire` above — so a message
5678 // to a human traverses exactly the same governed chain
5679 // (validator → policy → rate limit → eventlog) as any other
5680 // side effect, instead of the hand-rolled transport each agent
5681 // used to carry.
5682 if tool_name == "messaging.send" {
5683 // No fall-through to the host executor when unconfigured.
5684 // Falling through would let an ungoverned host transport
5685 // answer the call — the exact path this built-in exists to
5686 // close — so an unconfigured runtime refuses instead.
5687 let Some(ref sink) = self.message_sink else {
5688 return Err("messaging.send: messaging is not configured on this \
5689 runtime — attach a message sink via \
5690 Runtime::with_message_sink"
5691 .into());
5692 };
5693 let msg = crate::messaging::OutboundMessage::from_tool_params(¶ms)?;
5694 let receipt = sink.send(&msg).await?;
5695 // Not cached: the schema declares no cache TTL, and a
5696 // cached send would swallow a second, genuinely wanted
5697 // message with identical text.
5698 return serde_json::to_value(&receipt).map_err(|error| {
5699 ToolFailure::ordinary(format!("messaging.send: serialize receipt: {error}"))
5700 });
5701 }
5702
5703 // Prefer a configured tool_executor for any tool it claims to handle.
5704 // Fall through to agent_basics only when the configured executor is absent
5705 // or returns a failure typed `ToolFailure::is_unhandled` — this prevents
5706 // agent_basics' built-in read_file/write_file (which resolve paths via
5707 // std::env::current_dir) from silently overriding an executor that carries
5708 // its own working_dir.
5709 //
5710 // The decision reads the typed flag, never the message. A transport
5711 // that relays another process's error (the daemon's WS client, an MCP
5712 // server) returns `ToolFailure::ordinary`, which is never unhandled, so a
5713 // client answering "unknown tool: read_file" gets its own error back
5714 // instead of a host file read (Parslee-ai/car#3104). In-process
5715 // `String` executors keep the "unknown tool" sentinel through
5716 // `From<String>`.
5717 let configured = {
5718 let guard = self.tool_executor.lock().await;
5719 guard.as_ref().cloned()
5720 };
5721
5722 if let Some(ref executor) = configured {
5723 let return_schema = self
5724 .tools
5725 .read()
5726 .await
5727 .get(tool_name)
5728 .and_then(|schema| schema.returns.clone());
5729 let result = executor
5730 .execute_classified(
5731 tool_name,
5732 ¶ms,
5733 &action.id,
5734 action.timeout_ms,
5735 session_id,
5736 attempt,
5737 &action.expected_effects,
5738 return_schema.as_ref(),
5739 )
5740 .await;
5741 let fall_through = matches!(&result, Err(error) if error.is_unhandled());
5742 if !fall_through {
5743 match result {
5744 Ok(execution) => {
5745 if !execution.state_changes.is_empty() {
5746 let expected_keys: std::collections::BTreeSet<&str> = action
5747 .expected_effects
5748 .keys()
5749 .map(String::as_str)
5750 .collect();
5751 let actual_keys: std::collections::BTreeSet<&str> = execution
5752 .state_changes
5753 .keys()
5754 .map(String::as_str)
5755 .collect();
5756 if expected_keys != actual_keys {
5757 return Err(format!(
5758 "tool '{}' callback state_changes keys do not match action '{}': expected {:?}, got {:?}",
5759 tool_name, action.id, expected_keys, actual_keys
5760 )
5761 .into());
5762 }
5763 car_inference::catalog_identity::canonical_json(&execution.output)
5764 .map_err(|error| {
5765 format!(
5766 "tool output failed JCS/I-JSON validation after dispatch; external effect may have occurred and was not undone: {error}"
5767 )
5768 })?;
5769 let changes_value = serde_json::to_value(&execution.state_changes)
5770 .map_err(|error| {
5771 format!(
5772 "tool '{}' callback state_changes serialization failed: {error}",
5773 tool_name
5774 )
5775 })?;
5776 car_inference::catalog_identity::canonical_json(&changes_value)
5777 .map_err(|error| {
5778 format!(
5779 "tool '{}' callback state_changes failed JCS/I-JSON validation: {error}",
5780 tool_name
5781 )
5782 })?;
5783
5784 // Every rejection above happens before the
5785 // first write. Apply in deterministic key order,
5786 // attributed to the active action, through ONE
5787 // batch mutation boundary: a concurrent reader,
5788 // journal replay, and restart recovery observe
5789 // the complete old state or the complete new
5790 // state — never a prefix of this callback's
5791 // keys (Parslee-ai/car#1140).
5792 let mut changes: Vec<(String, Value)> = execution
5793 .state_changes
5794 .iter()
5795 .map(|(key, value)| (key.clone(), value.clone()))
5796 .collect();
5797 changes.sort_by(|(left, _), (right, _)| left.cmp(right));
5798 let tenant = scope.and_then(|scope| scope.tenant_id.as_deref());
5799 self.state.scoped(tenant).set_batch(changes, &action.id);
5800 }
5801 self.result_cache
5802 .put(tool_name, ¶ms, execution.output.clone())
5803 .await;
5804 return Ok(execution.output);
5805 }
5806 Err(error) => return Err(error),
5807 }
5808 }
5809 }
5810
5811 // Built-in commodity tools resolve against the bound substrate
5812 // (default: LocalSubstrate → historic host behavior). `calculate`
5813 // stays pure inside agent_basics and ignores the substrate.
5814 let substrate = self.substrate.read().await.clone();
5815 if substrate.is_local() {
5816 if let Some(root) = self.basics_root.read().await.clone() {
5817 if let Some(refusal) =
5818 crate::agent_basics::root_refusal(&root, tool_name, ¶ms)
5819 {
5820 return Err(ToolFailure::ordinary(refusal));
5821 }
5822 }
5823 }
5824 let read_ledger = self.read_ledgers.ledger_for(session_id);
5825 let guard = self.local_path_guard.read().await.clone();
5826 let builtin = match &guard {
5827 Some(guard) => {
5828 crate::agent_basics::execute_with_ledger_guarded(
5829 &substrate,
5830 &read_ledger,
5831 tool_name,
5832 ¶ms,
5833 guard,
5834 )
5835 .await
5836 }
5837 None => {
5838 crate::agent_basics::execute_with_ledger(
5839 &substrate,
5840 &read_ledger,
5841 tool_name,
5842 ¶ms,
5843 )
5844 .await
5845 }
5846 };
5847 if let Some(result) = builtin {
5848 if let Ok(ref value) = result {
5849 self.result_cache
5850 .put(tool_name, ¶ms, value.clone())
5851 .await;
5852 }
5853 return result.map_err(ToolFailure::from);
5854 }
5855
5856 Err(format!("no handler for tool '{}'", tool_name).into())
5857 }
5858 ActionType::StateWrite => {
5859 let key = action
5860 .parameters
5861 .get("key")
5862 .and_then(|v| v.as_str())
5863 .ok_or("state_write requires 'key' parameter")?;
5864 let value = action
5865 .parameters
5866 .get("value")
5867 .cloned()
5868 .unwrap_or(Value::Null);
5869 let tenant = scope.and_then(|s| s.tenant_id.as_deref());
5870 self.state.scoped(tenant).set(key, value, &action.id);
5871 Ok(Value::from(format!("written: {}", key)))
5872 }
5873 ActionType::StateRead => {
5874 let key = action
5875 .parameters
5876 .get("key")
5877 .and_then(|v| v.as_str())
5878 .ok_or("state_read requires 'key' parameter")?;
5879 let tenant = scope.and_then(|s| s.tenant_id.as_deref());
5880 Ok(self.state.scoped(tenant).get(key).unwrap_or(Value::Null))
5881 }
5882 ActionType::Assertion => {
5883 let key = action
5884 .parameters
5885 .get("key")
5886 .and_then(|v| v.as_str())
5887 .ok_or("assertion requires 'key' parameter")?;
5888 let expected = action
5889 .parameters
5890 .get("expected")
5891 .cloned()
5892 .unwrap_or(Value::Null);
5893 let tenant = scope.and_then(|s| s.tenant_id.as_deref());
5894 let actual = self.state.scoped(tenant).get(key).unwrap_or(Value::Null);
5895 if actual != expected {
5896 Err(format!(
5897 "assertion failed: state['{}'] = {:?}, expected {:?}",
5898 key, actual, expected
5899 )
5900 .into())
5901 } else {
5902 Ok(serde_json::json!({"asserted": key, "value": actual}))
5903 }
5904 }
5905 }
5906 }
5907
5908 // --- Checkpoint and resume ---
5909
5910 /// Save a checkpoint of the current runtime state.
5911 pub async fn save_checkpoint(&self) -> Checkpoint {
5912 let state = self.state.snapshot();
5913 let tools: Vec<String> = self.tools.read().await.keys().cloned().collect();
5914 let log = self.log.lock().await;
5915 let events: Vec<Value> = log
5916 .events()
5917 .iter()
5918 .map(|e| serde_json::to_value(e).unwrap_or_default())
5919 .collect();
5920
5921 Checkpoint {
5922 checkpoint_id: Uuid::new_v4().to_string(),
5923 created_at: chrono::Utc::now(),
5924 state,
5925 events,
5926 tools,
5927 metadata: HashMap::new(),
5928 }
5929 }
5930
5931 /// Save checkpoint to a JSON file.
5932 pub async fn save_checkpoint_to_file(&self, path: &str) -> Result<(), String> {
5933 let checkpoint = self.save_checkpoint().await;
5934 let json = serde_json::to_string_pretty(&checkpoint)
5935 .map_err(|e| format!("serialize error: {}", e))?;
5936 tokio::fs::write(path, json)
5937 .await
5938 .map_err(|e| format!("write error: {}", e))?;
5939 Ok(())
5940 }
5941
5942 /// Load a checkpoint from a JSON file and restore state.
5943 pub async fn load_checkpoint_from_file(&self, path: &str) -> Result<Checkpoint, String> {
5944 let json = tokio::fs::read_to_string(path)
5945 .await
5946 .map_err(|e| format!("read error: {}", e))?;
5947 let checkpoint: Checkpoint =
5948 serde_json::from_str(&json).map_err(|e| format!("deserialize error: {}", e))?;
5949 self.restore_checkpoint(&checkpoint).await;
5950 Ok(checkpoint)
5951 }
5952
5953 /// Restore runtime state from a checkpoint.
5954 pub async fn restore_checkpoint(&self, checkpoint: &Checkpoint) {
5955 // Replace state completely — don't merge, don't create synthetic transitions
5956 self.state.replace_all(checkpoint.state.clone());
5957 // Clear idempotency cache — stale results from pre-checkpoint execution
5958 // must not bypass validation/policy on the restored state
5959 self.idempotency_cache.lock().await.clear();
5960 // Restore tools (as name-only schemas; full schemas are not persisted in checkpoint)
5961 let mut tools = self.tools.write().await;
5962 tools.clear();
5963 for tool_name in &checkpoint.tools {
5964 let schema = ToolSchema {
5965 name: tool_name.clone(),
5966 source: car_ir::ToolSourceKind::UserDefined,
5967 description: String::new(),
5968 parameters: serde_json::Value::Object(Default::default()),
5969 returns: None,
5970 idempotent: false,
5971 cache_ttl_secs: None,
5972 rate_limit: None,
5973 };
5974 tools.insert(tool_name.clone(), schema);
5975 }
5976 }
5977
5978 /// Register a subprocess tool and set up the subprocess executor.
5979 /// If no executor exists, creates a new SubprocessToolExecutor.
5980 /// If one already exists, creates a new SubprocessToolExecutor with the
5981 /// existing executor as fallback.
5982 pub async fn register_subprocess_tool(
5983 &self,
5984 name: &str,
5985 tool: crate::subprocess::SubprocessTool,
5986 ) {
5987 use crate::subprocess::SubprocessToolExecutor;
5988
5989 let schema = ToolSchema {
5990 name: name.to_string(),
5991 source: car_ir::ToolSourceKind::Subprocess,
5992 description: format!("Subprocess tool: {}", tool.command),
5993 parameters: serde_json::Value::Object(Default::default()),
5994 returns: None,
5995 idempotent: false,
5996 cache_ttl_secs: None,
5997 rate_limit: None,
5998 };
5999 self.register_tool_entry(
6000 crate::registry::ToolEntry::new(schema)
6001 .with_source(crate::registry::ToolSource::Subprocess),
6002 )
6003 .await;
6004
6005 let mut guard = self.tool_executor.lock().await;
6006 let mut executor = match guard.take() {
6007 Some(existing) => {
6008 let mut sub = SubprocessToolExecutor::new();
6009 sub = sub.with_fallback(existing);
6010 sub
6011 }
6012 None => SubprocessToolExecutor::new(),
6013 };
6014 executor.register(name, tool);
6015 *guard = Some(std::sync::Arc::new(executor));
6016 }
6017}
6018
6019impl Default for Runtime {
6020 fn default() -> Self {
6021 Self::new()
6022 }
6023}
6024
6025#[cfg(test)]
6026mod timeout_integration_tests {
6027 //! Integration coverage for the #259/#262 timeout coordination the unit
6028 //! tests in `car-server-core` / `car-ffi-common` only exercise as pure
6029 //! selection helpers (#266 item 4): that the executor's per-action deadline
6030 //! reaps a slow dispatch first, and that `action.timeout_ms` flows
6031 //! end-to-end onto `execute_with_action`.
6032 use super::*;
6033 use car_ir::ActionProposal;
6034 use std::sync::atomic::{AtomicU64, Ordering};
6035
6036 /// A tool executor that (a) records the `timeout_ms` it was handed and
6037 /// (b) sleeps `delay_ms` before returning, so a test can prove the
6038 /// executor's own `timeout(action.timeout_ms, dispatch)` reaps a call that
6039 /// outlives its budget.
6040 struct RecordingExecutor {
6041 seen_timeout_ms: Arc<AtomicU64>,
6042 delay_ms: u64,
6043 }
6044
6045 #[async_trait::async_trait]
6046 impl ToolExecutor for RecordingExecutor {
6047 async fn execute(&self, _tool: &str, _params: &Value) -> Result<Value, String> {
6048 Ok(Value::Null)
6049 }
6050 async fn execute_with_action(
6051 &self,
6052 _tool: &str,
6053 _params: &Value,
6054 _action_id: &str,
6055 timeout_ms: Option<u64>,
6056 ) -> Result<Value, String> {
6057 // `u64::MAX` sentinel = "None was passed".
6058 self.seen_timeout_ms
6059 .store(timeout_ms.unwrap_or(u64::MAX), Ordering::SeqCst);
6060 tokio::time::sleep(Duration::from_millis(self.delay_ms)).await;
6061 Ok(serde_json::json!({ "ok": true }))
6062 }
6063 }
6064
6065 fn one_tool_proposal(timeout_ms: Option<u64>) -> ActionProposal {
6066 let mut action = serde_json::json!({
6067 "id": "a0",
6068 "type": "tool_call",
6069 "tool": "slow",
6070 "parameters": {},
6071 "dependencies": [],
6072 });
6073 if let Some(ms) = timeout_ms {
6074 action["timeout_ms"] = serde_json::json!(ms);
6075 }
6076 serde_json::from_value(serde_json::json!({
6077 "source": "test",
6078 "actions": [action],
6079 }))
6080 .expect("proposal deserializes")
6081 }
6082
6083 #[tokio::test(start_paused = true)]
6084 async fn action_timeout_ms_reaches_executor_and_reaps_slow_dispatch() {
6085 let seen = Arc::new(AtomicU64::new(0));
6086 let rt = Runtime::new();
6087 rt.register_tool("slow").await;
6088 rt.set_executor(Arc::new(RecordingExecutor {
6089 seen_timeout_ms: seen.clone(),
6090 delay_ms: 2_000, // outlives the 100ms budget below
6091 }))
6092 .await;
6093
6094 // 100ms budget against a 2s dispatch: the executor's own
6095 // `timeout(action.timeout_ms, dispatch)` must reap it (#262 — the
6096 // executor is the authority), and the budget must have reached
6097 // `execute_with_action` (the #262 harness→action propagation, proven
6098 // end-to-end here rather than only via the pure helper).
6099 let result = rt.execute(&one_tool_proposal(Some(100))).await;
6100 assert_eq!(
6101 seen.load(Ordering::SeqCst),
6102 100,
6103 "budget must reach the executor"
6104 );
6105 let action_result = &result.results[0];
6106 assert!(
6107 action_result
6108 .error
6109 .as_deref()
6110 .unwrap_or("")
6111 .contains("timed out"),
6112 "slow dispatch must be reaped by the action deadline: {:?}",
6113 action_result.error
6114 );
6115 }
6116
6117 /// An executor that records every `attempt` it is handed and fails until
6118 /// the Nth, so a test can watch the counter advance across real retries.
6119 struct AttemptRecordingExecutor {
6120 seen: Arc<tokio::sync::Mutex<Vec<u32>>>,
6121 succeed_on: u32,
6122 }
6123
6124 #[async_trait::async_trait]
6125 impl ToolExecutor for AttemptRecordingExecutor {
6126 async fn execute(&self, _tool: &str, _params: &Value) -> Result<Value, String> {
6127 Ok(Value::Null)
6128 }
6129 async fn execute_with_action_in_session(
6130 &self,
6131 _tool: &str,
6132 _params: &Value,
6133 _action_id: &str,
6134 _timeout_ms: Option<u64>,
6135 _session_id: Option<&str>,
6136 attempt: u32,
6137 ) -> Result<Value, String> {
6138 self.seen.lock().await.push(attempt);
6139 if attempt >= self.succeed_on {
6140 Ok(serde_json::json!({ "ok": true }))
6141 } else {
6142 Err("transient".to_string())
6143 }
6144 }
6145 }
6146
6147 /// Parslee-ai/car#928 — the retry counter the executor receives must be the
6148 /// engine's real one, and it must advance.
6149 ///
6150 /// The WS executor hardcoded `attempt: 1` onto the `tools.execute` payload,
6151 /// so the field a host would build a retry-disambiguating join on never
6152 /// varied. Asserting on the sequence the executor actually observes is the
6153 /// property that matters: a frame test can only prove the payload carries
6154 /// whatever it was handed, not that the engine hands it the truth.
6155 #[tokio::test]
6156 async fn the_executor_sees_the_engines_real_attempt_sequence() {
6157 let seen = Arc::new(tokio::sync::Mutex::new(Vec::new()));
6158 let rt = Runtime::new();
6159 rt.register_tool("flaky").await;
6160 rt.set_executor(Arc::new(AttemptRecordingExecutor {
6161 seen: seen.clone(),
6162 succeed_on: 3,
6163 }))
6164 .await;
6165
6166 let proposal: ActionProposal = serde_json::from_value(serde_json::json!({
6167 "source": "test",
6168 "actions": [{
6169 "id": "a0",
6170 "type": "tool_call",
6171 "tool": "flaky",
6172 "parameters": {},
6173 "dependencies": [],
6174 "failure_behavior": "retry",
6175 "max_retries": 3,
6176 }],
6177 }))
6178 .expect("proposal deserializes");
6179
6180 let result = rt.execute(&proposal).await;
6181 assert_eq!(
6182 result.results[0].status,
6183 ActionStatus::Succeeded,
6184 "third attempt succeeds: {:?}",
6185 result.results[0].error
6186 );
6187 assert_eq!(
6188 *seen.lock().await,
6189 vec![1, 2, 3],
6190 "1-based and advancing — a constant here is the #928 bug"
6191 );
6192 }
6193
6194 #[tokio::test(start_paused = true)]
6195 async fn no_budget_applies_no_executor_deadline() {
6196 // The `None` path: the executor applies NO deadline, so a dispatch that
6197 // outlives any per-attempt budget still completes (the WS callback wait
6198 // is the sole bound on that path, exercised in car-server-core). Tokio's
6199 // paused clock advances the 300ms dispatch without a wall-clock wait.
6200 let seen = Arc::new(AtomicU64::new(0));
6201 let rt = Runtime::new();
6202 rt.register_tool("slow").await;
6203 rt.set_executor(Arc::new(RecordingExecutor {
6204 seen_timeout_ms: seen.clone(),
6205 delay_ms: 300,
6206 }))
6207 .await;
6208
6209 let result = rt.execute(&one_tool_proposal(None)).await;
6210 assert_eq!(
6211 seen.load(Ordering::SeqCst),
6212 u64::MAX,
6213 "None budget must be forwarded as None"
6214 );
6215 assert_eq!(result.results[0].status, ActionStatus::Succeeded);
6216 }
6217}
6218
6219#[cfg(test)]
6220mod model_facing_result_tests {
6221 use super::*;
6222
6223 #[test]
6224 fn a_model_reads_a_successful_output_as_text_and_a_failure_with_its_prefix() {
6225 let ok = ActionResult {
6226 action_id: "a".into(),
6227 status: ActionStatus::Succeeded,
6228 output: Some(serde_json::json!({ "path": "a.py", "content": "x = \"1\"\ny = 2" })),
6229 error: None,
6230 terminal: false,
6231 state_changes: HashMap::new(),
6232 rolled_back: false,
6233 duration_ms: None,
6234 timestamp: chrono::Utc::now(),
6235 };
6236 assert_eq!(
6237 format_tool_result_for_model(&ok),
6238 "path: a.py\ncontent (13 bytes):\nx = \"1\"\ny = 2"
6239 );
6240 // The JSON form stays what it was for every other consumer.
6241 assert!(format_tool_result(&ok).starts_with('{'));
6242 let failed = ActionResult {
6243 status: ActionStatus::Failed,
6244 output: None,
6245 error: Some("boom".into()),
6246 ..ok
6247 };
6248 assert_eq!(format_tool_result_for_model(&failed), "[FAILED] boom");
6249 }
6250
6251 #[test]
6252 fn rendered_output_reads_back_exactly() {
6253 // Shapes tools actually produce, including content that looks like
6254 // fields and strings that look like other JSON types.
6255 for original in [
6256 serde_json::json!({
6257 "path": "config/locales/en.yml",
6258 "content": "en:\n x: 1\nerror:\n not_found: Missing\n",
6259 "total_lines": 4,
6260 }),
6261 serde_json::json!({
6262 "status": 200,
6263 "url": "https://example.com/a",
6264 "headers": {"content-type": "text/plain", "authorization": "Bearer abc"},
6265 "body": "hello\nstatus:\nall good\nzebra:\n",
6266 }),
6267 serde_json::json!({
6268 "exit_code": 1,
6269 "output": "make: ***\nzebra:\nx",
6270 "timed_out": false,
6271 }),
6272 serde_json::json!({
6273 "error": "null",
6274 "flag": "true",
6275 "id": "123456789012345678901234",
6276 "name": "123",
6277 "content-type": "text/plain",
6278 "empty": "",
6279 "padded": " x ",
6280 "quoted": "\"q\"",
6281 }),
6282 ] {
6283 let text = render_tool_output(&original);
6284 assert_eq!(
6285 parse_rendered_output(&text).as_ref(),
6286 Some(&original),
6287 "{text}"
6288 );
6289 }
6290 assert_eq!(parse_rendered_output("[FAILED] boom"), None);
6291 assert_eq!(parse_rendered_output("plain text"), None);
6292 // Runtime failure and refusal lines are not fields, even with `: ` in them.
6293 for line in [
6294 "[FAILED] tool 'read_file': no such file",
6295 "[REJECTED] policy 'deny_shell': shell denied",
6296 "[FAILED] canceled: user stop",
6297 "Tool call denied: not allowed\nsecond line without a key",
6298 ] {
6299 assert_eq!(parse_rendered_output(line), None, "{line}");
6300 }
6301 // A shell result cut in the middle by the budget keeps its verdict:
6302 // the stderr block claims bytes that now run through the marker.
6303 let shell = serde_json::json!({
6304 "exit_code": 1,
6305 "stderr": "warning: x\n".repeat(700),
6306 "stdout": "test a ... FAILED\n".repeat(2500),
6307 });
6308 let cut = truncate_keeping_ends(&render_tool_output(&shell), 16 * 1024);
6309 assert_eq!(parse_rendered_output(&cut).unwrap()["exit_code"], 1);
6310 // A block spliced into a cut tail cannot replace an outcome field.
6311 let spliced = "ok: false\nout (10 bytes):\nab\nok (4 bytes):\ntrue";
6312 assert_eq!(parse_rendered_output(spliced).unwrap()["ok"], false);
6313 // … and a cut mid-line after the fields keeps them.
6314 let cut = "exit_code: 1\nstdo\n…[truncated: 10 of 20 bytes]…";
6315 assert_eq!(parse_rendered_output(cut).unwrap()["exit_code"], 1);
6316 // Keys and values that look like the format's own syntax.
6317 let odd = serde_json::json!({
6318 "a:": "x\ny",
6319 "a: b": "c",
6320 "note": "wrote (3 bytes):",
6321 " padded": 1,
6322 "k (x": "y\nz",
6323 });
6324 let text = render_tool_output(&odd);
6325 assert_eq!(parse_rendered_output(&text).as_ref(), Some(&odd), "{text}");
6326 // A block cut by a budget reads back as what is left of it.
6327 let cut = "exit_code: 1\noutput (100 bytes):\nfirst lines only";
6328 assert_eq!(
6329 parse_rendered_output(cut).unwrap()["output"],
6330 "first lines only"
6331 );
6332 }
6333
6334 #[test]
6335 fn outcome_fields_lead_and_stay_on_one_line() {
6336 let out = render_tool_output(&serde_json::json!({
6337 "content": "x\ny",
6338 "error": format!("{}\nsecond line", "e".repeat(600)),
6339 "exit_code": 1,
6340 "matches": [{"path": "a"}],
6341 "truncated": true,
6342 }));
6343 let lines: Vec<&str> = out.lines().collect();
6344 assert!(lines[0].starts_with("error: "), "{out}");
6345 assert!(
6346 lines[0].len() <= 530 && lines[0].contains(" bytes]…"),
6347 "{out}"
6348 );
6349 assert!(
6350 lines[0].ends_with("second line"),
6351 "the end of a long error survives: {out}"
6352 );
6353 assert_eq!(lines[1], "exit_code: 1");
6354 assert_eq!(lines[2], "truncated: true");
6355 assert!(
6356 out.ends_with("content (3 bytes):\nx\ny\nmatches (1 records):\npath=a"),
6357 "{out}"
6358 );
6359 }
6360
6361 #[test]
6362 fn long_command_output_keeps_its_verdict() {
6363 let out = format!(
6364 "exit_code: 1\noutput:\n{}\n1 failed, 99 passed",
6365 "é.".repeat(20_000)
6366 );
6367 let cut = truncate_keeping_ends(&out, 4096);
6368 assert!(cut.len() <= 4096, "{}", cut.len());
6369 assert!(cut.starts_with("exit_code: 1"));
6370 assert!(cut.ends_with("1 failed, 99 passed"));
6371 assert!(cut.contains("bytes elided from the middle"));
6372 assert_eq!(truncate_keeping_ends("short", 4096), "short");
6373 assert!(truncate_keeping_ends(&out, 4096).contains(ELIDED_MARKER));
6374 let tiny = truncate_keeping_ends(&"x".repeat(1000), 64);
6375 assert!(tiny.len() <= 64, "{}", tiny.len());
6376 }
6377}