car_server_core/session.rs
1//! Server-side session state — shared across all connections.
2
3use car_engine::{AdmissionGate, GateContext, GateOutcome, Runtime, ToolExecutor};
4use car_eventlog::EventLog;
5use car_ir::ToolFailure;
6use car_proto::{ToolCancelRequest, ToolExecuteRequest, ToolExecuteResponse};
7use futures::Sink;
8use serde::{Deserialize, Serialize};
9use serde_json::Value;
10use sha2::{Digest, Sha256};
11use std::collections::HashMap;
12use std::path::{Path, PathBuf};
13use std::pin::Pin;
14use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
15use std::sync::Arc;
16use tokio::sync::{mpsc, oneshot, Mutex};
17use tokio_tungstenite::tungstenite::{Error as WsError, Message};
18
19/// Type-erased WebSocket sink. The dispatch loop accepts either a
20/// `WebSocketStream<TcpStream>` (the legacy car-server TCP listener)
21/// or a `WebSocketStream<UnixStream>` (the daemon-as-default UDS
22/// listener) — both implement `Sink<Message, Error = WsError>` after
23/// the tungstenite handshake. Erasing the type here avoids cascading
24/// a generic parameter through every WsChannel / Session / ServerState
25/// touchpoint in the dispatcher.
26pub type WsSink = Pin<Box<dyn Sink<Message, Error = WsError> + Send + Unpin + 'static>>;
27
28/// Grace window applied on disconnect before a still-open run is marked
29/// `Incomplete` (agent run tracing, U1 — R5). Short on purpose: it only
30/// has to cover the gap between a healthy `runs.complete` being
31/// dispatched on a spawned task and its terminal record landing, not any
32/// real work. Long enough to absorb that scheduling jitter, short enough
33/// that a genuinely abandoned run is reported `Incomplete` promptly.
34pub const RUN_COMPLETE_GRACE: std::time::Duration = std::time::Duration::from_millis(250);
35
36/// Bounded in-process lease for a disconnected run whose producer negotiated
37/// `runs.resume.v1`. Ten seconds leaves practical headroom for a loaded Mac's
38/// Python controller to establish a fresh local socket, authenticate, and
39/// negotiate capabilities without turning this into restart recovery. Runs
40/// without the capability keep the original 250 ms behavior; an unclaimed
41/// resumable run still becomes `Incomplete`.
42pub const RUN_RESUME_LEASE: std::time::Duration = std::time::Duration::from_secs(10);
43
44/// Maximum time an async run-lifecycle mutation waits for the journal writer's
45/// durability acknowledgement. Five seconds matches the server's existing
46/// acknowledgement/grace budget and remains well inside the default mutating
47/// handler deadline. If an operator override cancels earlier, the async append
48/// still marks the exact row pending before its first yield.
49const CRITICAL_JOURNAL_ACKNOWLEDGEMENT_TIMEOUT: std::time::Duration =
50 std::time::Duration::from_secs(5);
51
52/// Startup uses the same durability budget as live lifecycle mutations. A
53/// stalled journal must fail the constructor before the daemon listens rather
54/// than occupying the startup thread forever.
55const STARTUP_RECONCILIATION_ACKNOWLEDGEMENT_TIMEOUT: std::time::Duration =
56 CRITICAL_JOURNAL_ACKNOWLEDGEMENT_TIMEOUT;
57
58/// Per-run observation bound for disconnect cleanup: the ten-second resume
59/// lease, two bounded five-second journal acknowledgement attempts (the second
60/// is an exact retry after durability-unknown), and scheduler headroom. Cleanup
61/// never clears the run or binding unless an acknowledgement arrives.
62pub const RUN_DISCONNECT_CLEANUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(22);
63
64const RUN_SUBSCRIBE_SUMMARY_STATE_RETRY_LIMIT: usize = 3;
65
66/// Stable admission reason returned after a callback terminally halts its
67/// WebSocket session. The halt is intentionally connection-local and dies on
68/// reconnect; durable run outcomes remain the responsibility of `runs.complete`.
69pub const SESSION_HALTED_REASON: &str =
70 "session halted by a terminal tool failure; reconnect or ask a host to call session.clear_halt";
71
72struct SessionHaltAdmissionGate {
73 halted: Arc<AtomicBool>,
74}
75
76#[async_trait::async_trait]
77impl AdmissionGate for SessionHaltAdmissionGate {
78 fn name(&self) -> &str {
79 "session_halt"
80 }
81
82 async fn check(
83 &self,
84 _proposal: &car_ir::ActionProposal,
85 _ctx: &GateContext<'_>,
86 ) -> GateOutcome {
87 if self.halted.load(Ordering::Acquire) {
88 GateOutcome::reject_all(SESSION_HALTED_REASON)
89 } else {
90 GateOutcome::Allow
91 }
92 }
93}
94
95/// Server-side credentials for continuing an A2A-owned A2UI surface.
96///
97/// This intentionally lives outside `car_a2ui::A2uiSurfaceOwner` so
98/// renderers can inspect surface ownership without receiving secrets.
99#[derive(Debug, Clone, Serialize, Deserialize)]
100#[serde(rename_all = "camelCase", tag = "type")]
101pub enum A2aRouteAuth {
102 None,
103 Bearer { token: String },
104 Header { name: String, value: String },
105}
106
107#[cfg(test)]
108mod run_reservation_liveness_tests {
109 use super::*;
110 use crate::run_store::RunStoreLookupGate;
111 use std::time::Duration;
112
113 fn pristine_run(run_id: &str, client_id: &str) -> RunMeta {
114 RunMeta {
115 run_id: run_id.to_string(),
116 agent_id: "agent-release".to_string(),
117 client_id: client_id.to_string(),
118 active_client_id: client_id.to_string(),
119 resume_predecessor_client_id: None,
120 resume_lease: None,
121 intent: "release without blocking the registry".to_string(),
122 outcome_description: None,
123 started_at: chrono::Utc::now(),
124 termination: None,
125 ended_at: None,
126 turns: Vec::new(),
127 start_committed: false,
128 pending_terminal: None,
129 cancellation_pending: None,
130 cancellation_receipt: None,
131 trace_corruption: None,
132 durability_generation: 0,
133 }
134 }
135
136 #[tokio::test]
137 async fn blocked_release_lookup_does_not_hold_global_runs_lock() {
138 let tmp = tempfile::TempDir::new().unwrap();
139 let gate = RunStoreLookupGate::default();
140 let state = Arc::new(ServerState::with_config(
141 ServerStateConfig::new(tmp.path().join("journals"))
142 .with_run_store_lookup_gate(gate.clone()),
143 ));
144 let session = state
145 .create_session("release-client", Arc::new(WsChannel::test_stub()))
146 .await
147 .unwrap();
148 let expected = pristine_run("release-candidate", &session.client_id);
149 assert!(matches!(
150 state.reserve_run(expected.clone()).await,
151 Ok(RunReservation::New)
152 ));
153
154 gate.block_next();
155 let release_state = state.clone();
156 let release_session = session.clone();
157 let release_expected = expected.clone();
158 let release = tokio::spawn(async move {
159 release_state
160 .release_unpersisted_run_reservation(&release_session, &release_expected)
161 .await
162 });
163 let wait_gate = gate.clone();
164 assert!(
165 tokio::task::spawn_blocking(move || {
166 wait_gate.wait_until_entered(Duration::from_secs(3))
167 })
168 .await
169 .unwrap(),
170 "release must reach the blocked durable lookup"
171 );
172
173 let registry = tokio::time::timeout(Duration::from_secs(1), state.runs.lock())
174 .await
175 .expect("blocked release lookup must not hold the global runs lock");
176 drop(registry);
177 gate.release();
178 assert_eq!(release.await.unwrap(), Ok(()));
179 assert!(state.run_meta(&expected.run_id).await.is_none());
180 }
181}
182
183/// Shared write half of the WebSocket, plus pending callback channels.
184/// `write` is type-erased via [`WsSink`] so the dispatcher can run
185/// against any transport-specific WebSocketStream (TCP or UDS today;
186/// axum-bridged in future) without templatizing every consumer.
187pub struct WsChannel {
188 pub write: Mutex<WsSink>,
189 /// Pending tool execution callbacks: request_id → oneshot sender
190 pub pending: Mutex<HashMap<String, oneshot::Sender<ToolExecuteResponse>>>,
191 /// Body-free correlation for active callbacks. A session owns at most one
192 /// active run, so every entry belongs to that run while its lifecycle
193 /// guard is held by proposal execution.
194 pub active_actions: Mutex<HashMap<String, String>>,
195 pub next_id: AtomicU64,
196}
197
198/// A WS connection is a host event sink (#418): `HostState` broadcasts
199/// `host.event` frames to subscribers through this trait, so it never has to
200/// know about the concrete `WsChannel` / tungstenite types.
201#[async_trait::async_trait]
202impl crate::host::EventSubscriber for WsChannel {
203 async fn send_text(&self, json: String) {
204 use futures::SinkExt;
205 // Bound both lock acquisition and the socket write. Host events are
206 // emitted from reducer transitions; a suspended client must not hold
207 // that reducer's announcement lock (and browser controls behind it)
208 // indefinitely through TCP backpressure.
209 let _ = tokio::time::timeout(std::time::Duration::from_secs(10), async {
210 self.write
211 .lock()
212 .await
213 .send(Message::Text(json.into()))
214 .await
215 })
216 .await;
217 }
218}
219
220impl WsChannel {
221 pub fn next_request_id(&self) -> String {
222 let id = self.next_id.fetch_add(1, Ordering::SeqCst);
223 format!("cb-{}", id)
224 }
225
226 /// Test-only stub that returns a WsChannel whose write sink drains
227 /// to nowhere. Used by `host.rs` tests that need a real
228 /// `Arc<WsChannel>` in the subscribers map (to exercise membership
229 /// checks like the cross-session resolve fan-out) without
230 /// constructing a tungstenite handshake. Never writes are
231 /// performed against this stub; if anything tries, the drain sink
232 /// quietly absorbs.
233 #[cfg(test)]
234 pub fn test_stub() -> Self {
235 use futures::sink::SinkExt;
236 let sink: WsSink = Box::pin(
237 futures::sink::drain()
238 .sink_map_err(|_| tokio_tungstenite::tungstenite::Error::ConnectionClosed),
239 );
240 WsChannel {
241 write: Mutex::new(sink),
242 pending: Mutex::new(HashMap::new()),
243 active_actions: Mutex::new(HashMap::new()),
244 next_id: AtomicU64::new(0),
245 }
246 }
247
248 /// Test-only channel whose write sink is *observable*: every
249 /// `Message::Text` payload written to it is appended to the returned
250 /// shared Vec. `test_stub` drains to nowhere, which makes it useless
251 /// for asserting that a specific frame — e.g. the `tools.cancel`
252 /// notification — actually reached the wire.
253 ///
254 /// The Vec is behind a `std::sync::Mutex` (not tokio's) because the
255 /// sink closure is synchronous; the lock is taken and released inside
256 /// the closure before the returned future is handed back, so a test
257 /// holding the lock to assert can never deadlock a concurrent write.
258 #[cfg(test)]
259 pub fn test_capture() -> (Self, std::sync::Arc<std::sync::Mutex<Vec<String>>>) {
260 let frames = std::sync::Arc::new(std::sync::Mutex::new(Vec::<String>::new()));
261 let sink: WsSink = Box::pin(futures::sink::unfold(
262 frames.clone(),
263 |frames: std::sync::Arc<std::sync::Mutex<Vec<String>>>, msg: Message| {
264 if let Message::Text(text) = &msg {
265 // Scoped so the lock is released before we yield the
266 // (already-ready) future back to the sink machinery.
267 if let Ok(mut held) = frames.lock() {
268 held.push(text.as_str().to_string());
269 }
270 }
271 futures::future::ready(Ok::<_, WsError>(frames))
272 },
273 ));
274 (
275 WsChannel {
276 write: Mutex::new(sink),
277 pending: Mutex::new(HashMap::new()),
278 active_actions: Mutex::new(HashMap::new()),
279 next_id: AtomicU64::new(0),
280 },
281 frames,
282 )
283 }
284}
285
286/// In-flight `agents.chat` session bookkeeping. Created when a host
287/// client calls `agents.chat`, removed when the agent emits a terminal
288/// `agent.chat.event` (`kind: "done"` or `"error"`), when either side
289/// disconnects, or when the host cancels via `agents.chat.cancel`.
290///
291/// The session_id is host-supplied (or server-generated when omitted)
292/// and threads through every `agent.chat.event` notification so the
293/// server can route streamed deltas back to the originating host
294/// without needing per-session subscriptions. See
295/// `docs/proposals/agent-chat-surface.md` for the wire contract.
296#[derive(Debug, Clone)]
297pub struct ChatSession {
298 /// Agent that owns this chat — populated from
299 /// `attached_agents` at `agents.chat` dispatch time.
300 pub agent_id: String,
301 /// Client id of the host that issued `agents.chat`. The server
302 /// forwards `agent.chat.event` notifications back to *this* host
303 /// only, so two CarHost windows chatting with the same agent are
304 /// independent streams.
305 pub host_client_id: String,
306 /// Unix-seconds creation time — used by the future stale-session
307 /// sweeper to drop sessions whose agent died without emitting a
308 /// terminal event.
309 pub created_at: u64,
310 /// In-process cancellation flag for daemon-owned chat runs. External
311 /// attached agents receive `agent.chat.cancel` instead.
312 pub local_cancel: Option<Arc<AtomicBool>>,
313}
314
315/// Host-visible standing goal for a chat session. `goal.set` stores this keyed
316/// by `session_id`; `agents.chat` reuses it for that session when the turn does
317/// not pass an inline `goal`.
318#[derive(Debug, Clone, Serialize, Deserialize)]
319pub struct ChatGoalState {
320 pub session_id: String,
321 pub check: String,
322 pub max_iterations: u32,
323 pub status: String,
324 pub last_iteration: Option<u32>,
325 pub last_met: Option<bool>,
326 pub last_grounded: Option<bool>,
327 pub last_reason: Option<String>,
328 pub terminal_kind: Option<String>,
329 pub terminal_message: Option<String>,
330 pub updated_at: u64,
331}
332
333fn chat_goals_path_from_journal_dir(journal_dir: &Path) -> PathBuf {
334 let car_dir = journal_dir
335 .parent()
336 .map(PathBuf::from)
337 .unwrap_or_else(|| PathBuf::from("."));
338 car_dir.join("chat-goals.json")
339}
340
341fn load_chat_goals_from_disk(journal_dir: &Path) -> HashMap<String, ChatGoalState> {
342 let path = chat_goals_path_from_journal_dir(journal_dir);
343 let Ok(text) = std::fs::read_to_string(&path) else {
344 return HashMap::new();
345 };
346 match serde_json::from_str::<HashMap<String, ChatGoalState>>(&text) {
347 Ok(mut goals) => {
348 for goal in goals.values_mut() {
349 if goal.status == "running" {
350 goal.status = "active".to_string();
351 }
352 }
353 goals
354 }
355 Err(e) => {
356 tracing::warn!(
357 path = %path.display(),
358 error = %e,
359 "chat-goals store was unreadable; starting with an empty in-memory goal registry"
360 );
361 HashMap::new()
362 }
363 }
364}
365
366fn write_chat_goals_atomic(
367 path: &std::path::Path,
368 goals: &HashMap<String, ChatGoalState>,
369) -> std::io::Result<()> {
370 use std::sync::atomic::{AtomicU64, Ordering};
371 static SEQ: AtomicU64 = AtomicU64::new(0);
372 if let Some(parent) = path.parent() {
373 std::fs::create_dir_all(parent)?;
374 }
375 let json = serde_json::to_vec_pretty(goals)?;
376 let seq = SEQ.fetch_add(1, Ordering::Relaxed);
377 let mut tmp_os = path.as_os_str().to_owned();
378 tmp_os.push(format!(".tmp.{}.{}", std::process::id(), seq));
379 let tmp = PathBuf::from(tmp_os);
380 std::fs::write(&tmp, json)?;
381 std::fs::rename(&tmp, path)
382}
383
384/// One normalized chunk of an `agent.chat` stream, delivered to an in-process
385/// [`chat_collectors`](ServerState::chat_collectors) sink (the A2A bridge). A
386/// `token` chunk carries a `delta`; a terminal chunk has `kind` `"done"`
387/// (success) or `"error"` (with `error` text).
388#[derive(Debug, Clone)]
389pub struct ChatStreamChunk {
390 pub kind: String,
391 pub delta: Option<String>,
392 pub error: Option<String>,
393}
394
395/// An in-process `agent.chat` collector entry: the sink that
396/// [`try_forward_agent_chat_event`](crate::handler::try_forward_agent_chat_event)
397/// feeds, plus the host client whose disconnect should abandon it. Holding the
398/// sole sender here means dropping the entry closes the stream, so the
399/// collecting task wakes immediately instead of waiting out its timeout.
400pub struct ChatCollector {
401 pub tx: tokio::sync::mpsc::UnboundedSender<ChatStreamChunk>,
402 pub host_client_id: String,
403}
404
405#[derive(Debug, Clone)]
406#[doc(hidden)]
407pub struct RunResumeLease {
408 pub disconnected_client_id: String,
409 pub expires_at: tokio::time::Instant,
410}
411
412/// Deterministic test seam at the authenticated completion boundary. It is
413/// inert unless explicitly armed through [`ServerStateConfig`].
414#[derive(Debug, Clone, Default)]
415#[doc(hidden)]
416pub struct RunCompletionFenceGate {
417 armed: Arc<std::sync::atomic::AtomicBool>,
418 entered: Arc<tokio::sync::Notify>,
419 release: Arc<tokio::sync::Notify>,
420}
421
422impl RunCompletionFenceGate {
423 pub fn block_next(&self) {
424 self.armed.store(true, std::sync::atomic::Ordering::Release);
425 }
426
427 pub async fn wait_until_entered(&self, timeout: std::time::Duration) -> bool {
428 tokio::time::timeout(timeout, self.entered.notified())
429 .await
430 .is_ok()
431 }
432
433 pub fn release(&self) {
434 self.armed
435 .store(false, std::sync::atomic::Ordering::Release);
436 self.release.notify_one();
437 }
438
439 async fn wait_if_armed(&self) {
440 if !self.armed.load(std::sync::atomic::Ordering::Acquire) {
441 return;
442 }
443 self.entered.notify_one();
444 if self.armed.load(std::sync::atomic::Ordering::Acquire) {
445 self.release.notified().await;
446 }
447 }
448}
449
450/// Daemon-side record of a single agent run (agent run tracing, U1).
451///
452/// Keyed by `run_id` in the process-wide [`ServerState::runs`] registry
453/// so the record survives the WS connection that produced it — needed
454/// both for the disconnect grace window (R5) and so the disk store (U3)
455/// can flush a run even after its session is gone. U1 keeps this purely
456/// in memory; persistence is U3.
457#[derive(Debug, Clone)]
458pub struct RunMeta {
459 pub run_id: String,
460 /// Owning agent. Set at `runs.start` from the resolved id; used by
461 /// the read RPCs' ownership check (U5/KTD10) and the disk key (U3).
462 pub agent_id: String,
463 /// Immutable WS `client_id` that durably called `runs.start`. This remains
464 /// the provenance identity written to `RunStarted`, proposal receipts, and
465 /// `RunEnded`, even if an authenticated replacement socket resumes the
466 /// in-memory run.
467 pub client_id: String,
468 /// The one live socket currently authorized to continue the run. Equal to
469 /// `client_id` for a fresh run; changed only by authenticated orphan
470 /// reclaim after the previous owner has disappeared from `sessions`.
471 pub active_client_id: String,
472 /// Stable predecessor returned by an idempotent `runs.resume` retry on the
473 /// winning replacement socket.
474 pub resume_predecessor_client_id: Option<String>,
475 /// Ephemeral, in-process authority window installed atomically when the
476 /// active socket leaves the session registry. It is intentionally absent
477 /// from RunStore/journals and therefore never survives daemon restart.
478 #[doc(hidden)]
479 pub resume_lease: Option<RunResumeLease>,
480 pub intent: String,
481 pub outcome_description: Option<String>,
482 pub started_at: chrono::DateTime<chrono::Utc>,
483 /// `None` while the run is in progress; `Some` once a terminal
484 /// record (a reported outcome or an `Incomplete` marker) is
485 /// written. The presence of this field is the "is this run still
486 /// open?" signal the grace window checks.
487 pub termination: Option<car_proto::RunTermination>,
488 /// When the terminal record was written, if any.
489 pub ended_at: Option<chrono::DateTime<chrono::Utc>>,
490 /// The ordered per-turn trace recorded for this run (agent run
491 /// tracing, U2). Each entry is a [`car_proto::RunRecord::Turn`]
492 /// produced by the recorder from a submitted proposal + its
493 /// `ActionResult`s. U1 leaves this empty; the U2 recorder appends to
494 /// it via [`ServerState::record_run_turns`]. U3 flushes this buffer
495 /// to disk and U4 broadcasts it — both read it through
496 /// [`ServerState::run_turns`]. The `turn.index` field is monotonic
497 /// across the run's proposals, so this Vec is the clean ordered
498 /// stream those units consume.
499 pub turns: Vec<car_proto::RunRecord>,
500 /// True only after the exact RunStarted row reached flush+fsync and the
501 /// authenticated run_started journal event received the same durability
502 /// acknowledgement. A reserved-but-uncommitted run blocks all proposal
503 /// writes and can only be resumed by its owning client.
504 pub start_committed: bool,
505 /// Stable terminal transaction preimage retained across a failed
506 /// RunStore or journal durability boundary. Retries must present the same
507 /// RunTermination and finish this exact RunEnded; a conflicting terminal
508 /// is rejected.
509 pub pending_terminal: Option<car_proto::RunEnded>,
510 /// Durable cancellation request. Presence quarantines new proposals and
511 /// `runs.complete` until CAR either confirms a Cancelled terminal or an
512 /// operator independently resolves the unconfirmed stop.
513 pub cancellation_pending: Option<car_proto::RunCancellationRequested>,
514 /// Stable response for same-key idempotent retries.
515 pub cancellation_receipt: Option<car_proto::RunCancelResponse>,
516 /// Stable fail-closed error once the durable JSONL is known corrupt.
517 /// A quarantined run accepts no more proposals/turns, cannot complete,
518 /// and cannot install or retain live subscribers.
519 pub trace_corruption: Option<String>,
520 /// Monotonic in-memory commit generation for durability work which drops
521 /// the global run-registry lock while filesystem I/O executes.
522 #[doc(hidden)]
523 pub durability_generation: u64,
524}
525
526#[derive(Debug, Clone)]
527pub enum RunReservation {
528 New,
529 Existing(RunMeta),
530}
531
532/// In-memory ownership result for the authenticated WebSocket resume path.
533/// `active_client_id` is the fenced socket allowed to use the original
534/// durable run provenance now.
535#[derive(Debug, Clone, PartialEq, Eq)]
536pub(crate) struct RunResumeBinding {
537 pub run_id: String,
538 pub agent_id: String,
539 pub active_client_id: String,
540 pub resumed_from_client_id: String,
541}
542
543/// Canonical terminal identity shared by the durable `RunEnded` row, terminal
544/// journal event, and `runs.complete` response. The input is exactly the RFC
545/// 8785/JCS serialization of `RunTermination`; it deliberately does not include
546/// timestamps, run ids, client ids, or Daily artifact bytes.
547pub(crate) fn run_completion_digest(
548 termination: &car_proto::RunTermination,
549) -> Result<String, String> {
550 let canonical = car_inference::catalog_identity::canonical_json(termination)?;
551 Ok(format!("{:x}", Sha256::digest(canonical.as_bytes())))
552}
553
554fn run_trace_corruption_message(run_id: &str, detail: impl std::fmt::Display) -> String {
555 format!(
556 "{} run `{run_id}`: {detail}",
557 car_proto::RUN_TRACE_CORRUPTION_MESSAGE_PREFIX
558 )
559}
560
561fn same_run_termination(
562 left: &car_proto::RunTermination,
563 right: &car_proto::RunTermination,
564) -> bool {
565 match (run_completion_digest(left), run_completion_digest(right)) {
566 (Ok(left), Ok(right)) => left == right,
567 _ => false,
568 }
569}
570
571fn same_run_ended(left: &car_proto::RunEnded, right: &car_proto::RunEnded) -> bool {
572 left.run_id == right.run_id
573 && left.client_id == right.client_id
574 && left.agent_id == right.agent_id
575 && left.completion_digest == right.completion_digest
576 && left.ended_at == right.ended_at
577 && same_run_termination(&left.termination, &right.termination)
578}
579
580struct StartupJournalCache {
581 journal_dir: PathBuf,
582 failures: Option<car_eventlog::JournalFailureInjector>,
583 logs: HashMap<String, EventLog>,
584}
585
586impl StartupJournalCache {
587 fn new(journal_dir: &Path, failures: Option<&car_eventlog::JournalFailureInjector>) -> Self {
588 Self {
589 journal_dir: journal_dir.to_path_buf(),
590 failures: failures.cloned(),
591 logs: HashMap::new(),
592 }
593 }
594
595 fn get(&mut self, client_id: &str) -> Result<&mut EventLog, String> {
596 if !self.logs.contains_key(client_id) {
597 let path = self.journal_dir.join(format!("{client_id}.jsonl"));
598 let log = match (path.exists(), self.failures.as_ref()) {
599 (true, Some(failures)) => {
600 EventLog::load_with_journal_failure_injector(&path, failures.clone())
601 .map_err(|error| error.to_string())
602 }
603 (true, None) => EventLog::load(&path).map_err(|error| error.to_string()),
604 (false, Some(failures)) => Ok(EventLog::with_journal_failure_injector(
605 path,
606 failures.clone(),
607 )),
608 (false, None) => Ok(EventLog::with_journal(path)),
609 }?;
610 self.logs.insert(client_id.to_string(), log);
611 }
612 self.logs
613 .get_mut(client_id)
614 .ok_or_else(|| "startup journal cache insertion failed".to_string())
615 }
616}
617
618fn append_recovered_cancellation_result_journal(
619 journal_dir: PathBuf,
620 failures: Option<car_eventlog::JournalFailureInjector>,
621 client_id: String,
622 result: car_proto::RunCancelResponse,
623) -> Result<bool, String> {
624 let mut journals = StartupJournalCache::new(&journal_dir, failures.as_ref());
625 let log = journals.get(&client_id)?;
626 log.bind_run(&result.run_id, &client_id)?;
627 let Value::Object(data) = serde_json::to_value(&result).map_err(|error| error.to_string())?
628 else {
629 return Err("recovered cancellation result did not serialize as an object".into());
630 };
631 let data: HashMap<String, Value> = data.into_iter().collect();
632 let already_durable = log.events().iter().any(|event| {
633 event.kind == car_eventlog::EventKind::RunCancellationResult
634 && event.run_id.as_deref() == Some(result.run_id.as_str())
635 && event.client_id.as_deref() == Some(client_id.as_str())
636 && event.action_id.as_deref() == result.action_id.as_deref()
637 && event.proposal_id.is_none()
638 && event.data == data
639 });
640 log.append_critical_bounded(
641 car_eventlog::EventKind::RunCancellationResult,
642 result.action_id.as_deref(),
643 None,
644 data,
645 CRITICAL_JOURNAL_ACKNOWLEDGEMENT_TIMEOUT,
646 )
647 .map_err(|error| format!("recovered cancellation journal append failed: {error}"))?;
648 Ok(!already_durable)
649}
650
651fn reconcile_durable_run_journals(
652 journals: &mut StartupJournalCache,
653 store: &crate::run_store::RunStore,
654 acknowledgement_timeout: std::time::Duration,
655) -> Result<(), String> {
656 type DurableRunBoundaries = (
657 car_proto::RunStarted,
658 Option<car_proto::RunEnded>,
659 Option<car_proto::RunCancellationRequested>,
660 Option<car_proto::RunCancelResponse>,
661 );
662 let mut by_client: HashMap<String, Vec<DurableRunBoundaries>> = HashMap::new();
663 store.visit_run_boundaries(|started, ended, requested, result| {
664 let Some(client_id) = started.client_id.as_deref() else {
665 return;
666 };
667 if client_id.is_empty()
668 || client_id.contains('/')
669 || client_id.contains('\\')
670 || client_id == "."
671 || client_id == ".."
672 {
673 tracing::error!(run_id = %started.run_id, "run-journal outbox has unsafe client_id");
674 return;
675 }
676 by_client
677 .entry(client_id.to_string())
678 .or_default()
679 .push((started, ended, requested, result));
680 });
681
682 for (client_id, mut runs) in by_client {
683 runs.sort_by_key(|(started, _, _, _)| started.started_at);
684 let log = match journals.get(&client_id) {
685 Ok(log) => log,
686 Err(error) => {
687 tracing::error!(%client_id, %error, "cannot load run journal for outbox reconciliation");
688 continue;
689 }
690 };
691 for (started, ended, requested, result) in runs {
692 if let Err(error) = log.bind_run(&started.run_id, &client_id) {
693 tracing::error!(run_id = %started.run_id, %error, "cannot bind run journal during outbox reconciliation");
694 continue;
695 }
696 let mut start_data = HashMap::from([
697 (
698 "agent_id".to_string(),
699 Value::from(started.agent_id.clone()),
700 ),
701 ("intent".to_string(), Value::from(started.intent.clone())),
702 (
703 "started_at".to_string(),
704 serde_json::to_value(started.started_at).unwrap_or(Value::Null),
705 ),
706 ]);
707 if let Some(description) = &started.outcome_description {
708 start_data.insert(
709 "outcome_description".to_string(),
710 Value::from(description.clone()),
711 );
712 }
713 log.append_critical_bounded(
714 car_eventlog::EventKind::RunStarted,
715 None,
716 None,
717 start_data,
718 acknowledgement_timeout,
719 )
720 .map_err(|error| {
721 format!(
722 "startup reconciliation failed for run {} run_started: {error}",
723 started.run_id
724 )
725 })?;
726
727 if let Some(requested) = requested {
728 let Value::Object(data) = serde_json::to_value(&requested).unwrap_or(Value::Null)
729 else {
730 continue;
731 };
732 log.append_critical_bounded(
733 car_eventlog::EventKind::RunCancellationRequested,
734 requested.action_id.as_deref(),
735 None,
736 data.into_iter().collect(),
737 acknowledgement_timeout,
738 )
739 .map_err(|error| {
740 format!(
741 "startup reconciliation failed for run {} cancellation request: {error}",
742 started.run_id
743 )
744 })?;
745 }
746
747 if ended.is_none() {
748 if let Some(result) = result {
749 let Value::Object(data) = serde_json::to_value(&result).unwrap_or(Value::Null)
750 else {
751 continue;
752 };
753 log.append_critical_bounded(
754 car_eventlog::EventKind::RunCancellationResult,
755 result.action_id.as_deref(),
756 None,
757 data.into_iter().collect(),
758 acknowledgement_timeout,
759 )
760 .map_err(|error| {
761 format!(
762 "startup reconciliation failed for run {} cancellation result: {error}",
763 started.run_id
764 )
765 })?;
766 }
767 }
768
769 let Some(ended) = ended else {
770 continue;
771 };
772 let Some(completion_digest) = ended.completion_digest.as_deref() else {
773 log.clear_run_binding(&started.run_id, &client_id)?;
774 continue;
775 };
776 if ended.client_id.as_deref() != Some(client_id.as_str()) {
777 tracing::error!(run_id = %started.run_id, "RunEnded owner does not match RunStarted during outbox reconciliation");
778 log.clear_run_binding(&started.run_id, &client_id)?;
779 continue;
780 }
781 let termination_kind = match &ended.termination {
782 car_proto::RunTermination::Outcome { .. } => "outcome",
783 car_proto::RunTermination::Incomplete => "incomplete",
784 car_proto::RunTermination::Cancelled { .. } => "cancelled",
785 };
786 let data = HashMap::from([
787 (
788 "termination_kind".to_string(),
789 Value::from(termination_kind),
790 ),
791 (
792 "completion_digest".to_string(),
793 Value::from(completion_digest),
794 ),
795 (
796 "termination".to_string(),
797 serde_json::to_value(&ended.termination).unwrap_or(Value::Null),
798 ),
799 ]);
800 log.append_critical_bounded(
801 car_eventlog::EventKind::RunCompleted,
802 None,
803 None,
804 data,
805 acknowledgement_timeout,
806 )
807 .map_err(|error| {
808 format!(
809 "startup reconciliation failed for run {} run_completed: {error}",
810 started.run_id
811 )
812 })?;
813 log.clear_run_binding(&started.run_id, &client_id)?;
814 }
815 }
816 Ok(())
817}
818
819/// Finish proposal terminals whose actions already ran before the daemon lost
820/// the journal acknowledgement. The outbox owns the exact normal-serde result
821/// and JCS digest, so startup only replays that terminal row; it never invokes
822/// the runtime or a tool executor.
823fn reconcile_pending_proposal_journals(
824 journals: &mut StartupJournalCache,
825 store: &crate::run_store::RunStore,
826 acknowledgement_timeout: std::time::Duration,
827) -> Result<(), String> {
828 let pending_proposals = match store.all_pending_proposals() {
829 Ok(pending) => pending,
830 Err(error) => {
831 tracing::error!(%error, "cannot enumerate proposal-finalization outbox; all unresolved runs remain quarantined");
832 return Ok(());
833 }
834 };
835 for pending in pending_proposals {
836 match store.completed_proposal(
837 &pending.run_id,
838 &pending.client_id,
839 pending.requested_policy_session_id.as_deref(),
840 &pending.original_submission,
841 ) {
842 Ok(Some(receipt)) => {
843 if let Err(error) = store.cleanup_completed_proposal_guards(&receipt) {
844 tracing::error!(run_id = %pending.run_id, %error, "completed proposal response is durable; startup guard cleanup remains pending");
845 }
846 continue;
847 }
848 Ok(None) => {}
849 Err(error) => {
850 tracing::error!(run_id = %pending.run_id, %error, "cannot validate exact completed proposal response during startup cleanup");
851 continue;
852 }
853 }
854 let (started, marker) = match store.pending_provenance(&pending) {
855 Ok(provenance) => provenance,
856 Err(error) => {
857 tracing::error!(run_id = %pending.run_id, %error, "proposal-finalization provenance is invalid; preserving outcome-unknown quarantine");
858 continue;
859 }
860 };
861 let Some(client_id) = started.client_id.as_deref() else {
862 tracing::error!(run_id = %pending.run_id, "durable RunStarted is missing client_id");
863 continue;
864 };
865 if client_id.is_empty()
866 || client_id.contains('/')
867 || client_id.contains('\\')
868 || client_id == "."
869 || client_id == ".."
870 {
871 tracing::error!(run_id = %pending.run_id, "proposal-finalization outbox has unsafe client_id");
872 continue;
873 }
874 let canonical = match car_inference::catalog_identity::canonical_json(
875 &pending.proposal_result,
876 ) {
877 Ok(canonical) => canonical,
878 Err(error) => {
879 tracing::error!(run_id = %pending.run_id, %error, "proposal-finalization result is not RFC 8785 canonicalizable");
880 continue;
881 }
882 };
883 let digest = format!("{:x}", Sha256::digest(canonical.as_bytes()));
884 if digest != pending.result_digest {
885 tracing::error!(run_id = %pending.run_id, "proposal-finalization digest does not match its durable result preimage");
886 continue;
887 }
888
889 let log = match journals.get(client_id) {
890 Ok(log) => log,
891 Err(error) => {
892 tracing::error!(run_id = %pending.run_id, %error, "cannot load proposal journal for outbox reconciliation");
893 continue;
894 }
895 };
896 if let Err(error) = log.bind_run(&started.run_id, client_id) {
897 tracing::error!(run_id = %pending.run_id, %error, "cannot bind proposal journal during outbox reconciliation");
898 continue;
899 }
900 if let Some(policy_session_id) = marker.policy_session_id.as_deref() {
901 if let Err(error) = log.bind_policy_session(policy_session_id) {
902 tracing::error!(run_id = %pending.run_id, %error, "cannot restore proposal policy binding during outbox reconciliation");
903 continue;
904 }
905 }
906 store.ensure_proposal_turns(&pending).map_err(|error| {
907 format!(
908 "startup reconciliation failed for run {} proposal trace: {error}",
909 pending.run_id
910 )
911 })?;
912 log.append_critical_bounded(
913 car_eventlog::EventKind::ProposalCompleted,
914 None,
915 Some(&pending.final_proposal_id),
916 pending.event_data(),
917 acknowledgement_timeout,
918 )
919 .map_err(|error| {
920 format!(
921 "startup reconciliation failed for run {} proposal_completed: {error}",
922 pending.run_id
923 )
924 })?;
925 if let Some(policy_session_id) = marker.policy_session_id.as_deref() {
926 log.clear_policy_session(policy_session_id)?;
927 }
928 let receipt = match store.write_completed_proposal(&pending) {
929 Ok(receipt) => receipt,
930 Err(error) => {
931 tracing::error!(run_id = %pending.run_id, %error, "cannot persist reconciled completed proposal response");
932 continue;
933 }
934 };
935 if let Err(error) = store.cleanup_completed_proposal_guards(&receipt) {
936 tracing::error!(run_id = %pending.run_id, %error, "completed proposal response is durable; startup guard cleanup remains pending");
937 continue;
938 }
939 }
940 Ok(())
941}
942
943/// A completed response can outlive either cleanup guard when the daemon dies
944/// between independently-fsynced unlink boundaries. Reconcile only under the
945/// receipt's exact typed authority; never dispatch or append another event.
946fn reconcile_completed_proposal_guards(store: &crate::run_store::RunStore) {
947 if let Err(error) = store.reconcile_completed_proposal_migration() {
948 tracing::error!(%error, "cannot migrate completed proposal response index; guards remain quarantined");
949 }
950}
951
952/// Per-run turn ceiling — a runaway-loop backstop sized well above any
953/// healthy main-agent-only cycle (tens of turns), never a trimmer. This is
954/// a TRUE hard cap, and it is enforced HERE — inside
955/// [`ServerState::record_run_turns`], under the `runs` lock — not only in
956/// the WS handler's pre-check. The handler keeps a fast-path pre-check off a
957/// lock-free snapshot, but the dispatcher spawns a task per frame, so
958/// pipelined `runs.record_turns` batches can all read a sub-ceiling snapshot
959/// and pass that pre-check before any of them appends. Only the under-lock
960/// check below is atomic with the append, so it is the one that actually
961/// bounds the run (ADV-1). A batch that would take the run PAST this many
962/// recorded turns is refused WHOLE.
963pub const RECORD_TURNS_RUN_CEILING: usize = 2000;
964
965/// Outcome of [`ServerState::record_run_turns`] — distinguishes the three
966/// reasons a batch can fail to land so the caller maps each to the right
967/// `runs.record_turns` drop reason (ADV-1). Before this enum the function
968/// returned a bare `usize` (the new total, or `0` for "nothing appended"),
969/// which collapsed an under-lock CEILING refusal and an unknown/terminal run
970/// into the same `0` — the handler then mislabeled a ceiling refusal as
971/// `run_terminal`.
972#[derive(Debug, Clone, PartialEq, Eq)]
973pub enum RecordRunTurnsOutcome {
974 /// The batch was appended; the run's new total turn count.
975 Appended { new_total: usize },
976 /// The batch was refused WHOLE because it would take the run past
977 /// [`RECORD_TURNS_RUN_CEILING`] (a runaway backstop). Maps to
978 /// `dropped: "run_turn_limit"`.
979 RefusedCeiling,
980 /// The run id is unknown to this process OR already terminal — nothing
981 /// was appended. Maps to `dropped: "run_terminal"` (the benign TOCTOU:
982 /// the run went terminal between the handler's pre-check and the
983 /// append), matching the prior silent-zero semantics.
984 UnknownOrTerminal,
985 /// The batch cannot advance canonical memory/subscribers. Ordinary I/O
986 /// failures mean the durable append boundary was not acknowledged and an
987 /// exact retry is safe. A `run trace corruption:` error instead means the
988 /// bytes exposed committed middle corruption; the run is quarantined and
989 /// must not be retried or completed.
990 PersistenceFailed(String),
991}
992
993pub(crate) enum RunSubscribePageResult {
994 Ready(car_proto::RunSubscribeResponse),
995 Durable {
996 agent_id: String,
997 status: car_proto::RunLiveStatus,
998 },
999}
1000
1001impl RunMeta {
1002 /// True once a terminal record (outcome or `Incomplete`) is set.
1003 pub fn is_terminal(&self) -> bool {
1004 self.termination.is_some()
1005 }
1006
1007 pub fn accepts_proposals(&self) -> bool {
1008 self.start_committed
1009 && self.pending_terminal.is_none()
1010 && self.cancellation_pending.is_none()
1011 && self.trace_corruption.is_none()
1012 && !self.is_terminal()
1013 }
1014
1015 /// The coarse live status of this run for the U4 subscribe snapshot /
1016 /// `runs.trace.event` (agent run tracing). Mirrors `RunStore`'s
1017 /// `RunStatus` but reads off the in-memory `RunMeta` so the live path
1018 /// never touches disk.
1019 pub fn live_status(&self) -> car_proto::RunLiveStatus {
1020 match &self.termination {
1021 None if self.cancellation_pending.is_some() => {
1022 car_proto::RunLiveStatus::CancellationPending
1023 }
1024 None => car_proto::RunLiveStatus::InProgress,
1025 Some(car_proto::RunTermination::Outcome { .. }) => car_proto::RunLiveStatus::Completed,
1026 Some(car_proto::RunTermination::Incomplete) => car_proto::RunLiveStatus::Incomplete,
1027 Some(car_proto::RunTermination::Cancelled { .. }) => {
1028 car_proto::RunLiveStatus::Cancelled
1029 }
1030 }
1031 }
1032
1033 /// Count of `RunRecord::Turn` entries in this run's buffer — the live
1034 /// turn cursor. Equals `turns.len()` because the buffer holds only
1035 /// `Turn` records (Started/Ended are not buffered here).
1036 pub fn turn_cursor(&self) -> usize {
1037 self.turns.len()
1038 }
1039}
1040
1041/// Default ceiling for the daemon→host `tools.execute` callback wait when
1042/// an action carries no explicit `timeout_ms`. Overridable via
1043/// `CAR_TOOL_TIMEOUT` (seconds). Raised from the old hardcoded 60s — real
1044/// tools (build steps, CLI drivers, slow APIs) routinely run longer, and a
1045/// 60s ceiling reaped them regardless of the agent's budget (Parslee-ai/car#259).
1046pub const DEFAULT_TOOL_TIMEOUT_MS: u64 = 300_000;
1047
1048/// Grace added to the callback wait above an action's own `timeout_ms`. The
1049/// executor (`car-engine`) already wraps each attempt in
1050/// `timeout(action.timeout_ms, dispatch)`; if this wait used the *same*
1051/// value it would race that outer deadline with zero slack. By waiting a
1052/// little longer, the executor's deadline fires first and reaps+rolls back
1053/// cleanly with its structured "action timed out" error — this wait is only
1054/// a transport backstop for the case the executor deadline doesn't apply.
1055const TOOL_TIMEOUT_GRACE_MS: u64 = 5_000;
1056
1057/// The callback-wait budget for one tool call.
1058///
1059/// - **`Some(X)`**: the executor is the authority (it bounds the attempt at
1060/// `X`); this wait is `X + grace`, a backstop that lets the executor reap
1061/// first. So the agent's per-action budget takes effect — the bug was that
1062/// this wait used a hardcoded 60s and ignored `X` entirely (car#259).
1063/// - **`None`**: the executor applies **no** deadline, so this wait is the
1064/// *sole* bound on the call — the `CAR_TOOL_TIMEOUT`-overridable
1065/// [`DEFAULT_TOOL_TIMEOUT_MS`].
1066fn tool_callback_timeout(action_timeout_ms: Option<u64>) -> std::time::Duration {
1067 let ms = match action_timeout_ms {
1068 Some(budget) => budget.saturating_add(TOOL_TIMEOUT_GRACE_MS),
1069 None => std::env::var("CAR_TOOL_TIMEOUT")
1070 .ok()
1071 .and_then(|s| s.trim().parse::<u64>().ok())
1072 .map(|secs| secs.saturating_mul(1000))
1073 .unwrap_or(DEFAULT_TOOL_TIMEOUT_MS),
1074 };
1075 std::time::Duration::from_millis(ms)
1076}
1077
1078/// Build and write the fire-and-forget `tools.cancel` notification for a
1079/// cancelled `tools.execute` callback (Parslee-ai/car#264).
1080///
1081/// This is the ONE place that constructs a [`ToolCancelRequest`] frame and
1082/// puts it on the wire; every cancellation path routes through
1083/// [`PendingToolCall`]'s drop, which calls this. Best-effort by design — the
1084/// daemon has already given up on the call, so a failed write changes nothing
1085/// it can act on.
1086pub(crate) async fn write_tool_cancel(
1087 channel: &WsChannel,
1088 request_id: String,
1089 action_id: String,
1090 reason: String,
1091) {
1092 use futures::SinkExt;
1093 let cancel = ToolCancelRequest {
1094 request_id,
1095 action_id,
1096 reason,
1097 };
1098 let notification = serde_json::json!({
1099 "jsonrpc": "2.0",
1100 "method": "tools.cancel",
1101 "params": cancel,
1102 });
1103 let Ok(text) = serde_json::to_string(¬ification) else {
1104 return;
1105 };
1106 let _ = channel
1107 .write
1108 .lock()
1109 .await
1110 .send(Message::Text(text.into()))
1111 .await;
1112}
1113
1114/// RAII guard covering one in-flight daemon→host `tools.execute` callback
1115/// (Parslee-ai/car#264).
1116///
1117/// Armed from the moment the oneshot sender lands in
1118/// [`WsChannel::pending`]; disarmed only when a `ToolExecuteResponse` is
1119/// actually received. If it is still armed when it drops, the call was
1120/// cancelled and the guard both releases the `pending` entry and tells the
1121/// host to abort the in-flight child.
1122///
1123/// Why RAII rather than an explicit cleanup branch: `tokio::time::timeout`
1124/// **drops** the inner future on expiry, and the daemon's callback wait is
1125/// deliberately the *outermost* of three deadlines
1126/// (`TOOL_TIMEOUT_GRACE_MS` above the executor's per-action budget). So on
1127/// every real reap the whole dispatch future is dropped while parked on
1128/// `rx`, and any code written after the `await` — including the explicit
1129/// timeout arm — never runs. A `Drop` impl is the only cleanup that survives
1130/// being dropped.
1131struct PendingToolCall {
1132 channel: Arc<WsChannel>,
1133 request_id: String,
1134 action_id: String,
1135 reason: String,
1136 armed: bool,
1137}
1138
1139impl PendingToolCall {
1140 fn new(channel: Arc<WsChannel>, request_id: String, action_id: String, reason: String) -> Self {
1141 Self {
1142 channel,
1143 request_id,
1144 action_id,
1145 reason,
1146 armed: true,
1147 }
1148 }
1149
1150 /// A response was delivered — this call completed, so no cancel is owed.
1151 fn disarm(&mut self) {
1152 self.armed = false;
1153 }
1154}
1155
1156impl Drop for PendingToolCall {
1157 fn drop(&mut self) {
1158 if !self.armed {
1159 return;
1160 }
1161 // `pending` is a `tokio::sync::Mutex` and the write is async, but
1162 // `Drop` is sync — so the cleanup has to be a detached task. Outside
1163 // a runtime (a plain `drop` in sync code) there is nothing to spawn
1164 // onto; skip quietly rather than panicking out of a destructor.
1165 let Ok(handle) = tokio::runtime::Handle::try_current() else {
1166 return;
1167 };
1168 let channel = self.channel.clone();
1169 let request_id = std::mem::take(&mut self.request_id);
1170 let action_id = std::mem::take(&mut self.action_id);
1171 let reason = std::mem::take(&mut self.reason);
1172 handle.spawn(async move {
1173 // Only cancel if WE claimed the pending entry. The response
1174 // demux in `handler.rs` removes it on the success path, and
1175 // connection teardown clears the whole map — in either case the
1176 // host is not waiting on anything we should abort.
1177 let claimed = channel.pending.lock().await.remove(&request_id).is_some();
1178 channel.active_actions.lock().await.remove(&request_id);
1179 if claimed {
1180 write_tool_cancel(&channel, request_id, action_id, reason).await;
1181 }
1182 });
1183 }
1184}
1185
1186/// Tool executor that sends callbacks to the client over WebSocket.
1187pub struct WsToolExecutor {
1188 pub channel: Arc<WsChannel>,
1189 negotiated_capabilities: Arc<std::sync::RwLock<std::collections::BTreeSet<String>>>,
1190 halted: Arc<AtomicBool>,
1191}
1192
1193#[async_trait::async_trait]
1194impl ToolExecutor for WsToolExecutor {
1195 async fn execute(&self, tool: &str, params: &Value) -> Result<Value, String> {
1196 // Legacy callers that don't have a proposal-level Action.id
1197 // (e.g. internal `executor.execute` chains in tests) — emit an
1198 // empty action_id so the client-side handler can still see the
1199 // payload shape and decide whether to fail loudly.
1200 self.execute_with_action(tool, params, "", None).await
1201 }
1202
1203 async fn execute_with_action(
1204 &self,
1205 tool: &str,
1206 params: &Value,
1207 action_id: &str,
1208 timeout_ms: Option<u64>,
1209 ) -> Result<Value, String> {
1210 self.execute_with_action_in_session(tool, params, action_id, timeout_ms, None, 1)
1211 .await
1212 }
1213
1214 async fn execute_with_action_in_session(
1215 &self,
1216 tool: &str,
1217 params: &Value,
1218 action_id: &str,
1219 timeout_ms: Option<u64>,
1220 session_id: Option<&str>,
1221 attempt: u32,
1222 ) -> Result<Value, String> {
1223 self.execute_callback(
1224 tool,
1225 params,
1226 action_id,
1227 timeout_ms,
1228 session_id,
1229 attempt,
1230 &HashMap::new(),
1231 None,
1232 false,
1233 )
1234 .await
1235 .map(|execution| execution.output)
1236 .map_err(|failure| failure.message)
1237 }
1238
1239 async fn execute_with_action_state_in_session(
1240 &self,
1241 tool: &str,
1242 params: &Value,
1243 action_id: &str,
1244 timeout_ms: Option<u64>,
1245 session_id: Option<&str>,
1246 attempt: u32,
1247 expected_effects: &HashMap<String, Value>,
1248 return_schema: Option<&Value>,
1249 ) -> Result<car_engine::ToolExecution, String> {
1250 self.execute_callback(
1251 tool,
1252 params,
1253 action_id,
1254 timeout_ms,
1255 session_id,
1256 attempt,
1257 expected_effects,
1258 return_schema,
1259 self.callback_state_negotiated(),
1260 )
1261 .await
1262 .map_err(|failure| failure.message)
1263 }
1264
1265 async fn execute_classified(
1266 &self,
1267 tool: &str,
1268 params: &Value,
1269 action_id: &str,
1270 timeout_ms: Option<u64>,
1271 session_id: Option<&str>,
1272 attempt: u32,
1273 expected_effects: &HashMap<String, Value>,
1274 return_schema: Option<&Value>,
1275 ) -> Result<car_engine::ToolExecution, ToolFailure> {
1276 self.execute_callback(
1277 tool,
1278 params,
1279 action_id,
1280 timeout_ms,
1281 session_id,
1282 attempt,
1283 expected_effects,
1284 return_schema,
1285 self.callback_state_negotiated(),
1286 )
1287 .await
1288 }
1289}
1290
1291impl WsToolExecutor {
1292 pub fn new(
1293 channel: Arc<WsChannel>,
1294 negotiated_capabilities: Arc<std::sync::RwLock<std::collections::BTreeSet<String>>>,
1295 halted: Arc<AtomicBool>,
1296 ) -> Self {
1297 Self {
1298 channel,
1299 negotiated_capabilities,
1300 halted,
1301 }
1302 }
1303
1304 #[cfg(test)]
1305 fn legacy(channel: Arc<WsChannel>) -> Self {
1306 Self::new(
1307 channel,
1308 Arc::new(std::sync::RwLock::new(std::collections::BTreeSet::new())),
1309 Arc::new(AtomicBool::new(false)),
1310 )
1311 }
1312
1313 fn callback_state_negotiated(&self) -> bool {
1314 self.negotiated_capabilities
1315 .read()
1316 .map(|capabilities| capabilities.contains(car_proto::TOOLS_CALLBACK_STATE_CAPABILITY))
1317 .unwrap_or(false)
1318 }
1319
1320 #[allow(clippy::too_many_arguments)]
1321 async fn execute_callback(
1322 &self,
1323 tool: &str,
1324 params: &Value,
1325 action_id: &str,
1326 timeout_ms: Option<u64>,
1327 session_id: Option<&str>,
1328 attempt: u32,
1329 expected_effects: &HashMap<String, Value>,
1330 return_schema: Option<&Value>,
1331 callback_state_negotiated: bool,
1332 ) -> Result<car_engine::ToolExecution, ToolFailure> {
1333 use futures::SinkExt;
1334
1335 // The JSON-RPC request id is the daemon's callback-routing key
1336 // (used by the pending-response map below). The `action_id`
1337 // FIELD on the payload is the originating proposal Action.id
1338 // surfaced to the host so process-wide handlers can route
1339 // concurrent callbacks back to per-call dispatchers
1340 // (Parslee-ai/car-releases#43 follow-up). They serve different
1341 // purposes and must stay distinct: routing id is daemon-side,
1342 // action id is host-side.
1343 let request_id = self.channel.next_request_id();
1344
1345 let callback = ToolExecuteRequest {
1346 action_id: action_id.to_string(),
1347 tool: tool.to_string(),
1348 parameters: params.clone(),
1349 // Surface the action's budget to the host so its own tool runner
1350 // can bound the work too (Parslee-ai/car#259) — was always None.
1351 timeout_ms,
1352 // The engine's real retry counter, 1-based. This was hardcoded to
1353 // `1`, which made the field a constant on the wire: a host building
1354 // the retry-disambiguating join the field exists for got a key that
1355 // never varied, and would only find out under the concurrent-retry
1356 // conditions the join is for (Parslee-ai/car#928).
1357 attempt,
1358 // Surface the routing id so the host can key a per-call abort
1359 // registry on it; a reap emits `tools.cancel` carrying the same id
1360 // (Parslee-ai/car#264).
1361 request_id: request_id.clone(),
1362 // Server-stamped correlation. The executor has always passed this
1363 // in; the parameter was `_session_id` and the value was discarded,
1364 // so every host that needed attribution had to rebuild it from
1365 // client-side convention (Parslee-ai/car#904).
1366 session_id: session_id.map(str::to_string),
1367 };
1368
1369 // Create a oneshot channel for the response
1370 let (tx, rx) = oneshot::channel();
1371 self.channel
1372 .pending
1373 .lock()
1374 .await
1375 .insert(request_id.clone(), tx);
1376 self.channel
1377 .active_actions
1378 .lock()
1379 .await
1380 .insert(request_id.clone(), action_id.to_string());
1381
1382 // From here on the call is in flight. The guard makes cleanup +
1383 // host-abort survive ANY cancellation of this future — the
1384 // executor's per-action `timeout_ms` deadline (the common case, it
1385 // fires `TOOL_TIMEOUT_GRACE_MS` before our own wait below), the
1386 // server's per-method deadline in `handler.rs`, or the callback wait
1387 // here — not just the one arm we can write code in
1388 // (Parslee-ai/car#264).
1389 let mut pending_call = PendingToolCall::new(
1390 self.channel.clone(),
1391 request_id.clone(),
1392 action_id.to_string(),
1393 format!("tool '{tool}' call cancelled before completion (request {request_id})"),
1394 );
1395
1396 // Send the callback to the client as a JSON-RPC request
1397 let rpc_request = serde_json::json!({
1398 "jsonrpc": "2.0",
1399 "method": "tools.execute",
1400 "params": callback,
1401 "id": request_id,
1402 });
1403
1404 let msg = Message::Text(
1405 serde_json::to_string(&rpc_request)
1406 .map_err(|e| e.to_string())?
1407 .into(),
1408 );
1409 self.channel
1410 .write
1411 .lock()
1412 .await
1413 .send(msg)
1414 .await
1415 .map_err(|e| format!("failed to send tool callback: {}", e))?;
1416
1417 // Wait for the client to respond, bounded by the action's budget
1418 // (or the configurable default) — NOT a hardcoded 60s. This is the
1419 // ceiling that previously ignored `Action.timeout_ms` and reaped
1420 // legitimately-long tool calls (Parslee-ai/car#259).
1421 let wait = tool_callback_timeout(timeout_ms);
1422 let response = match tokio::time::timeout(wait, rx).await {
1423 Ok(inner) => inner.map_err(|_| format!("tool '{}' callback channel closed", tool))?,
1424 Err(_) => {
1425 // Reaped by our own callback wait. Sharpen the guard's reason
1426 // to name the timeout, then let it do the cleanup + host abort
1427 // on the way out — the same path a dropped future takes
1428 // (Parslee-ai/car#264).
1429 let reason = format!("tool '{}' callback timed out ({}s)", tool, wait.as_secs());
1430 pending_call.reason = reason.clone();
1431 return Err(ToolFailure::ordinary(reason));
1432 }
1433 };
1434
1435 // A response was delivered (error field set or not): the call is
1436 // finished, so no cancel is owed to the host.
1437 self.channel.active_actions.lock().await.remove(&request_id);
1438 pending_call.disarm();
1439
1440 if let Some(err) = response.error {
1441 let failure = match response.terminal {
1442 false => ToolFailure::ordinary(err),
1443 true => {
1444 // Latch at the callback boundary, before returning to the
1445 // engine. Handler-level result inspection repeats this as
1446 // defense in depth, but this write closes the cancellation
1447 // gap between typed evidence arriving and execute() ending.
1448 self.halted.store(true, Ordering::Release);
1449 ToolFailure::terminal(err)
1450 }
1451 };
1452 return Err(failure);
1453 }
1454 decode_callback_execution(
1455 tool,
1456 action_id,
1457 response.output.unwrap_or(Value::Null),
1458 expected_effects,
1459 return_schema,
1460 callback_state_negotiated,
1461 )
1462 .map_err(ToolFailure::ordinary)
1463 }
1464}
1465
1466fn decode_callback_execution(
1467 tool: &str,
1468 action_id: &str,
1469 result: Value,
1470 expected_effects: &HashMap<String, Value>,
1471 return_schema: Option<&Value>,
1472 callback_state_negotiated: bool,
1473) -> Result<car_engine::ToolExecution, String> {
1474 if !callback_state_negotiated {
1475 return Ok(car_engine::ToolExecution::output_only(result));
1476 }
1477 let envelope = result.as_object().filter(|object| {
1478 object.len() == 2 && object.contains_key("output") && object.contains_key("state_changes")
1479 });
1480
1481 if expected_effects.is_empty() && envelope.is_none() {
1482 return Ok(car_engine::ToolExecution::output_only(result));
1483 }
1484 let envelope = envelope.ok_or_else(|| {
1485 format!(
1486 "tool '{tool}' callback for action '{action_id}' must return exact envelope {{output,state_changes}}"
1487 )
1488 })?;
1489 let output = envelope
1490 .get("output")
1491 .cloned()
1492 .expect("exact envelope contains output");
1493 let state_changes: HashMap<String, Value> = serde_json::from_value(
1494 envelope
1495 .get("state_changes")
1496 .cloned()
1497 .expect("exact envelope contains state_changes"),
1498 )
1499 .map_err(|_| {
1500 format!("tool '{tool}' callback for action '{action_id}' state_changes must be an object")
1501 })?;
1502 let expected_keys: std::collections::BTreeSet<&str> =
1503 expected_effects.keys().map(String::as_str).collect();
1504 let actual_keys: std::collections::BTreeSet<&str> =
1505 state_changes.keys().map(String::as_str).collect();
1506 if expected_keys != actual_keys {
1507 return Err(format!(
1508 "tool '{tool}' callback state_changes keys do not match action '{action_id}': expected {:?}, got {:?}",
1509 expected_keys, actual_keys
1510 ));
1511 }
1512 if let Some(schema) = return_schema {
1513 car_engine::validate_tool_output(tool, schema, &output)?;
1514 }
1515 let changes_value = serde_json::to_value(&state_changes).map_err(|error| {
1516 format!("tool '{tool}' callback state_changes serialization failed: {error}")
1517 })?;
1518 car_inference::catalog_identity::canonical_json(&changes_value).map_err(|error| {
1519 format!("tool '{tool}' callback state_changes failed JCS/I-JSON validation: {error}")
1520 })?;
1521 Ok(car_engine::ToolExecution {
1522 output,
1523 state_changes,
1524 })
1525}
1526
1527#[cfg(test)]
1528mod tool_cancel_tests {
1529 use super::*;
1530 use serde_json::json;
1531 use std::time::Duration;
1532
1533 /// Every `Message::Text` the executor wrote, parsed as JSON.
1534 fn written(frames: &std::sync::Arc<std::sync::Mutex<Vec<String>>>) -> Vec<Value> {
1535 frames
1536 .lock()
1537 .unwrap()
1538 .iter()
1539 .map(|t| serde_json::from_str::<Value>(t).expect("frame is JSON"))
1540 .collect()
1541 }
1542
1543 fn frame_with_method<'a>(frames: &'a [Value], method: &str) -> Option<&'a Value> {
1544 frames.iter().find(|f| f["method"] == json!(method))
1545 }
1546
1547 /// Let detached cleanup tasks (the `PendingToolCall` drop spawn) run.
1548 async fn settle() {
1549 for _ in 0..8 {
1550 tokio::task::yield_now().await;
1551 }
1552 tokio::time::sleep(Duration::from_millis(100)).await;
1553 for _ in 0..8 {
1554 tokio::task::yield_now().await;
1555 }
1556 }
1557
1558 /// The daemon's own callback wait expiring still cancels — the arm that
1559 /// used to hold the inline cleanup now delegates to the drop guard, and
1560 /// the observable behaviour (Err text, cancel frame, drained `pending`)
1561 /// must be unchanged.
1562 #[tokio::test(start_paused = true)]
1563 async fn callback_wait_expiry_cancels() {
1564 let (channel, frames) = WsChannel::test_capture();
1565 let channel = Arc::new(channel);
1566 let executor = WsToolExecutor::legacy(channel.clone());
1567
1568 // 1ms budget → a ~5s callback wait; paused time makes that instant.
1569 let err = executor
1570 .execute_with_action_in_session("drive_cli", &json!({}), "a0", Some(1), None, 1)
1571 .await
1572 .expect_err("the callback wait must expire");
1573 assert!(
1574 err.contains("tool 'drive_cli' callback timed out"),
1575 "unexpected error text: {err}"
1576 );
1577
1578 settle().await;
1579
1580 let frames = written(&frames);
1581 let execute = frame_with_method(&frames, "tools.execute").expect("tools.execute was sent");
1582 let cancel = frame_with_method(&frames, "tools.cancel")
1583 .expect("a tools.cancel notification must reach the wire");
1584 assert_eq!(
1585 cancel["params"]["request_id"], execute["params"]["request_id"],
1586 "cancel must correlate to the originating request_id"
1587 );
1588 assert_eq!(cancel["params"]["action_id"], json!("a0"));
1589 assert!(
1590 cancel["id"].is_null(),
1591 "cancel is a notification, not a request"
1592 );
1593 assert!(
1594 channel.pending.lock().await.is_empty(),
1595 "the pending entry must be released"
1596 );
1597 }
1598
1599 /// Parslee-ai/car#904 — the execution session must reach the wire.
1600 ///
1601 /// The executor has always passed a session id in; the parameter was named
1602 /// `_session_id` and the value was dropped on the floor, so every host that
1603 /// needed to know which mission a callback belonged to had to rebuild it
1604 /// from client-side convention. Asserted on the actual `tools.execute`
1605 /// frame rather than on the struct, because the struct field existing is
1606 /// not the property that was missing.
1607 #[tokio::test(start_paused = true)]
1608 async fn tools_execute_carries_the_execution_session() {
1609 let (channel, frames) = WsChannel::test_capture();
1610 let channel = Arc::new(channel);
1611 let executor = WsToolExecutor::legacy(channel.clone());
1612
1613 let _ = tokio::time::timeout(
1614 Duration::from_millis(50),
1615 executor.execute_with_action_in_session(
1616 "search",
1617 &json!({}),
1618 "a0",
1619 Some(10_000),
1620 Some("sess-42"),
1621 1,
1622 ),
1623 )
1624 .await;
1625 settle().await;
1626
1627 let frames = written(&frames);
1628 let execute = frame_with_method(&frames, "tools.execute").expect("tools.execute was sent");
1629 assert_eq!(
1630 execute["params"]["session_id"],
1631 json!("sess-42"),
1632 "the daemon must stamp the session it already knows, not leave \
1633 attribution to the host"
1634 );
1635 }
1636
1637 /// Parslee-ai/car#928 — the retry counter on the wire must be the engine's
1638 /// real one.
1639 ///
1640 /// It was hardcoded to `1` here, so the field a host would build its
1641 /// retry-disambiguating join on never varied. That is worse than the field
1642 /// being absent: the join looks correct and degenerates silently, and only
1643 /// under the concurrent-retry conditions it exists to handle.
1644 #[tokio::test(start_paused = true)]
1645 async fn tools_execute_carries_the_real_attempt_number() {
1646 for attempt in [1u32, 2, 7] {
1647 let (channel, frames) = WsChannel::test_capture();
1648 let channel = Arc::new(channel);
1649 let executor = WsToolExecutor::legacy(channel.clone());
1650
1651 let _ = tokio::time::timeout(
1652 Duration::from_millis(50),
1653 executor.execute_with_action_in_session(
1654 "search",
1655 &json!({}),
1656 "a0",
1657 Some(10_000),
1658 Some("sess-1"),
1659 attempt,
1660 ),
1661 )
1662 .await;
1663 settle().await;
1664
1665 let frames = written(&frames);
1666 let execute =
1667 frame_with_method(&frames, "tools.execute").expect("tools.execute was sent");
1668 assert_eq!(
1669 execute["params"]["attempt"],
1670 json!(attempt),
1671 "the payload must carry the engine's attempt, not a constant"
1672 );
1673 }
1674 }
1675
1676 /// The other half: a caller with no session must omit the key rather than
1677 /// send a null. `execute()` and in-process executors legitimately have
1678 /// none, and a null would make every such payload differ from the pre-#904
1679 /// shape for no gain.
1680 #[tokio::test(start_paused = true)]
1681 async fn a_sessionless_call_omits_the_session_key() {
1682 let (channel, frames) = WsChannel::test_capture();
1683 let channel = Arc::new(channel);
1684 let executor = WsToolExecutor::legacy(channel.clone());
1685
1686 let _ = tokio::time::timeout(
1687 Duration::from_millis(50),
1688 executor.execute_with_action_in_session(
1689 "search",
1690 &json!({}),
1691 "a0",
1692 Some(10_000),
1693 None,
1694 1,
1695 ),
1696 )
1697 .await;
1698 settle().await;
1699
1700 let frames = written(&frames);
1701 let execute = frame_with_method(&frames, "tools.execute").expect("tools.execute was sent");
1702 assert!(
1703 execute["params"].get("session_id").is_none(),
1704 "a sessionless call must omit the key entirely, got: {}",
1705 execute["params"]
1706 );
1707 }
1708
1709 /// REGRESSION (Parslee-ai/car#264): the executor's per-action deadline
1710 /// fires `TOOL_TIMEOUT_GRACE_MS` BEFORE the daemon's callback wait, so it
1711 /// drops this whole future while it is parked on `rx`. Before the drop
1712 /// guard, nothing ran on that path: no `tools.cancel` frame was ever
1713 /// written and the `pending` entry leaked until connection teardown,
1714 /// orphaning the host's in-flight child.
1715 #[tokio::test(start_paused = true)]
1716 async fn engine_deadline_drop_cancels() {
1717 let (channel, frames) = WsChannel::test_capture();
1718 let channel = Arc::new(channel);
1719 let executor = WsToolExecutor::legacy(channel.clone());
1720
1721 // Outer deadline (standing in for the engine's per-action timeout)
1722 // at 50ms; the callback wait would be 10s + 5s grace.
1723 let outcome = tokio::time::timeout(
1724 Duration::from_millis(50),
1725 executor.execute_with_action_in_session(
1726 "drive_cli",
1727 &json!({}),
1728 "a0",
1729 Some(10_000),
1730 None,
1731 1,
1732 ),
1733 )
1734 .await;
1735 assert!(
1736 outcome.is_err(),
1737 "the outer deadline must fire first and drop the dispatch future"
1738 );
1739
1740 settle().await;
1741
1742 let frames = written(&frames);
1743 let execute = frame_with_method(&frames, "tools.execute").expect("tools.execute was sent");
1744 let cancel = frame_with_method(&frames, "tools.cancel").expect(
1745 "dropping the dispatch future must still emit tools.cancel so the host child isn't orphaned",
1746 );
1747 assert_eq!(
1748 cancel["params"]["request_id"], execute["params"]["request_id"],
1749 "cancel must correlate to the originating request_id"
1750 );
1751 assert!(
1752 channel.pending.lock().await.is_empty(),
1753 "the pending entry must not leak"
1754 );
1755 }
1756
1757 /// The happy path must stay silent: a delivered response disarms the
1758 /// guard, so no host abort is emitted for a call that completed.
1759 #[tokio::test]
1760 async fn success_emits_no_cancel() {
1761 let (channel, frames) = WsChannel::test_capture();
1762 let channel = Arc::new(channel);
1763 let executor = WsToolExecutor::legacy(channel.clone());
1764
1765 let responder = channel.clone();
1766 let host = tokio::spawn(async move {
1767 loop {
1768 let claimed = {
1769 let mut pending = responder.pending.lock().await;
1770 let key = pending.keys().next().cloned();
1771 key.and_then(|k| pending.remove(&k).map(|tx| (k, tx)))
1772 };
1773 if let Some((request_id, tx)) = claimed {
1774 let _ = tx.send(ToolExecuteResponse {
1775 action_id: request_id,
1776 output: Some(json!({"ok": true})),
1777 error: None,
1778 terminal: false,
1779 });
1780 return;
1781 }
1782 tokio::task::yield_now().await;
1783 }
1784 });
1785
1786 let out = executor
1787 .execute_with_action_in_session("drive_cli", &json!({}), "a0", Some(10_000), None, 1)
1788 .await
1789 .expect("the host responded, so the call succeeds");
1790 assert_eq!(out, json!({"ok": true}));
1791 host.await.expect("responder task finishes");
1792
1793 settle().await;
1794
1795 let frames = written(&frames);
1796 assert!(
1797 frame_with_method(&frames, "tools.execute").is_some(),
1798 "tools.execute was sent"
1799 );
1800 assert!(
1801 frame_with_method(&frames, "tools.cancel").is_none(),
1802 "a completed call must not emit tools.cancel"
1803 );
1804 assert!(channel.pending.lock().await.is_empty());
1805 }
1806}
1807
1808/// The bare commodity-tool names owned by the engine's bound substrate.
1809/// When a session binds a non-Local substrate (`session.bindSubstrate`),
1810/// these names must reach the engine's substrate-after-fall_through path
1811/// (`executor.rs`), NOT the WS tool callback. Everything else — connector
1812/// `mcp_{slug}_*` routes and host-executed client tools — is untouched.
1813///
1814/// `calculate` is deliberately excluded: it is a pure, environment-free
1815/// tool that the engine already handles in-process via `agent_basics`, so
1816/// it never needs to be re-routed to a substrate.
1817pub(crate) const SUBSTRATE_OWNED_TOOLS: &[&str] = &[
1818 "read_file",
1819 "write_file",
1820 "edit_file",
1821 "list_dir",
1822 "find_files",
1823 "grep_files",
1824];
1825
1826/// Executor wrapper used **only** by substrate-bound sessions
1827/// (`session.bindSubstrate`). It is the low-blast-radius "option (a)" from
1828/// `docs/execution-substrate.md` §3: keep the existing
1829/// `share_with_fallback(ws_executor)` composition intact for connector
1830/// routes and host tool callbacks, but for the bare substrate-owned
1831/// built-in names ([`SUBSTRATE_OWNED_TOOLS`]) return an `"unknown tool:
1832/// <name>"` error so the engine's dispatch (`executor.rs`) takes the
1833/// `fall_through` branch and routes those names to the runtime's bound
1834/// substrate (the connector/VM [`car_engine::McpSubstrate`]) instead of the WS client.
1835///
1836/// This preserves every existing contract — GUI/connector/coder sessions
1837/// never construct one of these — and is what makes a connector-driven
1838/// session *coherent*: its file/exec built-ins land on the same machine as
1839/// its `mcp_{slug}_*` tools, not split between the WS client and a remote
1840/// MCP server.
1841pub struct SubstrateShadowExecutor {
1842 inner: Arc<dyn ToolExecutor>,
1843}
1844
1845impl SubstrateShadowExecutor {
1846 pub fn new(inner: Arc<dyn ToolExecutor>) -> Self {
1847 Self { inner }
1848 }
1849}
1850
1851#[async_trait::async_trait]
1852impl ToolExecutor for SubstrateShadowExecutor {
1853 async fn execute(&self, tool: &str, params: &Value) -> Result<Value, String> {
1854 self.execute_with_action(tool, params, "", None).await
1855 }
1856
1857 async fn execute_with_action(
1858 &self,
1859 tool: &str,
1860 params: &Value,
1861 action_id: &str,
1862 timeout_ms: Option<u64>,
1863 ) -> Result<Value, String> {
1864 self.execute_with_action_in_session(tool, params, action_id, timeout_ms, None, 1)
1865 .await
1866 }
1867
1868 async fn execute_with_action_in_session(
1869 &self,
1870 tool: &str,
1871 params: &Value,
1872 action_id: &str,
1873 timeout_ms: Option<u64>,
1874 session_id: Option<&str>,
1875 attempt: u32,
1876 ) -> Result<Value, String> {
1877 if SUBSTRATE_OWNED_TOOLS.contains(&tool) {
1878 // Signal the engine to fall through to the bound substrate.
1879 // The exact `"unknown tool"` prefix is the contract the engine
1880 // dispatch matches on (`executor.rs`).
1881 return Err(format!("unknown tool: {tool}"));
1882 }
1883 self.inner
1884 .execute_with_action_in_session(
1885 tool, params, action_id, timeout_ms, session_id, attempt,
1886 )
1887 .await
1888 }
1889
1890 async fn execute_with_action_state_in_session(
1891 &self,
1892 tool: &str,
1893 params: &Value,
1894 action_id: &str,
1895 timeout_ms: Option<u64>,
1896 session_id: Option<&str>,
1897 attempt: u32,
1898 expected_effects: &HashMap<String, Value>,
1899 return_schema: Option<&Value>,
1900 ) -> Result<car_engine::ToolExecution, String> {
1901 if SUBSTRATE_OWNED_TOOLS.contains(&tool) {
1902 return Err(format!("unknown tool: {tool}"));
1903 }
1904 self.inner
1905 .execute_with_action_state_in_session(
1906 tool,
1907 params,
1908 action_id,
1909 timeout_ms,
1910 session_id,
1911 attempt,
1912 expected_effects,
1913 return_schema,
1914 )
1915 .await
1916 }
1917
1918 async fn execute_classified(
1919 &self,
1920 tool: &str,
1921 params: &Value,
1922 action_id: &str,
1923 timeout_ms: Option<u64>,
1924 session_id: Option<&str>,
1925 attempt: u32,
1926 expected_effects: &HashMap<String, Value>,
1927 return_schema: Option<&Value>,
1928 ) -> Result<car_engine::ToolExecution, ToolFailure> {
1929 if SUBSTRATE_OWNED_TOOLS.contains(&tool) {
1930 return Err(ToolFailure::ordinary(format!("unknown tool: {tool}")));
1931 }
1932 self.inner
1933 .execute_classified(
1934 tool,
1935 params,
1936 action_id,
1937 timeout_ms,
1938 session_id,
1939 attempt,
1940 expected_effects,
1941 return_schema,
1942 )
1943 .await
1944 }
1945}
1946
1947/// Voice event sink that forwards events to a specific WebSocket client
1948/// as `voice.event` JSON-RPC notifications.
1949///
1950/// Each `voice.transcribe_stream.start` call constructs one of these
1951/// bound to the originating client's [`WsChannel`], so a client only
1952/// receives events for sessions it started.
1953pub struct WsVoiceEventSink {
1954 pub channel: Arc<WsChannel>,
1955}
1956
1957impl car_voice::VoiceEventSink for WsVoiceEventSink {
1958 fn send(&self, session_id: &str, event_json: String) {
1959 use futures::SinkExt;
1960 let channel = self.channel.clone();
1961 let session_id = session_id.to_string();
1962 tokio::spawn(async move {
1963 let payload: Value = serde_json::from_str(&event_json)
1964 .unwrap_or_else(|_| Value::String(event_json.clone()));
1965 let notification = serde_json::json!({
1966 "jsonrpc": "2.0",
1967 "method": "voice.event",
1968 "params": {
1969 "session_id": session_id,
1970 "event": payload,
1971 },
1972 });
1973 let Ok(text) = serde_json::to_string(¬ification) else {
1974 return;
1975 };
1976 let _ = channel
1977 .write
1978 .lock()
1979 .await
1980 .send(Message::Text(text.into()))
1981 .await;
1982 });
1983 }
1984
1985 fn send_binary(&self, frame: Vec<u8>) {
1986 use futures::SinkExt;
1987 let channel = self.channel.clone();
1988 tokio::spawn(async move {
1989 let _ = channel
1990 .write
1991 .lock()
1992 .await
1993 .send(Message::Binary(frame.into()))
1994 .await;
1995 });
1996 }
1997}
1998
1999/// Per-meeting fanout sink that ingests transcript text into a
2000/// session-scoped memgine using the `Arc<tokio::sync::Mutex<...>>`
2001/// wrapper, then forwards every event upstream untouched.
2002///
2003/// Lives here (not in `car-ffi-common`) because the engine handle uses
2004/// `tokio::sync::Mutex` per the "one-wrapper rule" — the FFI-common
2005/// `MeetingMemgineFanout` still uses `std::sync::Mutex` for the NAPI/
2006/// PyO3 bindings, which keep their sync wrappers. Each binding owns the
2007/// fanout that matches its lock primitive; the parsing/formatting logic
2008/// itself is shared via [`car_meeting::extract_transcript_for_ingest`].
2009///
2010/// `send` is called from the voice drain task and must be non-blocking,
2011/// so the lock acquisition is shipped to a `tokio::spawn`. Transcript
2012/// events are independent so reordering across spawned tasks is fine.
2013pub struct WsMemgineIngestSink {
2014 pub meeting_id: String,
2015 pub engine: Arc<Mutex<car_memgine::MemgineEngine>>,
2016 pub upstream: Arc<dyn car_voice::VoiceEventSink>,
2017}
2018
2019impl car_voice::VoiceEventSink for WsMemgineIngestSink {
2020 fn send(&self, voice_session_id: &str, event_json: String) {
2021 if let Ok(value) = serde_json::from_str::<Value>(&event_json) {
2022 if let Some((speaker, text)) = car_meeting::extract_transcript_for_ingest(
2023 &value,
2024 &self.meeting_id,
2025 voice_session_id,
2026 ) {
2027 let engine = self.engine.clone();
2028 tokio::spawn(async move {
2029 let mut guard = engine.lock().await;
2030 guard.ingest_conversation(&speaker, &text, chrono::Utc::now());
2031 });
2032 }
2033 }
2034 self.upstream.send(voice_session_id, event_json);
2035 }
2036}
2037
2038/// The last assistant turn a chat session produced, remembered so the *next*
2039/// user turn can be classified against it (the conversation-outcome signal).
2040/// The `trace_id`/`model_id` are carried straight from the `InferenceResult`
2041/// that produced the turn — never reconstructed from message order — so credit
2042/// always lands on the exact model/trace that generated the judged turn.
2043#[derive(Debug, Clone)]
2044pub struct LastChatTurn {
2045 /// The user request this assistant turn answered (for restatement similarity).
2046 pub user_text: String,
2047 pub assistant_text: String,
2048 pub trace_id: String,
2049 pub model_id: String,
2050}
2051
2052/// Per-client session.
2053pub struct ClientSession {
2054 pub client_id: String,
2055 pub runtime: Arc<Runtime>,
2056 pub channel: Arc<WsChannel>,
2057 pub host: Arc<crate::host::HostState>,
2058 /// Memgine handle. Wrapped in `tokio::sync::Mutex` so dispatcher
2059 /// handlers can hold the lock across `.await` points without
2060 /// risking poisoning. Migrated from `std::sync::Mutex` in the
2061 /// car-server-core extraction (U1) per the "one-wrapper rule".
2062 pub memgine: Arc<Mutex<car_memgine::MemgineEngine>>,
2063 /// Lazy browser session — first `browser.run` call launches Chromium,
2064 /// subsequent calls reuse it so element IDs resolve across invocations
2065 /// within the same WebSocket connection.
2066 pub browser: car_ffi_common::browser::BrowserSessionSlot,
2067 /// Per-connection auth state. Starts `false`; flips to `true`
2068 /// after a successful `session.auth` handshake. Always considered
2069 /// authenticated when `ServerState::auth_token` is unset (auth
2070 /// disabled). Closes Parslee-ai/car-releases#32.
2071 pub authenticated: Arc<std::sync::atomic::AtomicBool>,
2072 /// Negotiated daemon JSON-RPC protocol for this WebSocket connection.
2073 /// Starts at `0` (unnegotiated) and is set to
2074 /// `car_proto::PROTOCOL_VERSION` only after an exact-version
2075 /// `server.handshake`. The state is connection-scoped by construction, so
2076 /// a reconnect always has to negotiate again before using handshake-gated
2077 /// host/auth methods.
2078 pub negotiated_protocol_version: std::sync::atomic::AtomicU32,
2079 /// Capability names negotiated by this connection's authenticated
2080 /// `server.handshake`. Connection-scoped for the same reason as the wire
2081 /// version: a reconnect cannot inherit another socket's proof.
2082 pub negotiated_capabilities: Arc<std::sync::RwLock<std::collections::BTreeSet<String>>>,
2083 /// Bounded lifecycle state for inferences started on this exact socket.
2084 /// Keeping it per-session prevents an authenticated peer from probing IDs
2085 /// owned by another connection.
2086 pub inference_control: Arc<crate::inference_control::InferenceRegistry>,
2087 /// Host-management role (Parslee-ai/car#254). Starts `false`; flips
2088 /// to `true` only when the connection presents the per-launch host
2089 /// token via `session.auth { host_token }` (validated against
2090 /// `ServerState::host_token`). `authorize_run_access` requires this
2091 /// for cross-agent run-trace reads — being merely `host.subscribe`d
2092 /// is no longer sufficient, which is what closes the self-elevation
2093 /// hole. Cleared implicitly when the connection drops.
2094 pub is_host: std::sync::atomic::AtomicBool,
2095 /// Bound agent identity (#169). `Some(id)` once a lifecycle-agent
2096 /// child has called `session.auth { token, agent_id }` and the
2097 /// supervisor confirmed `agent_id` is supervised + token matches.
2098 /// Used by `agents.list` to surface which managed agents have
2099 /// actually attached vs. just being marked `Running` at the
2100 /// process level. Cleared at disconnect by `remove_session`.
2101 pub agent_id: Arc<tokio::sync::Mutex<Option<String>>>,
2102 /// Optional daemon-method allowlist bound by successful supervised-agent
2103 /// authentication. `None` is the backward-compatible unrestricted state;
2104 /// `Some(empty)` denies every application method. The dispatcher reads it
2105 /// once before any method-specific handler or notification interceptor.
2106 pub agent_method_allowlist: Arc<std::sync::RwLock<Option<std::collections::BTreeSet<String>>>>,
2107 /// Canonical schema digests for the narrow reverse-callback tools this
2108 /// client registered. Server-owned registry entries never land here.
2109 pub callback_tool_schema_digests:
2110 Arc<tokio::sync::RwLock<std::collections::HashMap<String, String>>>,
2111 /// Bound memory namespace (`session.auth { memory_namespace }`, #79/#80).
2112 ///
2113 /// The namespace identity has to survive the handshake, not just the
2114 /// lookup that binds `bound_memgine`. Without it the daemon knows *which
2115 /// graph* a session is using but not *what to call it*, so nothing can
2116 /// write that graph back to
2117 /// `~/.car/memory/memory-namespaces/<encoded-ns>.json` — which is
2118 /// why namespace memory was durable only until the next restart
2119 /// (car-releases#82). Paired with `bound_memgine` exactly as `agent_id` is.
2120 pub memory_namespace: tokio::sync::Mutex<Option<String>>,
2121 /// Bound persistent memgine (#170). `Some` after `session.auth`
2122 /// successfully attaches the connection to a daemon-owned
2123 /// per-agent memgine (paired with `agent_id`). Memory handlers
2124 /// route through [`ClientSession::effective_memgine`] which
2125 /// returns this when set, falling back to the ephemeral
2126 /// `memgine` field for browser/host/CLI connections.
2127 pub bound_memgine: tokio::sync::Mutex<Option<Arc<Mutex<car_memgine::MemgineEngine>>>>,
2128 /// The run currently bracketed on this connection (agent run
2129 /// tracing, U1). Set by `runs.start` **before** that handler
2130 /// responds, so the per-turn recorder (U2) always reads the
2131 /// `run_id` the bracket established — no race across the
2132 /// concurrently-spawned dispatch tasks (KTD3). Cleared by
2133 /// `runs.complete`. On disconnect with a still-set current run and
2134 /// no recorded terminal, the daemon marks it `Incomplete` (R5).
2135 pub current_run_id: tokio::sync::Mutex<Option<String>>,
2136 /// Serializes the authenticated run bracket on one WebSocket: start,
2137 /// proposal lifecycle, complete, and disconnect terminalization cannot
2138 /// interleave and produce ambiguous journal ordering.
2139 pub run_lifecycle_guard: Arc<tokio::sync::Mutex<()>>,
2140 /// Per-session permission-tier gate (survey §3.4.3/§5.2.5). Backs the
2141 /// `permission.*` JSON-RPC methods: the session holds the granted standing
2142 /// tier + risk classifier. Defaults to `SandboxEdit`, the tier
2143 /// `docs/websocket-protocol.md` publishes as the session default.
2144 ///
2145 /// `Arc`, not a bare lock, because this is also the gate
2146 /// [`crate::permission_gate::PermissionAdmissionGate`] enforces at
2147 /// proposal admission (Parslee-ai/car#890). ONE gate object behind both:
2148 /// a second instance would let the advisory `permission.evaluate` and the
2149 /// enforcement point disagree about the same action, which is precisely
2150 /// the bug class the admission gate was added to close.
2151 ///
2152 /// NOTE (kernel review C1): approval *records* do NOT live here. The
2153 /// gate's embedded ledger is deliberately unused in the daemon — every
2154 /// fingerprint-keyed HITL read/write goes through the SHARED
2155 /// [`ServerState::approval_ledger`] (journal-backed), so an approval
2156 /// recorded on one connection is visible to every other and survives
2157 /// restart. Tier state stays per-session; the approval store is
2158 /// daemon-wide.
2159 pub permission_gate: Arc<tokio::sync::RwLock<car_policy::PermissionGate>>,
2160 /// Connection-local fail-stop latch. A typed terminal tool callback sets
2161 /// it after its proposal has aborted and rolled back; the admission gate
2162 /// sharing this exact atomic rejects every later proposal on this socket.
2163 /// It is deliberately not durable and is discarded on reconnect.
2164 pub halted: Arc<AtomicBool>,
2165 /// Non-overlap guard for `evolution.run` on this session (kernel review
2166 /// S3): the dispatcher spawns requests concurrently even on one
2167 /// connection, and two interleaved cycles would double-dispatch the
2168 /// evolution mechanisms. A second `evolution.run` while one is in flight
2169 /// errs instead of overlapping.
2170 pub evolution_guard: crate::evolution::CycleGuard,
2171 /// Last assistant turn produced on this session's chat path, so the next
2172 /// user turn can be scored as a conversation outcome (Part B). `None` until
2173 /// the first chat reply; only set for declared-chat inference.
2174 pub last_chat_turn: tokio::sync::Mutex<Option<LastChatTurn>>,
2175 /// In-flight chat-infer count. Concurrent infers on one session have no
2176 /// well-defined turn adjacency (the dispatcher spawns a task per frame), so
2177 /// the outcome scorer only records a turn that began while the session was
2178 /// idle — otherwise it would score one infer's trace against an unrelated
2179 /// concurrent user turn (an active mislabel of routing stats). neo #1.
2180 pub chat_inflight: std::sync::atomic::AtomicUsize,
2181 /// Tenant identity bound at `session.auth { tenant_id }` (linus
2182 /// review C-4). Once bound, every tenant-scoped handler uses THIS
2183 /// identity: a per-request `tenant_id` param may restate it but a
2184 /// mismatch is rejected — an authenticated connection cannot hop
2185 /// into another tenant's namespace by editing request params.
2186 /// `None` (unbound) preserves the legacy per-request behavior.
2187 pub tenant: tokio::sync::Mutex<Option<String>>,
2188 /// Whether this connection has a live `tools.stream.event` forwarder
2189 /// task (C2). Set by `tools.stream.subscribe`; a concurrent second
2190 /// subscribe is a no-op instead of spawning a duplicate forwarder
2191 /// (which would double every notification). `Arc` so the forwarder
2192 /// task can RESET it on exit (write timeout on a stalled-but-
2193 /// recovering socket, broadcast closed) — a later re-subscribe then
2194 /// spawns a fresh forwarder rather than silently no-op'ing (Q2).
2195 pub tool_stream_subscribed: std::sync::Arc<std::sync::atomic::AtomicBool>,
2196}
2197
2198impl ClientSession {
2199 /// Returns the memgine handle the memory.* handlers should use:
2200 /// the bound per-agent memgine when this session attached via
2201 /// `session.auth { agent_id }` (#169 + #170), otherwise the
2202 /// ephemeral per-WS memgine. Cheap (one async lock + Arc clone).
2203 pub async fn effective_memgine(&self) -> Arc<Mutex<car_memgine::MemgineEngine>> {
2204 if let Some(eng) = self.bound_memgine.lock().await.as_ref() {
2205 return eng.clone();
2206 }
2207 self.memgine.clone()
2208 }
2209
2210 pub async fn bind_run_journal(&self, run_id: &str) -> Result<(), String> {
2211 self.runtime
2212 .event_log_handle()
2213 .lock()
2214 .await
2215 .bind_run(run_id, &self.client_id)
2216 }
2217
2218 pub async fn require_run_journal_binding(&self, run_id: &str) -> Result<(), String> {
2219 let log = self.runtime.event_log_handle();
2220 let log = log.lock().await;
2221 match log.active_run_binding() {
2222 Some((bound_run, bound_client, _))
2223 if bound_run == run_id && bound_client == self.client_id =>
2224 {
2225 Ok(())
2226 }
2227 Some((bound_run, bound_client, _)) => Err(format!(
2228 "active journal binding mismatch: expected run_id `{run_id}` / client_id `{}`, got `{bound_run}` / `{bound_client}`",
2229 self.client_id
2230 )),
2231 None => Err(format!(
2232 "active run `{run_id}` has no authenticated journal binding"
2233 )),
2234 }
2235 }
2236
2237 pub async fn clear_run_journal_binding(&self, run_id: &str) -> Result<(), String> {
2238 self.runtime
2239 .event_log_handle()
2240 .lock()
2241 .await
2242 .clear_run_binding(run_id, &self.client_id)
2243 }
2244
2245 pub async fn append_run_terminal_event(
2246 &self,
2247 ended: &car_proto::RunEnded,
2248 ) -> Result<(), String> {
2249 self.append_run_terminal_event_once(ended, &self.client_id)
2250 .await
2251 .map_err(|error| error.to_string())
2252 }
2253
2254 /// Append a terminal from an authenticated replacement socket while
2255 /// preserving the immutable client identity in the durable run trace.
2256 pub(crate) async fn append_resumed_run_terminal_event(
2257 &self,
2258 ended: &car_proto::RunEnded,
2259 durable_client_id: &str,
2260 ) -> Result<(), String> {
2261 self.append_run_terminal_event_once(ended, durable_client_id)
2262 .await
2263 .map_err(|error| error.to_string())
2264 }
2265
2266 pub async fn append_run_cancellation_requested_event(
2267 &self,
2268 requested: &car_proto::RunCancellationRequested,
2269 ) -> Result<(), String> {
2270 self.require_run_journal_binding(&requested.run_id).await?;
2271 let value = serde_json::to_value(requested).map_err(|error| error.to_string())?;
2272 let Value::Object(data) = value else {
2273 return Err("cancellation request did not serialize as an object".into());
2274 };
2275 self.runtime
2276 .event_log_handle()
2277 .lock()
2278 .await
2279 .append_critical_async(
2280 car_eventlog::EventKind::RunCancellationRequested,
2281 requested.action_id.as_deref(),
2282 None,
2283 data.into_iter().collect(),
2284 CRITICAL_JOURNAL_ACKNOWLEDGEMENT_TIMEOUT,
2285 )
2286 .await
2287 .map_err(|error| error.to_string())?;
2288 Ok(())
2289 }
2290
2291 pub async fn append_run_cancellation_result_event(
2292 &self,
2293 result: &car_proto::RunCancelResponse,
2294 ) -> Result<(), String> {
2295 self.require_run_journal_binding(&result.run_id).await?;
2296 let value = serde_json::to_value(result).map_err(|error| error.to_string())?;
2297 let Value::Object(data) = value else {
2298 return Err("cancellation result did not serialize as an object".into());
2299 };
2300 self.runtime
2301 .event_log_handle()
2302 .lock()
2303 .await
2304 .append_critical_async(
2305 car_eventlog::EventKind::RunCancellationResult,
2306 result.action_id.as_deref(),
2307 None,
2308 data.into_iter().collect(),
2309 CRITICAL_JOURNAL_ACKNOWLEDGEMENT_TIMEOUT,
2310 )
2311 .await
2312 .map_err(|error| error.to_string())?;
2313 Ok(())
2314 }
2315
2316 async fn append_run_terminal_event_once(
2317 &self,
2318 ended: &car_proto::RunEnded,
2319 durable_client_id: &str,
2320 ) -> Result<(), car_eventlog::CriticalAppendError> {
2321 let rejected = |reason| car_eventlog::CriticalAppendError::Rejected { reason };
2322 self.require_run_journal_binding(&ended.run_id)
2323 .await
2324 .map_err(rejected)?;
2325 if ended.client_id.as_deref() != Some(durable_client_id) {
2326 return Err(rejected(
2327 "RunEnded client_id does not match the durable run owner".into(),
2328 ));
2329 }
2330 let completion_digest = ended
2331 .completion_digest
2332 .as_deref()
2333 .ok_or_else(|| rejected("RunEnded is missing completion_digest".into()))?;
2334 let termination_kind = match &ended.termination {
2335 car_proto::RunTermination::Outcome { .. } => "outcome",
2336 car_proto::RunTermination::Incomplete => "incomplete",
2337 car_proto::RunTermination::Cancelled { .. } => "cancelled",
2338 };
2339 let log = self.runtime.event_log_handle();
2340 let mut log = log.lock().await;
2341 log.append_critical_async(
2342 car_eventlog::EventKind::RunCompleted,
2343 None,
2344 None,
2345 [
2346 (
2347 "termination_kind".to_string(),
2348 Value::from(termination_kind),
2349 ),
2350 (
2351 "completion_digest".to_string(),
2352 Value::from(completion_digest),
2353 ),
2354 (
2355 "termination".to_string(),
2356 serde_json::to_value(&ended.termination)
2357 .map_err(|error| rejected(error.to_string()))?,
2358 ),
2359 ]
2360 .into(),
2361 CRITICAL_JOURNAL_ACKNOWLEDGEMENT_TIMEOUT,
2362 )
2363 .await?;
2364 Ok(())
2365 }
2366
2367 pub async fn append_run_started_event(
2368 &self,
2369 started: &car_proto::RunStarted,
2370 ) -> Result<(), String> {
2371 self.require_run_journal_binding(&started.run_id).await?;
2372 if started.client_id.as_deref() != Some(self.client_id.as_str()) {
2373 return Err("RunStarted client_id does not match the live ClientSession".into());
2374 }
2375 let mut data = HashMap::from([
2376 (
2377 "agent_id".to_string(),
2378 Value::from(started.agent_id.clone()),
2379 ),
2380 ("intent".to_string(), Value::from(started.intent.clone())),
2381 (
2382 "started_at".to_string(),
2383 serde_json::to_value(started.started_at).unwrap_or(Value::Null),
2384 ),
2385 ]);
2386 if let Some(description) = &started.outcome_description {
2387 data.insert(
2388 "outcome_description".to_string(),
2389 Value::from(description.clone()),
2390 );
2391 }
2392 self.runtime
2393 .event_log_handle()
2394 .lock()
2395 .await
2396 .append_critical_async(
2397 car_eventlog::EventKind::RunStarted,
2398 None,
2399 None,
2400 data,
2401 CRITICAL_JOURNAL_ACKNOWLEDGEMENT_TIMEOUT,
2402 )
2403 .await
2404 .map_err(|error| error.to_string())?;
2405 Ok(())
2406 }
2407}
2408
2409/// Builder for constructing a [`ServerState`] with embedder-supplied
2410/// dependencies. Embedders (e.g. `tokhn-daemon`) use this to inject
2411/// their own memgine handle and other shared infrastructure; the
2412/// Approval-gate policy for high-risk WS methods.
2413///
2414/// Every method in `methods` must be acknowledged via
2415/// `host.resolve_approval` before the dispatcher will route the
2416/// request to its handler. The dispatcher waits up to `timeout` for
2417/// a resolution; on timeout (or any non-`approve` resolution) the
2418/// request fails with JSON-RPC error `-32003`.
2419///
2420/// Default: gate enabled, the host-automation surface
2421/// (`automation.run_applescript`, `automation.run_powershell`,
2422/// `automation.shortcuts.run`, `messages.send`, `mail.send`, `vision.ocr`),
2423/// 60-second timeout.
2424/// `car-server --no-approvals` (or embedders calling
2425/// [`ServerStateConfig::with_approval_gate`] with `enabled=false`)
2426/// turns it off — only appropriate when no untrusted caller can
2427/// reach the WS port.
2428#[derive(Debug, Clone)]
2429pub struct ApprovalGate {
2430 /// Master switch. When `false`, every method dispatches without
2431 /// raising an approval — the pre-2026-05 behaviour.
2432 pub enabled: bool,
2433 /// Methods that require approval. Match is by exact method-name
2434 /// string against the JSON-RPC `method` field.
2435 pub methods: std::collections::HashSet<String>,
2436 /// How long to wait for the user to resolve the approval before
2437 /// timing out and surfacing an error to the caller.
2438 pub timeout: std::time::Duration,
2439}
2440
2441impl Default for ApprovalGate {
2442 fn default() -> Self {
2443 let methods = [
2444 "automation.run_applescript",
2445 "automation.run_powershell",
2446 "automation.shortcuts.run",
2447 "messages.send",
2448 "mail.send",
2449 "vision.ocr",
2450 ]
2451 .iter()
2452 .map(|s| s.to_string())
2453 .collect();
2454 Self {
2455 enabled: true,
2456 methods,
2457 timeout: std::time::Duration::from_secs(60),
2458 }
2459 }
2460}
2461
2462impl ApprovalGate {
2463 /// Disable the gate entirely. Equivalent to passing
2464 /// `car-server --no-approvals`. Only appropriate when no
2465 /// untrusted caller can reach the WS port.
2466 pub fn disabled() -> Self {
2467 Self {
2468 enabled: false,
2469 methods: std::collections::HashSet::new(),
2470 timeout: std::time::Duration::from_secs(60),
2471 }
2472 }
2473
2474 /// `true` if this method must be acknowledged before dispatch.
2475 pub fn requires_approval(&self, method: &str) -> bool {
2476 self.enabled && self.methods.contains(method)
2477 }
2478}
2479
2480/// standalone `car-server` binary uses [`ServerState::standalone`]
2481/// which calls `with_config` under the hood.
2482pub struct ServerStateConfig {
2483 pub journal_dir: PathBuf,
2484 /// Peer-message audit journal. `None` derives a path once from
2485 /// [`Self::journal_dir`] and the construction-time CAR state root. Tests
2486 /// and embedders can override it without mutating process-global `CAR_HOME`.
2487 pub peer_audit_journal: Option<PathBuf>,
2488 /// Optional pre-constructed memgine engine. When `None`, each
2489 /// `create_session` call builds a fresh engine; embedders that want
2490 /// to share a single engine across sessions can supply a clone of
2491 /// their `Arc<Mutex<MemgineEngine>>` here.
2492 pub shared_memgine: Option<Arc<Mutex<car_memgine::MemgineEngine>>>,
2493 /// Optional pre-constructed inference engine.
2494 pub inference: Option<Arc<car_inference::InferenceEngine>>,
2495 /// Optional embedder-supplied A2A runtime. Used by the in-core
2496 /// `A2aDispatcher` to execute peer-driven proposals. When `None`,
2497 /// the dispatcher uses a fresh `Runtime` with `register_agent_basics`
2498 /// — peer agents see CAR's built-in tools and nothing else,
2499 /// matching the behaviour of the standalone `start_a2a_listener`.
2500 pub a2a_runtime: Option<Arc<car_engine::Runtime>>,
2501 /// Optional embedder-supplied A2A task store. When `None`,
2502 /// defaults to `InMemoryTaskStore`. tokhn-style embedders that
2503 /// want a polling-friendly persistent store plug it in here.
2504 pub a2a_store: Option<Arc<dyn car_a2a::TaskStore>>,
2505 /// Optional embedder-supplied agent card factory. When `None`,
2506 /// the dispatcher serves a card built from the A2A runtime's
2507 /// tool schemas at construction time, advertising its public URL
2508 /// as `ws://127.0.0.1:9100/` (the WS surface the dispatcher itself
2509 /// is reachable on).
2510 pub a2a_card_source: Option<Arc<car_a2a::AgentCardSource>>,
2511 /// Approval-gate policy. When `None`, the dispatcher uses
2512 /// [`ApprovalGate::default`] (gate ON, the macOS-automation
2513 /// surface gated, 60s timeout). Pass
2514 /// [`ApprovalGate::disabled`] to opt out — only appropriate
2515 /// when no untrusted caller can reach the WS port.
2516 pub approval_gate: Option<ApprovalGate>,
2517 /// Journal path for the daemon-wide shared HITL approval ledger
2518 /// ([`ServerState::approval_ledger`]). `None` = the default
2519 /// `~/.car/approvals.jsonl`. Tests and embedders that must not touch the
2520 /// user's real home point this at their own path.
2521 pub approval_journal: Option<PathBuf>,
2522 /// Directory for the daemon-wide trajectory store
2523 /// ([`ServerState::trajectory_store`]). `None` = the default
2524 /// `~/.car/trajectories/`. Tests and embedders that must not touch the
2525 /// user's real home point this at their own path, same as
2526 /// `approval_journal`.
2527 pub trajectory_dir: Option<PathBuf>,
2528 /// Deterministic durability fault seam used by direct lifecycle tests.
2529 pub run_store_failures: Option<crate::run_store::RunStoreFailureInjector>,
2530 /// Deterministic blocking-read seam used by run-liveness tests.
2531 #[doc(hidden)]
2532 pub run_store_summary_read_gate: Option<crate::run_store::RunStoreSummaryReadGate>,
2533 /// Test-only scheduling seam around durable run-summary publication.
2534 #[doc(hidden)]
2535 pub run_store_summary_write_gate: Option<crate::run_store::RunStoreSummaryWriteGate>,
2536 /// Deterministic blocking append seam used by run-liveness tests.
2537 #[doc(hidden)]
2538 pub run_store_append_gate: Option<crate::run_store::RunStoreAppendGate>,
2539 #[doc(hidden)]
2540 pub run_completion_fence_gate: Option<RunCompletionFenceGate>,
2541 /// Deterministic blocking durable-lookup seam used by run-liveness tests.
2542 #[doc(hidden)]
2543 pub run_store_lookup_gate: Option<crate::run_store::RunStoreLookupGate>,
2544 /// Deterministic first-use directory-entry durability seam for RunStore.
2545 pub run_store_private_path_failures: Option<car_secrets::PrivatePathDurabilityFailureInjector>,
2546 /// Deterministic critical-journal fault seam used by direct lifecycle tests.
2547 pub journal_failures: Option<car_eventlog::JournalFailureInjector>,
2548 /// Private self-heal detection and bounded-attempt ledger. The standalone daemon sets this to
2549 /// `<CAR_HOME>/selfheal/detections.jsonl`; embedders/tests can isolate it.
2550 pub selfheal_ledger: Option<PathBuf>,
2551 /// Detection cadence. Eligible default-on auto-fix runs from the same
2552 /// non-overlapping tick and is independently controlled by config.toml.
2553 pub selfheal_interval_secs: u64,
2554 /// Deterministic evidence seam for integration tests and embedders.
2555 #[doc(hidden)]
2556 pub selfheal_evidence: Option<crate::selfheal::SelfhealEvidence>,
2557 /// Bounded source-checkout probe. Embedders fail closed unless they supply
2558 /// candidates; the standalone daemon derives this from process-local paths.
2559 pub selfheal_source_probe: crate::selfheal::SelfhealSourceProbe,
2560 /// Deterministic seam for the checkout-built replay-verb eligibility probe.
2561 #[doc(hidden)]
2562 pub selfheal_replay_verb_probe: crate::selfheal::SelfhealReplayVerbProbe,
2563 /// Test-only override for the startup reconciliation acknowledgement bound.
2564 #[doc(hidden)]
2565 pub startup_reconciliation_acknowledgement_timeout: std::time::Duration,
2566 /// Test-only override for the bounded in-process resume lease.
2567 #[doc(hidden)]
2568 pub run_resume_lease: std::time::Duration,
2569}
2570
2571impl ServerStateConfig {
2572 /// Minimal config suitable for the standalone car-server binary:
2573 /// only the journal dir is required; everything else is lazily
2574 /// constructed at first use.
2575 pub fn new(journal_dir: PathBuf) -> Self {
2576 Self {
2577 journal_dir,
2578 peer_audit_journal: None,
2579 shared_memgine: None,
2580 inference: None,
2581 a2a_runtime: None,
2582 a2a_store: None,
2583 a2a_card_source: None,
2584 approval_gate: None,
2585 approval_journal: None,
2586 trajectory_dir: None,
2587 run_store_failures: None,
2588 run_store_summary_read_gate: None,
2589 run_store_summary_write_gate: None,
2590 run_store_append_gate: None,
2591 run_completion_fence_gate: None,
2592 run_store_lookup_gate: None,
2593 run_store_private_path_failures: None,
2594 journal_failures: None,
2595 selfheal_ledger: None,
2596 selfheal_interval_secs: crate::selfheal::DEFAULT_SELFHEAL_INTERVAL_SECS,
2597 selfheal_evidence: None,
2598 selfheal_source_probe: crate::selfheal::SelfhealSourceProbe::empty(),
2599 selfheal_replay_verb_probe: crate::selfheal::SelfhealReplayVerbProbe::default(),
2600 startup_reconciliation_acknowledgement_timeout:
2601 STARTUP_RECONCILIATION_ACKNOWLEDGEMENT_TIMEOUT,
2602 run_resume_lease: RUN_RESUME_LEASE,
2603 }
2604 }
2605
2606 pub fn with_shared_memgine(mut self, engine: Arc<Mutex<car_memgine::MemgineEngine>>) -> Self {
2607 self.shared_memgine = Some(engine);
2608 self
2609 }
2610
2611 /// Override the peer-message audit journal path.
2612 pub fn with_peer_audit_journal(mut self, path: PathBuf) -> Self {
2613 self.peer_audit_journal = Some(path);
2614 self
2615 }
2616
2617 /// Override the shared approval-ledger journal path (default
2618 /// `~/.car/approvals.jsonl`).
2619 pub fn with_approval_journal(mut self, path: PathBuf) -> Self {
2620 self.approval_journal = Some(path);
2621 self
2622 }
2623
2624 pub fn with_trajectory_dir(mut self, dir: PathBuf) -> Self {
2625 self.trajectory_dir = Some(dir);
2626 self
2627 }
2628
2629 pub fn with_run_store_failures(
2630 mut self,
2631 failures: crate::run_store::RunStoreFailureInjector,
2632 ) -> Self {
2633 self.run_store_failures = Some(failures);
2634 self
2635 }
2636
2637 #[doc(hidden)]
2638 pub fn with_run_store_summary_read_gate(
2639 mut self,
2640 gate: crate::run_store::RunStoreSummaryReadGate,
2641 ) -> Self {
2642 self.run_store_summary_read_gate = Some(gate);
2643 self
2644 }
2645
2646 #[doc(hidden)]
2647 pub fn with_run_store_summary_write_gate(
2648 mut self,
2649 gate: crate::run_store::RunStoreSummaryWriteGate,
2650 ) -> Self {
2651 self.run_store_summary_write_gate = Some(gate);
2652 self
2653 }
2654
2655 #[doc(hidden)]
2656 pub fn with_run_store_append_gate(
2657 mut self,
2658 gate: crate::run_store::RunStoreAppendGate,
2659 ) -> Self {
2660 self.run_store_append_gate = Some(gate);
2661 self
2662 }
2663
2664 #[doc(hidden)]
2665 pub fn with_run_completion_fence_gate(mut self, gate: RunCompletionFenceGate) -> Self {
2666 self.run_completion_fence_gate = Some(gate);
2667 self
2668 }
2669
2670 #[doc(hidden)]
2671 pub fn with_run_resume_lease(mut self, lease: std::time::Duration) -> Self {
2672 self.run_resume_lease = lease;
2673 self
2674 }
2675
2676 #[doc(hidden)]
2677 pub fn with_run_store_lookup_gate(
2678 mut self,
2679 gate: crate::run_store::RunStoreLookupGate,
2680 ) -> Self {
2681 self.run_store_lookup_gate = Some(gate);
2682 self
2683 }
2684
2685 pub fn with_run_store_private_path_failures(
2686 mut self,
2687 failures: car_secrets::PrivatePathDurabilityFailureInjector,
2688 ) -> Self {
2689 self.run_store_private_path_failures = Some(failures);
2690 self
2691 }
2692
2693 pub fn with_journal_failures(mut self, failures: car_eventlog::JournalFailureInjector) -> Self {
2694 self.journal_failures = Some(failures);
2695 self
2696 }
2697
2698 pub fn with_selfheal_ledger(mut self, path: PathBuf) -> Self {
2699 self.selfheal_ledger = Some(path);
2700 self
2701 }
2702
2703 pub fn with_selfheal_interval_secs(mut self, interval_secs: u64) -> Self {
2704 self.selfheal_interval_secs = interval_secs.max(1);
2705 self
2706 }
2707
2708 #[doc(hidden)]
2709 pub fn with_selfheal_evidence(mut self, evidence: crate::selfheal::SelfhealEvidence) -> Self {
2710 self.selfheal_evidence = Some(evidence);
2711 self
2712 }
2713
2714 pub fn with_selfheal_source_probe(
2715 mut self,
2716 probe: crate::selfheal::SelfhealSourceProbe,
2717 ) -> Self {
2718 self.selfheal_source_probe = probe;
2719 self
2720 }
2721
2722 #[doc(hidden)]
2723 pub fn with_selfheal_replay_verb_probe(
2724 mut self,
2725 probe: crate::selfheal::SelfhealReplayVerbProbe,
2726 ) -> Self {
2727 self.selfheal_replay_verb_probe = probe;
2728 self
2729 }
2730
2731 #[doc(hidden)]
2732 pub fn with_startup_reconciliation_acknowledgement_timeout(
2733 mut self,
2734 timeout: std::time::Duration,
2735 ) -> Self {
2736 self.startup_reconciliation_acknowledgement_timeout = timeout;
2737 self
2738 }
2739
2740 pub fn with_inference(mut self, engine: Arc<car_inference::InferenceEngine>) -> Self {
2741 self.inference = Some(engine);
2742 self
2743 }
2744
2745 /// Plug in an embedder-supplied runtime for the A2A dispatcher.
2746 /// Use case: tokhn-daemon wants peers to see its OPA preflight
2747 /// tooling, not just CAR's `register_agent_basics` defaults.
2748 pub fn with_a2a_runtime(mut self, runtime: Arc<car_engine::Runtime>) -> Self {
2749 self.a2a_runtime = Some(runtime);
2750 self
2751 }
2752
2753 /// Plug in an embedder-supplied task store for the A2A
2754 /// dispatcher. Use case: tokhn's polling-friendly persistent
2755 /// store keyed by their session id.
2756 pub fn with_a2a_store(mut self, store: Arc<dyn car_a2a::TaskStore>) -> Self {
2757 self.a2a_store = Some(store);
2758 self
2759 }
2760
2761 /// Plug in an embedder-supplied agent card factory. The factory
2762 /// is invoked on every `agent/getAuthenticatedExtendedCard`
2763 /// dispatch, so embedders can reflect runtime tool changes.
2764 pub fn with_a2a_card_source(mut self, source: Arc<car_a2a::AgentCardSource>) -> Self {
2765 self.a2a_card_source = Some(source);
2766 self
2767 }
2768
2769 /// Override the approval-gate policy. Pass
2770 /// [`ApprovalGate::disabled`] to skip the gate entirely (only
2771 /// appropriate when no untrusted caller can reach the WS port);
2772 /// pass a customised [`ApprovalGate`] to add or remove methods
2773 /// or to change the timeout.
2774 pub fn with_approval_gate(mut self, gate: ApprovalGate) -> Self {
2775 self.approval_gate = Some(gate);
2776 self
2777 }
2778}
2779
2780/// The opted-in shared org-scope delivery subsystem + which org it serves. See
2781/// [`ServerState::org_sync`]. Held behind the same `std::sync::Mutex` as the
2782/// personal subsystem holder (init is serialized by the oplog's advisory lock);
2783/// the subsystem itself lives behind a `tokio::sync::Mutex`.
2784pub struct OrgSyncHolder {
2785 pub org: String,
2786 pub subsystem: Arc<tokio::sync::Mutex<crate::sync::SyncSubsystem>>,
2787}
2788
2789/// Which subsystem a write routes to. Pure decision, extracted so it is unit-
2790/// testable without a full daemon.
2791#[derive(Debug, PartialEq, Eq)]
2792enum SyncRoute {
2793 User,
2794 Org,
2795}
2796
2797/// Route by scope: a `Scope::Shared{org}` whose org matches the opted-in
2798/// `org_holder` goes to the org delivery subsystem; EVERYTHING else — `Personal`,
2799/// a different org, or no opted-in holder — goes to the personal subsystem. So
2800/// with `org_holder = None` (org-scope off) every scope routes `User`, byte-
2801/// identical to before.
2802fn route_for_scope(scope: &car_sync::Scope, org_holder: Option<&str>) -> SyncRoute {
2803 if let car_sync::Scope::Shared { org } = scope {
2804 if org_holder == Some(org.as_str()) {
2805 return SyncRoute::Org;
2806 }
2807 }
2808 SyncRoute::User
2809}
2810
2811/// Global server state shared across all connections.
2812pub struct ServerState {
2813 pub journal_dir: PathBuf,
2814 /// Resolved once when the state is constructed. Peer-message writes must
2815 /// use this path and never consult process-global `CAR_HOME` themselves.
2816 pub peer_audit_journal: PathBuf,
2817 pub sessions: Mutex<HashMap<String, Arc<ClientSession>>>,
2818 /// Serializes only session registry insertion/removal with the final
2819 /// liveness check + owner swap in `runs.resume`. This avoids holding the
2820 /// global session map while waiting on per-run durability and turns the
2821 /// replacement-present / old-owner-absent decision into one atomic fence.
2822 run_resume_liveness: Mutex<()>,
2823 run_resume_lease: std::time::Duration,
2824 run_completion_fence_gate: Option<RunCompletionFenceGate>,
2825 pub inference: std::sync::OnceLock<Arc<car_inference::InferenceEngine>>,
2826 pub host: Arc<crate::host::HostState>,
2827 /// When `Some`, `create_session` clones this handle into every new
2828 /// `ClientSession.memgine` — embedders that want a single shared
2829 /// memgine across all WS sessions set this. Standalone car-server
2830 /// leaves it `None`, which gives each session its own engine
2831 /// (preserving today's behavior).
2832 pub shared_memgine: Option<Arc<Mutex<car_memgine::MemgineEngine>>>,
2833 /// Feedback doctor inputs resolved through [`ServerStateConfig`].
2834 pub(crate) feedback_diagnostics: crate::feedback::FeedbackDiagnostics,
2835 /// Daemon-wide trajectory store, cloned into **every** session's
2836 /// `Runtime` so execution traces persist to one place.
2837 ///
2838 /// Shared rather than per-session on purpose. A trajectory is evidence
2839 /// about how a tool behaves, and that evidence does not belong to the
2840 /// connection that happened to produce it — per-session stores would
2841 /// scatter the history across connections and reset it on every
2842 /// reconnect, leaving the derived success rates built from a handful of
2843 /// samples. This is the same reasoning that makes the approval ledger
2844 /// daemon-wide.
2845 ///
2846 /// Until now nothing called `Runtime::with_trajectory_store`, so
2847 /// `persist_trajectory` returned early on every execution and the store
2848 /// was dead code: `ToolFeedback::from_trajectories` existed with no data
2849 /// to read. Wiring it here is what makes the feedback loop real.
2850 pub trajectory_store: Arc<car_memgine::TrajectoryStore>,
2851 /// Daemon-wide self-healing detector/fixer + CAR_HOME ledger. Shared
2852 /// across sessions so `selfheal.run` and the cadence cannot overlap and
2853 /// every caller reads the same dismissals/detections.
2854 pub selfheal: Arc<crate::selfheal::SelfhealService>,
2855 /// The self-healing *repair* loop: reads a configured tracker, runs a coder
2856 /// session, gates the result on a multi-model panel, opens a pull request.
2857 ///
2858 /// Distinct from [`Self::selfheal`], which detects and never writes. Shared
2859 /// across sessions for the same reason: `heal.run` and the cadence must not
2860 /// overlap, and both must read the same claim ledger.
2861 pub heal: Arc<crate::coder::heal_service::HealService>,
2862 /// Process-wide voice session registry. Each
2863 /// `voice.transcribe_stream.start` call registers its own per-client
2864 /// [`WsVoiceEventSink`] so events route back to the originating WS
2865 /// connection only.
2866 pub voice_sessions: Arc<car_voice::VoiceSessionRegistry>,
2867 /// Process-wide meeting registry. Meeting ids are global; each
2868 /// meeting binds to the originating client's WS for upstream
2869 /// events but persists transcripts to the resolved
2870 /// `.car/meetings/<id>/` regardless of which client started it.
2871 pub meetings: Arc<car_meeting::MeetingRegistry>,
2872 /// Process-wide A2UI surface store. Agent-produced surfaces are
2873 /// visible to every host UI subscriber, independent of the
2874 /// WebSocket session that applied the update.
2875 pub a2ui: car_a2ui::A2uiSurfaceStore,
2876 /// In-process UI-improvement agent. Invoked from
2877 /// `handle_a2ui_render_report` with each inbound report; returned
2878 /// `Decision::Patch` envelopes are applied via the standard
2879 /// `apply_a2ui_envelope` path so all subscribers see the patch.
2880 /// `Arc` so the agent's interior `DashMap` state survives across
2881 /// handler calls even when `ServerState` is cheap-cloned.
2882 pub ui_agent: Arc<car_ui_agent::UIImprovementAgent>,
2883 /// Per-surface oscillation detector for the UI-improvement
2884 /// loop. Sits between the agent's `Decision::Patch` and the
2885 /// apply path so A→B→A patch cycles get cooled down without
2886 /// the agent itself having to track history. neo's review:
2887 /// "controllers use workqueue backoff; reconcilers stay
2888 /// stateless."
2889 pub ui_agent_oscillation: Arc<crate::ui_agent_loop::OscillationDetector>,
2890 /// Per-surface iteration budget. Backstop against runaway
2891 /// loops the oscillation detector misses — caps total agent-
2892 /// driven patches per surface at `DEFAULT_MAX_ITERATIONS`.
2893 pub ui_agent_budget: Arc<crate::ui_agent_loop::IterationBudget>,
2894 /// Process-wide concurrency gate for inference RPC handlers. Sized
2895 /// from host RAM at startup, overridable via
2896 /// [`crate::admission::ENV_MAX_CONCURRENT`]. Without this, N
2897 /// concurrent users multiply KV-cache and activation memory and
2898 /// take the host out (#114-adjacent: filed alongside the daemon
2899 /// always-on rework). The semaphore lives on `ServerState` so it
2900 /// is shared across every WebSocket session in the same process.
2901 pub admission: Arc<crate::admission::InferenceAdmission>,
2902 /// Server-side A2A continuation auth keyed by A2UI surface id.
2903 /// Kept out of `A2uiSurface.owner` so host renderers never see
2904 /// bearer/API-key material.
2905 pub a2ui_route_auth: Mutex<HashMap<String, A2aRouteAuth>>,
2906 /// Lifecycle-managed agents — declarative manifest at
2907 /// `~/.car/agents.json` driving spawn/restart/stop. Closes
2908 /// Parslee-ai/car-releases#27. Lazy-initialized so embedders that
2909 /// don't want process supervision don't pay the disk-touch cost
2910 /// at server start.
2911 pub supervisor: std::sync::OnceLock<Arc<car_registry::supervisor::Supervisor>>,
2912 /// Declarative (in-daemon) agent registry — a parallel store to the
2913 /// supervisor for agents the coder→agent loop builds. Lazy-initialized on
2914 /// first use (`~/.car/declagents.json`).
2915 pub declagents: std::sync::OnceLock<Arc<car_registry::declarative::DeclRegistry>>,
2916 /// Routing learning state — per-agent success stats + agent→agent edge
2917 /// weights for capability-similarity routing (`~/.car/routing.json`).
2918 /// Lazy-initialized on first use, sibling to [`Self::declagents`].
2919 pub routing: std::sync::OnceLock<Arc<car_registry::routing::RoutingStore>>,
2920 /// Multi-device sync + execution-lease subsystem (B6). Lazily opened on
2921 /// first `sync.*`/`lease.*` contact, rooted at `<journal_dir>/sync/` — a
2922 /// daemon-held [`car_sync::SyncSession`] over an `FsRelay` (the single-user
2923 /// two-device loopback works out of the box) plus an in-process
2924 /// linearizable [`car_sync::InMemoryLeaseCoordinator`]. **One subsystem per
2925 /// daemon = one device in the sync fleet.** The `std::sync::Mutex<Option>`
2926 /// serializes the fallible first-open (the oplog journal holds an exclusive
2927 /// advisory lock, so a double-open would fail) without holding the lock
2928 /// across the subsystem's async work — the subsystem itself lives behind a
2929 /// `tokio::sync::Mutex` inside the `Arc`. A distributed relay + a
2930 /// cross-daemon linearizable `LeaseCoordinator` backend is the documented
2931 /// B6 follow-up; the traits are its contract.
2932 pub sync: std::sync::Mutex<Option<Arc<tokio::sync::Mutex<crate::sync::SyncSubsystem>>>>,
2933 /// The SHARED org-scope delivery subsystem — a SECOND device oplog whose relay
2934 /// scope is `org:{orgId}`, so `Scope::Shared{org}` ops converge across an org's
2935 /// members. `None` unless org-scope is opted in (the same gated
2936 /// `PARSLEE_SYNC_ORG_SCOPE` branch that resolves the org key builds it);
2937 /// populated by `open_sync_subsystem` alongside the personal subsystem. Only
2938 /// the `Scope::Shared` write sites + `pump` route to it — the personal path
2939 /// (`sync_subsystem()` + all readers) is untouched. Slice 8a.
2940 pub org_sync: std::sync::Mutex<Option<OrgSyncHolder>>,
2941 /// Manifest path this daemon is *observing* but does NOT own.
2942 /// Set by `car-server` when boot-time supervisor construction
2943 /// fails with [`car_registry::supervisor::SupervisorError::AlreadyRunning`]
2944 /// — another car-server process on the host holds the exclusive
2945 /// lock on this manifest. In that state, `supervisor()` returns a
2946 /// clear "observe-only" error so mutation handlers refuse
2947 /// (preventing the duplicate-spawn bug from
2948 /// Parslee-ai/car-releases#44), while read-only handlers
2949 /// (`agents.list`, `agents.health`) fall back to
2950 /// [`car_registry::supervisor::Supervisor::list_from_manifest`] /
2951 /// [`car_registry::supervisor::Supervisor::health_from_manifest`]
2952 /// so operators can still inspect what the primary daemon is
2953 /// supervising.
2954 pub observer_manifest_path: std::sync::OnceLock<PathBuf>,
2955 /// In-core A2A dispatcher — embedders that consume `car-server-core`
2956 /// get A2A reachability "for free" without standing up a separate
2957 /// HTTP listener. Closes Parslee-ai/car-releases#28. Lazy-init so
2958 /// the embedder can override the runtime / task store / agent card
2959 /// via [`ServerStateConfig::with_a2a_runtime`] etc. before the
2960 /// first dispatch.
2961 pub a2a_dispatcher: std::sync::OnceLock<Arc<car_a2a::A2aDispatcher>>,
2962 /// WS clients subscribed to A2UI envelope events. After every
2963 /// successful `a2ui.apply` / `a2ui.ingest`, the resulting
2964 /// `A2uiApplyResult` is broadcast to every subscriber as an
2965 /// `a2ui.event` JSON-RPC notification. Closes
2966 /// Parslee-ai/car-releases#29. Subscribers register via the
2967 /// `a2ui/subscribe` method and are auto-cleaned on WS disconnect.
2968 pub a2ui_subscribers: Mutex<HashMap<String, Arc<WsChannel>>>,
2969 /// Per-launch auth token. When `Some`, the WS dispatcher rejects
2970 /// non-auth methods on unauthenticated sessions until the client
2971 /// calls `session.auth` with the matching value. When `None`,
2972 /// auth is disabled and every connection works as before. Set
2973 /// at startup by `car-server` unless `--no-auth` is passed
2974 /// (default flipped 2026-05); embedders that want to enable
2975 /// auth call [`ServerState::install_auth_token`]. Closes
2976 /// Parslee-ai/car-releases#32.
2977 /// Shared MCP tool executor backing remote connectors. One per
2978 /// process: every WS session's runtime executor is a
2979 /// [`car_engine::McpToolExecutor::share_with_fallback`] view over
2980 /// this, so connector tools route here while non-connector tools
2981 /// fall back to the per-session [`WsToolExecutor`]. Connectors
2982 /// (their live sessions + routes) live on this shared instance, so
2983 /// a connector enabled in one WS session is reachable from all.
2984 pub mcp_executor: Arc<car_engine::McpToolExecutor>,
2985 /// Remote MCP connector manager (`~/.car/connectors.json`).
2986 /// Lazy-initialized over [`mcp_executor`](Self::mcp_executor) so
2987 /// embedders that never touch connectors pay no disk cost. See
2988 /// [`ServerState::connectors`].
2989 pub connectors: std::sync::OnceLock<Arc<car_connectors::ConnectorManager>>,
2990 /// One-shot guard so persisted connectors are loaded + dialed at
2991 /// most once per process (see [`ServerState::ensure_connectors_loaded`]).
2992 connectors_loaded: std::sync::atomic::AtomicBool,
2993 /// Runtime approval-transport channel supervisor (Units 1/2/3). Lazy-
2994 /// initialized at daemon boot by `spawn_channel_pollers`, which records the
2995 /// channels it spawned at startup. The host-gated `messaging.config.set`
2996 /// handler reaches it via `ServerState` to spawn a channel's watcher the
2997 /// instant the user enables it — no restart (U1). Also holds the per-channel
2998 /// liveness `messaging.status` reads (U2) and the send path writes (U3).
2999 /// `None` for embedders that never boot the channel pollers.
3000 pub channel_supervisor: std::sync::OnceLock<Arc<crate::channel_supervisor::ChannelSupervisor>>,
3001 /// Daemon-owned operations which must outlive the WebSocket request that
3002 /// started them. The tasks hold no [`ClientSession`] or [`WsChannel`]; only
3003 /// their oneshot response waiters are connection-scoped.
3004 ///
3005 /// Two kinds of work live here. Credential-backed auth operations (attempt
3006 /// completion and ordinary auth mutations), so a socket close can release
3007 /// its sink immediately without cancelling a coordinator/keychain operation
3008 /// halfway through its durable result. And `coder.start`, which registers a
3009 /// session and provisions a worktree *before* a multi-minute contract
3010 /// derivation: once that side effect exists on disk, the run's lifetime
3011 /// must not be owned by the board that asked for it.
3012 pub durable_tasks: Mutex<tokio::task::JoinSet<String>>,
3013 /// Per-daemon incarnation token persisted into redeeming auth leases.
3014 /// Awaiting-callback reservations intentionally have no daemon owner and
3015 /// survive a restart; a redeeming lease owned by a different incarnation
3016 /// is an orphan and is terminally failed by completion-status reconciliation.
3017 pub auth_completion_owner_id: String,
3018 pub auth_token: std::sync::OnceLock<String>,
3019 /// Per-launch **host** token — a credential distinct from
3020 /// `auth_token`, granting the host-management role (host-class
3021 /// reads like cross-agent run traces). Critically it is **never**
3022 /// served over `GET /auth-token`: a session becomes host-role only
3023 /// by presenting this via `session.auth { host_token }`, and the
3024 /// only way to obtain it is reading the `0600` `host-token` file,
3025 /// which a different local user cannot. This is what stops any
3026 /// authenticated local client from self-elevating to host and
3027 /// reading every agent's run traces (Parslee-ai/car#254). When
3028 /// `None`, host-role can't be granted (no host reads).
3029 pub host_token: std::sync::OnceLock<String>,
3030 /// Mobile Parslee Core runtime URL this daemon can hand to consumer host apps.
3031 /// Set by `car-server` at startup from the daemon bind address or an
3032 /// explicit public URL override. The token remains the per-launch auth
3033 /// token; this field only names the WebSocket endpoint.
3034 pub mobile_runtime_url: std::sync::OnceLock<String>,
3035 /// Explicit public mobile runtime URL to register after an authorized
3036 /// credential action succeeds. Absent for the default loopback URL, which
3037 /// is discoverable locally but must not be advertised to Parslee cloud.
3038 pub mobile_registration_url: std::sync::OnceLock<String>,
3039 /// Parslee cloud identity activated by an explicit credential-backed
3040 /// request after `car auth login` has been completed.
3041 pub parslee_session: std::sync::OnceLock<crate::parslee_auth::ParsleeSession>,
3042 /// `agent_id -> client_id` map of currently-attached lifecycle
3043 /// agents (#169). Populated by the `session.auth` handler when a
3044 /// supervised child presents its `agent_id` + per-agent token;
3045 /// drained on disconnect by `remove_session`. Single-claim:
3046 /// a second connection presenting the same `agent_id` is
3047 /// rejected so the daemon-side per-agent state stays unambiguous.
3048 pub attached_agents: Mutex<HashMap<String, String>>,
3049 /// `agent_id -> persistent memgine` map (#170). Lazy-loaded on
3050 /// first connection per id from `~/.car/memory/agents/<id>.jsonl`,
3051 /// retained across daemon restart, surviving any single
3052 /// disconnect/reconnect of the supervised child. Connections
3053 /// that auth without an `agent_id` (browser, host, ad-hoc CLI)
3054 /// keep the per-WS ephemeral memgine on `ClientSession.memgine`
3055 /// — no behaviour change.
3056 pub agent_memgines: Mutex<HashMap<String, Arc<Mutex<car_memgine::MemgineEngine>>>>,
3057 /// Daemon-owned memgines keyed by a host-declared **memory namespace**
3058 /// (`session.auth { memory_namespace }`), lazily loaded from
3059 /// `~/.car/memory/memory-namespaces/<encoded-ns>.json` — the filename is a
3060 /// percent-encoding of the namespace, injective so two namespaces can never
3061 /// share a snapshot file (#891).
3062 ///
3063 /// A separate axis from `agent_memgines`, deliberately: one agent may work
3064 /// across several namespaces, and two hosts may share a namespace without
3065 /// sharing an identity (Parslee-ai/car-releases#79). A host that binds none
3066 /// keeps today's behaviour — the daemon's shared graph — so the MCP-to-WS
3067 /// shared knowledge base is untouched.
3068 pub namespace_memgines: Mutex<HashMap<String, Arc<Mutex<car_memgine::MemgineEngine>>>>,
3069 /// Live coder sessions keyed by `session_id` (built-in coding agent).
3070 /// Process-wide so a session outlives the WS connection that started
3071 /// it; terminal sessions stay listed for `coder.get`/`coder.list`
3072 /// history until daemon restart (snapshots persist under
3073 /// `~/.car/coder/`).
3074 pub coder_sessions: Mutex<crate::coder::rpc::CoderSessionMap>,
3075 /// Monotonic base for [`Self::coder_disk_gc_at`], taken at construction.
3076 pub coder_disk_gc_base: std::time::Instant,
3077 /// Seconds since [`Self::coder_disk_gc_base`] at the last coder state-dir
3078 /// sweep. Rate-limits the disk GC amortized onto `coder.start` (car#1339)
3079 /// so a burst of starts re-reads the directory once, not once each.
3080 ///
3081 /// Monotonic rather than Unix seconds because this is an INTERVAL: a wall
3082 /// clock that steps backwards would stamp a future value and suppress every
3083 /// later sweep for the daemon's lifetime, which is the bug this closes,
3084 /// reintroduced through the clock.
3085 ///
3086 /// On the state rather than a process-global static so each `ServerState` —
3087 /// every test builds its own — carries its own stamp, and one test's sweep
3088 /// cannot suppress another's.
3089 pub coder_disk_gc_at: AtomicU64,
3090 /// Live `coder.event` subscribers keyed by `(session_id, client_id)`
3091 /// — explicit fanout, same shape as `run_subscribers`. Lock order:
3092 /// the session's event buffer → this map; never the reverse.
3093 pub coder_subscribers: Mutex<HashMap<(String, String), Arc<WsChannel>>>,
3094 /// `coder.watch` board subscribers keyed by `client_id` — one
3095 /// subscription per connection covering EVERY session, so a board learns
3096 /// about runs started by any other client (`car code`, CarHost, milo)
3097 /// without polling. Fanned to as `coder.session_changed`. Lock order:
3098 /// a session's event buffer → `coder_subscribers` → this map; never the
3099 /// reverse.
3100 ///
3101 /// The value carries a **registration generation** alongside the channel.
3102 /// A board re-registers periodically (its `coder.watch` is idempotent and
3103 /// returns a fresh snapshot), and the fanout sheds a wedged watcher *after*
3104 /// releasing this lock — so a shed that removed by `client_id` alone would
3105 /// delete a registration made in that window and silently unwatch a healthy
3106 /// board. The shed compares the generation and removes only the entry it
3107 /// actually timed out on.
3108 pub coder_watchers: Mutex<HashMap<String, (u64, Arc<WsChannel>)>>,
3109 /// The one long-lived task that fans `coder.session_changed` out to
3110 /// [`coder_watchers`](Self::coder_watchers), lazily started on the first
3111 /// notification. Deliberately a single drain rather than a task per event:
3112 /// a running session emits on every tool call, and spawn-per-event against
3113 /// a half-open board socket accumulated blocked tasks without bound, each
3114 /// pinning the socket's write half past teardown (car#209's shape on a new
3115 /// path). Daemon-owned, so it is correctly absent from any connection's
3116 /// `conn_tasks`; it exits when this `ServerState` drops.
3117 pub(crate) coder_watch_notify: std::sync::OnceLock<mpsc::UnboundedSender<String>>,
3118 /// Live `coder.discuss.*` conversations keyed by `discussion_id`. In-memory
3119 /// only and deliberately NOT persisted across daemon restarts: a discussion
3120 /// is a thinking surface, not a run, and resuming one would mean rebuilding
3121 /// an assistant runtime and a model thread from a snapshot that never
3122 /// existed.
3123 pub coder_discussions: Mutex<crate::coder::discuss::DiscussionMap>,
3124 /// Open-discussion slots, one permit per allowed discussion. A permit is
3125 /// taken BEFORE `coder.discuss.start` does any of its expensive async work
3126 /// and lives inside the [`DiscussionEntry`](crate::coder::discuss::DiscussionEntry)
3127 /// it admitted, so it comes back when the discussion is dropped. Counting
3128 /// the map instead was a TOCTOU check — the count was read, the lock
3129 /// released, and two awaits (substrate bind + runtime build) ran before the
3130 /// insert, so N pipelined starts all passed a cap of N=8.
3131 pub(crate) coder_discussion_slots: Arc<tokio::sync::Semaphore>,
3132 /// In-flight `agents.chat` sessions keyed by `session_id`. See
3133 /// [`ChatSession`] for shape. Populated by `agents.chat`,
3134 /// cleared on terminal `agent.chat.event` or
3135 /// `agents.chat.cancel`. Disconnect cleanup happens in
3136 /// `remove_session` — any in-flight session bound to either the
3137 /// disconnecting host or agent client is dropped so subsequent
3138 /// stray notifications from a respawned agent fall on the floor
3139 /// rather than racing into a stale stream.
3140 pub chat_sessions: Mutex<HashMap<String, ChatSession>>,
3141 /// Per-recipient channel guards for peer messaging (`agents.message`).
3142 ///
3143 /// Keyed by recipient agent id. Bounds the *channel* rather than the
3144 /// sender's authority: size, identical repeats, per-sender rate, and queue
3145 /// depth. Two agents answering each other form a loop that no policy rule
3146 /// catches, because neither is misbehaving — this is what terminates it.
3147 ///
3148 /// Entries are dropped in `remove_session` when the agent detaches, so a
3149 /// restarted agent starts with a clean budget rather than inheriting the
3150 /// rate history of the process that used to own its name.
3151 pub peer_guards: Mutex<HashMap<String, car_peers::DeliveryGuard>>,
3152 /// Earned standing per sending principal — a peer key inbound, a local
3153 /// agent id outbound.
3154 ///
3155 /// **Deliberately not reaped on detach**, unlike [`Self::peer_guards`] one
3156 /// line above. That hook exists so a respawned agent does not inherit a
3157 /// dead process's rate and dedupe budget, which is *channel* state.
3158 /// Standing is the opposite kind: clearing it on disconnect would let a
3159 /// degraded agent wipe its record by reattaching, which is a cheaper
3160 /// erasure than the renaming this design already refuses to allow.
3161 /// Bounded by TTL and map cap instead, pruned on write.
3162 pub peer_standing: Mutex<HashMap<String, crate::peers::PeerStanding>>,
3163 /// Peer messages held for operator approval, oldest first.
3164 ///
3165 /// A message is held when the sender's `read_only` posture is
3166 /// `require_approval`. Bounded at [`car_peers::HOLD_CAP`]; past that the
3167 /// oldest is dropped, so a stream of held messages cannot grow without
3168 /// limit. Kept flat rather than per-recipient because approval is an
3169 /// operator action over the whole queue, not a per-agent one.
3170 ///
3171 /// Deliberately in memory only: a held message is a live decision awaiting a
3172 /// human, and one that outlived a daemon restart would be delivered into a
3173 /// world that had moved on.
3174 pub held_peer_messages: Mutex<std::collections::VecDeque<crate::peers::HeldPeerMessage>>,
3175 /// Live view of CAR daemons discovered on the local network.
3176 ///
3177 /// `None` when mDNS could not start (no multicast, a locked-down sandbox) —
3178 /// LAN discovery is then simply absent rather than the daemon failing to
3179 /// boot. A network with no peers and a network CAR cannot browse look the
3180 /// same to a caller, which is why the status surface reports which it is.
3181 pub lan_discovery: std::sync::Mutex<Option<car_a2a::lan::LanDirectory>>,
3182 /// This daemon's ed25519 peer identity, and the keys it accepts.
3183 ///
3184 /// The identity signs outbound cross-host requests; the trust set decides
3185 /// which inbound signers are CAR. Both are `None` until the A2A surface
3186 /// starts, because neither is meaningful without a network surface to
3187 /// authenticate.
3188 pub peer_identity: std::sync::Mutex<Option<Arc<car_a2a::peer_auth::PeerIdentity>>>,
3189 pub peer_trust: car_a2a::peer_auth::PeerTrust,
3190 /// In-process collectors for `agent.chat` streams that have **no** host UI
3191 /// to forward to — currently the A2A conversational bridge, which reverse-
3192 /// calls `agent.chat` on a host session and aggregates the streamed deltas
3193 /// into a single reply. Keyed by `session_id`. When a collector exists for a
3194 /// session, `try_forward_agent_chat_event` feeds chunks here instead of
3195 /// forwarding to a host channel. The collecting task owns the entry's
3196 /// lifetime (inserts before the reverse-call, removes when done/timed out).
3197 pub chat_collectors: Mutex<HashMap<String, ChatCollector>>,
3198 /// Standing deterministic chat goals keyed by `session_id`. This is a
3199 /// host-facing status/control registry, separate from ephemeral
3200 /// `chat_sessions` routing so a goal can be inspected after a turn finishes.
3201 pub chat_goals: Mutex<HashMap<String, ChatGoalState>>,
3202 /// Agent runs keyed by `run_id` (agent run tracing, U1). Process-
3203 /// wide (not per-session) so a run's record outlives the WS
3204 /// connection that produced it — the durable, connection-
3205 /// independent boundary `client_id` cannot be (R1). Populated by
3206 /// `runs.start`, made terminal by `runs.complete`, and swept to
3207 /// `Incomplete` on a mid-run disconnect past the grace window
3208 /// (R5). U2/U3 build the per-turn recorder and disk store on top
3209 /// of this registry.
3210 pub runs: Mutex<HashMap<String, RunMeta>>,
3211 /// Per-run serialization for trace append/fsync and terminal persistence.
3212 /// Filesystem work never holds the global [`runs`](Self::runs) lock.
3213 run_durability_locks: Mutex<HashMap<String, std::sync::Weak<Mutex<()>>>>,
3214 /// Live `runs.trace.event` subscribers keyed by `(run_id,
3215 /// host_client_id)` (agent run tracing, U4). Each value is a
3216 /// [`crate::host::RunTraceSubscriber`] — the producer side of a
3217 /// bounded channel whose dedicated drain task writes frames to that
3218 /// connection's WS. Two CarHost windows on one run register two
3219 /// distinct entries (explicit fanout — the built-in notification
3220 /// registry is single-subscriber-per-method).
3221 ///
3222 /// **Lock contract (invariant #1):** `runs.subscribe` snapshots the
3223 /// run's turns AND inserts the subscriber while holding the
3224 /// [`runs`](Self::runs) lock; the recorder (`record_run_turns`),
3225 /// `start_run`, `complete_run`, and `mark_run_incomplete` append to
3226 /// `runs` and push to `run_subscribers` while holding the SAME
3227 /// `runs` lock — so snapshot/register and append/notify are
3228 /// serialized. No turn appended in the snapshot/register window is
3229 /// dropped (gap) or double-delivered (dup). The lock order is always
3230 /// `runs` → `run_subscribers`; never the reverse.
3231 pub run_subscribers: Mutex<HashMap<(String, String), crate::host::RunTraceSubscriber>>,
3232 /// Every browser the drawer can reach (`browser.view.*`), keyed by
3233 /// conversation/agent-session — plus the one standing user session every
3234 /// conversation without an agent-attached browser shares. Owns the
3235 /// per-view snapshot/cursor/subscriber fanout; see
3236 /// [`crate::browser_view`].
3237 pub browser_views: Arc<crate::browser_view::BrowserViewRegistry>,
3238 /// Disk-backed run-trace store (agent run tracing, U3). Source of
3239 /// truth for REPLAY (U5) — persists each run's `RunStarted`, turns,
3240 /// and terminal record as JSONL under `~/.car/runs/{agent_id}/` so a
3241 /// run survives daemon restarts (R4), bounded by retention/GC (R6)
3242 /// and protected at rest with `0600`/`0700` perms + backup exclusion
3243 /// (R14). The in-memory [`runs`](Self::runs) buffer stays the source
3244 /// for the LIVE stream (U4); this store mirrors what was recorded.
3245 /// Derived from `journal_dir` at construction (sibling `runs/`).
3246 pub run_store: crate::run_store::RunStore,
3247 journal_failures: Option<car_eventlog::JournalFailureInjector>,
3248 /// Bound MCP HTTP-streamable URL (e.g.
3249 /// `"http://127.0.0.1:9102/mcp"`) — `car-server` installs this
3250 /// after binding the listener. Used by the
3251 /// `agents.invoke_external` handler to default
3252 /// `InvokeOptions.mcp_endpoint` so external agents
3253 /// (Claude Code today) load the daemon's CAR namespace via
3254 /// `--mcp-config` automatically. `None` when MCP isn't bound
3255 /// (e.g. `--mcp-bind disabled`).
3256 pub mcp_url: std::sync::OnceLock<String>,
3257 /// Approval gate for high-risk WS methods (audit 2026-05). The
3258 /// gate intercepts `automation.run_applescript`,
3259 /// `automation.shortcuts.run`, `messages.send`, `mail.send`, and
3260 /// `vision.ocr` before they dispatch, raises a
3261 /// `host.create_approval` for the user to act on, and waits
3262 /// (with a timeout) for `host.resolve_approval`. Approve →
3263 /// dispatch continues; deny / timeout → JSON-RPC error code
3264 /// `-32003`. The set of gated methods and the wait timeout are
3265 /// embedder-overridable via
3266 /// [`ServerStateConfig::with_approval_gate`].
3267 pub approval_gate: ApprovalGate,
3268 /// Out-of-process supervision (proposal item 5). Holds the subscribed
3269 /// supervisors and the intents parked on their verdicts; the matching
3270 /// [`crate::supervision::SupervisionGate`] is registered as an
3271 /// `AdmissionGate` on every session runtime alongside
3272 /// `StaticVerificationGate`.
3273 ///
3274 /// Daemon-wide rather than per-session, deliberately: a supervisor connects
3275 /// on its OWN WebSocket and supervises proposals submitted on OTHER
3276 /// sessions. A per-session registry would only ever see the supervisor's
3277 /// own (empty) traffic.
3278 ///
3279 /// Costs nothing until someone subscribes — the gate returns `Allow`
3280 /// without building an intent while the subscriber set is empty.
3281 pub supervision: Arc<crate::supervision::SupervisionRegistry>,
3282 /// The daemon-wide shared HITL approval ledger (kernel review C1) —
3283 /// journal-backed at `~/.car/approvals.jsonl` (override via
3284 /// [`ServerStateConfig::with_approval_journal`]). ONE ledger for all
3285 /// sessions: `permission.approve`/`reject` on any connection records
3286 /// here, and every ledger consumer (`permission.evaluate`/`pending`,
3287 /// `evolution.run`, `cascade.run`, `skill.enforce_deployment`/
3288 /// `ingest_governed`) reads here — so an approval granted by a host
3289 /// connection is visible to the agent connection that surfaced it and
3290 /// survives daemon restart. Per-session gates keep only tier/classifier
3291 /// state (see [`ClientSession::permission_gate`]). Falls back to an
3292 /// in-memory ledger (loudly warned, approvals NOT restart-durable) only
3293 /// when the journal is unopenable.
3294 pub approval_ledger: Arc<tokio::sync::RwLock<car_policy::ApprovalLedger>>,
3295 /// The in-process harness evaluator, when the binary installed one — the
3296 /// thing that lets `evolution.run` grade a harness candidate ITSELF
3297 /// instead of waiting for an operator to run `car-bench-harness` twice and
3298 /// hand both files back in.
3299 ///
3300 /// `None` on a build that did not install one (every embedder, and any
3301 /// binary other than `car-server`): `evolution.run` with `harness_measure`
3302 /// then ERRS rather than quietly falling back to HITL, because an opt-in
3303 /// that silently does nothing would report an unattended cycle that never
3304 /// measured anything.
3305 ///
3306 /// A `std::sync::RwLock` on purpose — no caller holds it across an await;
3307 /// they clone the `Arc` out and drop the guard.
3308 pub harness_measurer: std::sync::RwLock<Option<Arc<dyn crate::evolution::HarnessMeasurer>>>,
3309 /// A2A-runtime / store / card factory carried over from the
3310 /// embedder's [`ServerStateConfig`]. Consumed lazily on first
3311 /// `a2a_dispatcher()` call so embedders can construct
3312 /// `ServerState` without paying the runtime spin-up cost when
3313 /// they don't actually use the A2A surface.
3314 pub(crate) a2a_runtime: std::sync::Mutex<Option<Arc<car_engine::Runtime>>>,
3315 pub(crate) a2a_store: std::sync::Mutex<Option<Arc<dyn car_a2a::TaskStore>>>,
3316 pub(crate) a2a_card_source: std::sync::Mutex<Option<Arc<car_a2a::AgentCardSource>>>,
3317}
3318
3319/// Default location of the shared approval-ledger journal: `approvals.jsonl`
3320/// directly under the CAR state root, so it follows `CAR_HOME` rather than
3321/// pinning itself to `~/.car` (which is still where it lands when `CAR_HOME` is
3322/// unset — HOME, or USERPROFILE on Windows). Creates the root best-effort;
3323/// `None` when neither `CAR_HOME` nor a home directory is resolvable.
3324fn default_approval_journal_path() -> Option<PathBuf> {
3325 let dir = car_home::root()?;
3326 let _ = std::fs::create_dir_all(&dir);
3327 Some(dir.join("approvals.jsonl"))
3328}
3329
3330/// Resolve the peer audit journal once, while constructing [`ServerState`].
3331///
3332/// The standalone daemon's default journal directory is `<CAR_HOME>/journals`,
3333/// so preserve the established `<CAR_HOME>/peer-messages.jsonl` location in
3334/// that configuration. A custom journal directory — including every hermetic
3335/// test tempdir — owns its peer audit journal directly. No writer re-reads
3336/// `CAR_HOME`, so process-global state cannot redirect a configured test state.
3337fn default_peer_audit_journal_path(journal_dir: &Path) -> PathBuf {
3338 if let Some(car_dir) = car_home::root() {
3339 if journal_dir == car_dir.join("journals") {
3340 return car_dir.join("peer-messages.jsonl");
3341 }
3342 }
3343 journal_dir.join("peer-messages.jsonl")
3344}
3345
3346/// The daemon's state root: `CAR_HOME` when set, otherwise `~/.car` (HOME, or
3347/// USERPROFILE on Windows).
3348///
3349/// One convention, used by [`default_approval_journal_path`], the `messaging.*`
3350/// handlers and now the project policy loader — a second one would mean an
3351/// operator's rules living somewhere their pairing config does not. Going
3352/// through `car_home` is what keeps that true once an operator relocates the
3353/// root: everything moves together or nothing does.
3354fn car_home_dir() -> Option<PathBuf> {
3355 car_home::root()
3356}
3357
3358/// Register the declarative rules in `<car_dir>/policies/*.toml` on `runtime`.
3359///
3360/// **A malformed file is fatal, on purpose. Do not downgrade this to a
3361/// warning.** The loader's own words (`car_policy::load_policy_dir`): "a
3362/// security rule that fails to parse must surface, never be silently skipped."
3363/// A rule file exists to *stop* something; a typo that turns "never message
3364/// anyone outside the allowlist" into a no-op fails open, and it fails open
3365/// silently, at exactly the moment the operator believes they are protected. A
3366/// daemon that refuses to open the session is loud, immediate, and trivially
3367/// fixed; a daemon that logs a warning nobody reads is neither. The error text
3368/// names the offending file (`PolicyLoadError`'s `Display`) so the fix is
3369/// mechanical.
3370///
3371/// A **missing** directory is not a failure — most projects have no rules, and
3372/// `load_policy_dir` already returns an empty set for it.
3373///
3374/// Split out of [`ServerState::create_session`] so the fatal-vs-silent
3375/// behaviour is testable against a temp directory instead of the real `$HOME`.
3376pub(crate) async fn apply_project_policies(
3377 runtime: &Runtime,
3378 car_dir: &std::path::Path,
3379) -> Result<(), String> {
3380 match runtime.load_project_policies(car_dir).await {
3381 Ok(_) => Ok(()),
3382 Err(e) => Err(format!(
3383 "refusing to start with unreadable project policy rules: {e}. \
3384 Fix or remove the file — a policy rule that fails to parse is a \
3385 security control that would silently not exist."
3386 )),
3387 }
3388}
3389
3390impl ServerState {
3391 async fn run_durability_lock(&self, run_id: &str) -> Arc<Mutex<()>> {
3392 let mut locks = self.run_durability_locks.lock().await;
3393 if let Some(lock) = locks.get(run_id).and_then(std::sync::Weak::upgrade) {
3394 return lock;
3395 }
3396 locks.retain(|_, lock| lock.strong_count() > 0);
3397 let lock = Arc::new(Mutex::new(()));
3398 locks.insert(run_id.to_string(), Arc::downgrade(&lock));
3399 lock
3400 }
3401
3402 async fn quarantine_run_trace_locked(
3403 &self,
3404 runs: &mut HashMap<String, RunMeta>,
3405 run_id: &str,
3406 detail: String,
3407 ) -> String {
3408 let message = run_trace_corruption_message(run_id, detail);
3409 if let Some(meta) = runs.get_mut(run_id) {
3410 if meta.trace_corruption.as_deref() != Some(message.as_str()) {
3411 meta.durability_generation = meta.durability_generation.wrapping_add(1);
3412 }
3413 meta.trace_corruption = Some(message.clone());
3414 }
3415 let mut subs = self.run_subscribers.lock().await;
3416 subs.retain(|(subscribed_run, _), _| subscribed_run != run_id);
3417 message
3418 }
3419
3420 /// Publish corruption found by a strict replay read into live lifecycle
3421 /// state. Disk scanning and marker persistence happen before this call;
3422 /// only the per-run durability gate and short registry/subscriber mutation
3423 /// execute here, so a corrupt trace cannot remain writable or subscribed.
3424 pub(crate) async fn quarantine_run_trace_from_read(
3425 &self,
3426 run_id: &str,
3427 detail: String,
3428 ) -> String {
3429 let durability_lock = self.run_durability_lock(run_id).await;
3430 let _durability_guard = durability_lock.lock().await;
3431 let mut runs = self.runs.lock().await;
3432 self.quarantine_run_trace_locked(&mut runs, run_id, detail)
3433 .await
3434 }
3435
3436 /// Constructor for the standalone `car-server` binary. Each WS
3437 /// connection gets its own per-session memgine — matches the
3438 /// pre-extraction default and is correct for a single-process
3439 /// daemon serving one user at a time.
3440 ///
3441 /// **Embedders must not call this.** It silently leaves
3442 /// `shared_memgine = None`, which re-introduces the dual-memgine
3443 /// bug U7 was created to prevent (one engine in the embedder, a
3444 /// fresh one inside every WS session). Embedders use
3445 /// [`ServerState::embedded`] instead, which makes the shared
3446 /// engine handle a required argument so it cannot be forgotten.
3447 pub fn standalone(journal_dir: PathBuf) -> Self {
3448 Self::with_config(ServerStateConfig::new(journal_dir))
3449 }
3450
3451 /// The daemon-held multi-device-sync + execution-lease subsystem (B6),
3452 /// lazily opened on first `sync.*`/`lease.*` contact and rooted at
3453 /// `<journal_dir>/sync/`. Serialized first-open (see the [`Self::sync`]
3454 /// field docs): the oplog journal's exclusive advisory lock means only one
3455 /// open may succeed, so the fallible init runs under the field's
3456 /// `std::sync::Mutex` while the subsystem itself is shared behind a
3457 /// `tokio::sync::Mutex`.
3458 pub fn sync_subsystem(
3459 &self,
3460 ) -> Result<Arc<tokio::sync::Mutex<crate::sync::SyncSubsystem>>, String> {
3461 let mut guard = self
3462 .sync
3463 .lock()
3464 .map_err(|_| "sync subsystem init lock poisoned".to_string())?;
3465 if let Some(existing) = guard.as_ref() {
3466 return Ok(existing.clone());
3467 }
3468 let root = self.journal_dir.join("sync");
3469 let subsystem = self.open_sync_subsystem(&root)?;
3470 let arc = Arc::new(tokio::sync::Mutex::new(subsystem));
3471 *guard = Some(arc.clone());
3472 Ok(arc)
3473 }
3474
3475 /// Route a write to the right subsystem BY SCOPE: an opted-in
3476 /// `Scope::Shared{org}` goes to the shared org delivery subsystem (relay scope
3477 /// `org:{orgId}`, so it converges cross-member); everything else — `Personal`,
3478 /// or an org with no opted-in delivery — goes to the personal subsystem, which
3479 /// is byte-identical to before when org-scope is off. Used ONLY at the write
3480 /// sites; every reader keeps calling [`Self::sync_subsystem`]. (8a)
3481 pub fn subsystem_for_scope(
3482 &self,
3483 scope: &car_sync::Scope,
3484 ) -> Result<Arc<tokio::sync::Mutex<crate::sync::SyncSubsystem>>, String> {
3485 // Init the personal subsystem first — this also builds + stores the org
3486 // subsystem when opted in (via `open_sync_subsystem`).
3487 let user = self.sync_subsystem()?;
3488 let guard = self
3489 .org_sync
3490 .lock()
3491 .map_err(|_| "org sync lock poisoned".to_string())?;
3492 match route_for_scope(scope, guard.as_ref().map(|h| h.org.as_str())) {
3493 SyncRoute::Org => Ok(guard
3494 .as_ref()
3495 .expect("route_for_scope returns Org only when a holder exists")
3496 .subsystem
3497 .clone()),
3498 SyncRoute::User => Ok(user),
3499 }
3500 }
3501
3502 /// The opted-in shared org delivery subsystem, if any — for pumping both scopes.
3503 pub fn org_subsystem(&self) -> Option<Arc<tokio::sync::Mutex<crate::sync::SyncSubsystem>>> {
3504 match self.org_sync.lock() {
3505 Ok(g) => g.as_ref().map(|h| h.subsystem.clone()),
3506 // Surface the poison loudly — do NOT silently degrade to personal-only
3507 // sync (linus B). Poison only occurs on a panic while locked.
3508 Err(_) => {
3509 tracing::error!(
3510 "org_sync lock poisoned — skipping the org pump (org-scope delivery degraded)"
3511 );
3512 None
3513 }
3514 }
3515 }
3516
3517 /// Build the sync subsystem for `root`, selecting the backend from config.
3518 ///
3519 /// Default is the local shared-directory `FsRelay`. When
3520 /// `PARSLEE_SYNC_BACKEND=parslee` (env or keychain), it builds the
3521 /// Parslee-backed device: a `ParsleeSyncTransport` at
3522 /// `PARSLEE_CAR_SYNC_BASE_URL` (default `{api_base}/sync`), authed with the
3523 /// live refresh-aware bearer, scoped `user:{account_id}`, E2E-encrypted under
3524 /// a passphrase-derived key (`PARSLEE_SYNC_PASSPHRASE`, the zero-knowledge
3525 /// cross-device key — same passphrase → same key on every device, never sent
3526 /// to the server). Missing a Parslee login falls back to local (logged);
3527 /// missing the passphrase is a hard error (refusing to sync unencrypted).
3528 fn open_sync_subsystem(
3529 &self,
3530 root: &std::path::Path,
3531 ) -> Result<crate::sync::SyncSubsystem, String> {
3532 let backend = car_secrets::resolve_env_or_keychain("PARSLEE_SYNC_BACKEND")
3533 .unwrap_or_default()
3534 .to_lowercase();
3535 if backend != "parslee" {
3536 return crate::sync::SyncSubsystem::open(root);
3537 }
3538
3539 let Some(account_id) = self
3540 .parslee_session
3541 .get()
3542 .map(|s| s.identity.account_id.clone())
3543 .filter(|a| !a.is_empty())
3544 else {
3545 tracing::warn!(
3546 "sync backend=parslee but no Parslee login is present; using local sync until login"
3547 );
3548 return crate::sync::SyncSubsystem::open(root);
3549 };
3550
3551 let Some(passphrase) = car_secrets::resolve_env_or_keychain("PARSLEE_SYNC_PASSPHRASE")
3552 .filter(|p| !p.is_empty())
3553 else {
3554 return Err(
3555 "sync backend=parslee requires PARSLEE_SYNC_PASSPHRASE (the zero-knowledge \
3556 cross-device sync key) — set it in the keychain; refusing to sync unencrypted"
3557 .to_string(),
3558 );
3559 };
3560
3561 let base_url = car_secrets::resolve_env_or_keychain("PARSLEE_CAR_SYNC_BASE_URL")
3562 .filter(|u| !u.is_empty())
3563 .unwrap_or_else(|| {
3564 let api_base = car_auth::api_base(None);
3565 format!("{}/sync", api_base.trim_end_matches('/'))
3566 });
3567
3568 let bearer: car_parslee::sync_transport::BearerFn = Arc::new(car_auth::access_token);
3569 let transport = Arc::new(
3570 car_parslee::sync_transport::ParsleeSyncTransport::new(base_url, bearer)
3571 .map_err(|e| format!("sync: build Parslee transport: {e}"))?,
3572 );
3573 // Argon2id-stretch the passphrase into the master ONCE at open (memory-hard
3574 // work factor for a password-equivalent secret), then feed BOTH the
3575 // per-audience provider and the org identity from it — so Argon2id runs
3576 // exactly once, not once per derivation.
3577 let master = car_sync::StretchedMaster::from_passphrase(passphrase.as_bytes(), &account_id);
3578 let base: Arc<dyn car_sync::SyncKeyProvider> =
3579 Arc::new(car_sync::DerivedKeyProvider::from_master(&master));
3580
3581 // Org-scope shared-brain activation — REAL provisioning, but DEFAULT-OFF.
3582 // `PARSLEE_SYNC_ORG_SCOPE=<orgId>:<granter_hex>[,<granter_hex>...]` names the
3583 // org AND the admin-designated trusted granter key(s), held LOCALLY (never
3584 // backend-fetched — see `parse_org_scope_config`). Unset → `base`
3585 // (byte-identical to the personal-only path). Malformed → hard error above.
3586 // The configured org MUST match the signed-in `active_organization` or we
3587 // refuse loudly. Turning it ON for a real tenant is slice 12, still gated on
3588 // the cryptographer audit of the login_secret→identity entropy dependency.
3589 let mut pending_org: Option<OrgSyncHolder> = None;
3590 let provider: Arc<dyn car_sync::SyncKeyProvider> = match Self::parse_org_scope_config(
3591 car_secrets::resolve_env_or_keychain("PARSLEE_SYNC_ORG_SCOPE"),
3592 )? {
3593 None => base,
3594 Some((org_id, granters)) => {
3595 // Bind the configured org to the signed-in active organization — never
3596 // encrypt org ops for an org this account is not actually a member of.
3597 let active_org = self
3598 .parslee_session
3599 .get()
3600 .and_then(|s| s.identity.active_organization.clone())
3601 .filter(|o| !o.is_empty());
3602 if active_org.as_deref() != Some(org_id.as_str()) {
3603 return Err(format!(
3604 "PARSLEE_SYNC_ORG_SCOPE org {org_id:?} does not match the signed-in active \
3605 organization {active_org:?} — refusing to activate org scope"
3606 ));
3607 }
3608
3609 // Bring the directory trait into scope for `publish_pubkey`.
3610 use car_sync::OrgKeyDirectory as _;
3611 let my_secret = car_sync::derive_x25519_identity(&master, &account_id);
3612 let mut directory = car_sync::NetworkOrgKeyDirectory::new(
3613 transport.clone(),
3614 format!("org:{org_id}"),
3615 );
3616
3617 // Announce our X25519 identity pubkey so a granter can wrap K_org for
3618 // us (idempotent LWW). Non-fatal: a failure just means future grants
3619 // may not reach us; any already-resolved roots still work. This routes
3620 // to the bearer-authenticated backend, but whether the backend
3621 // ENFORCES "only this account may publish its own pubkey" (and "only a
3622 // K_org holder may publish a wrap") is the confidentiality-critical,
3623 // AUDIT-OWNED check — this is not a guarantee against a hostile backend.
3624 let my_pub = car_sync::x25519_public(&my_secret);
3625 let my_pub_hex: String = my_pub
3626 .as_bytes()
3627 .iter()
3628 .map(|b| format!("{b:02x}"))
3629 .collect();
3630 if let Err(e) = directory.publish_pubkey(&account_id, &my_pub_hex) {
3631 tracing::warn!(org = %org_id, error = %e, "org-scope: failed to publish member pubkey — future grants may not reach this member");
3632 }
3633
3634 // Resolve EVERY epoch of K_org we can unwrap (slice 10) → the
3635 // multi-epoch map the cipher selects by kid. `org → { epoch → K_org }`.
3636 // Fail-closed: Err (directory unreachable) and empty (ungranted) both
3637 // leave org scope on DenyCipher, NEVER demoted to the personal key.
3638 let mut roots = std::collections::HashMap::new();
3639 match car_sync::resolve_all_org_roots(
3640 &directory,
3641 &org_id,
3642 &my_secret,
3643 &account_id,
3644 &granters,
3645 ) {
3646 Ok(map) if !map.is_empty() => {
3647 tracing::info!(org = %org_id, epochs = ?map.keys().collect::<Vec<_>>(), "org-scope: resolved K_org generations");
3648 roots.insert(org_id.clone(), map);
3649 }
3650 Ok(_) => tracing::warn!(
3651 org = %org_id,
3652 "org-scope: no trusted grant for this member — org scope fails closed (DenyCipher)"
3653 ),
3654 Err(e) => tracing::warn!(
3655 org = %org_id, error = %e,
3656 "org-scope: org-key directory unreachable — org scope fails closed (DenyCipher), NOT demoted to the personal key"
3657 ),
3658 }
3659 // ALWAYS OrgAwareKeyProvider once opted in — NEVER the bare
3660 // DerivedKeyProvider, which has no Scope::Shared case and would
3661 // silently encrypt org ops under the PERSONAL key (a fail-open).
3662 let provider: Arc<dyn car_sync::SyncKeyProvider> =
3663 Arc::new(car_sync::OrgAwareKeyProvider::new(base, roots));
3664
3665 // Open the SHARED org delivery subsystem: a SECOND device oplog
3666 // under `root/org-{orgId}/` whose relay scope is `org:{orgId}`, with
3667 // the SAME OrgAwareKeyProvider (it keys Scope::Shared{org} on K_org).
3668 // Scope::Shared{org} writes route here (subsystem_for_scope) and
3669 // converge across the org's members. `org_id` is a canonical slug
3670 // (validated on parse), so it is filesystem-safe. (8a)
3671 let org_root = root.join(format!("org-{org_id}"));
3672 let org_subsystem = crate::sync::SyncSubsystem::open_remote(
3673 &org_root,
3674 transport.clone(),
3675 format!("org:{org_id}"),
3676 provider.clone(),
3677 )?;
3678 // Do NOT commit to `org_sync` yet — hold the org subsystem in a
3679 // local and only install it AFTER the personal open below succeeds.
3680 // Else a failed personal open would orphan the org journal's
3681 // exclusive lock and permanently wedge sync retry (linus A).
3682 pending_org = Some(OrgSyncHolder {
3683 org: org_id.clone(),
3684 subsystem: Arc::new(tokio::sync::Mutex::new(org_subsystem)),
3685 });
3686 tracing::info!(org = %org_id, "org-scope: opened shared org delivery subsystem (relay org:{org_id})");
3687 provider
3688 }
3689 };
3690 let user = crate::sync::SyncSubsystem::open_remote(
3691 root,
3692 transport,
3693 format!("user:{account_id}"),
3694 provider,
3695 )?;
3696 // Personal open succeeded — NOW commit the org holder. On the failure path
3697 // above, `pending_org` drops here, releasing the org journal lock so a
3698 // retry can re-open cleanly.
3699 if let Some(holder) = pending_org {
3700 *self
3701 .org_sync
3702 .lock()
3703 .map_err(|_| "org sync init lock poisoned".to_string())? = Some(holder);
3704 }
3705 Ok(user)
3706 }
3707
3708 /// Parse `PARSLEE_SYNC_ORG_SCOPE` (`<orgId>:<granter_hex>[,<granter_hex>...]`).
3709 /// `Ok(None)` when unset/empty → the personal-only path. `Err` on ANY malformed
3710 /// value — a typo must be a loud failure, never a silent fall-through to a
3711 /// wrong or unauthenticated state. Returns the org id + the (≥1) admin-designated
3712 /// trusted granter verifying keys.
3713 ///
3714 /// The granter set is held LOCALLY (operator-provisioned), NEVER fetched from
3715 /// the backend: whoever fills the `trusted` slice holds the authority, and if
3716 /// the same backend that serves the directory could name the granters it could
3717 /// name ITSELF a granter and substitute `K_org`. A backend-sourced / signed
3718 /// granter manifest is an audit-owned trust decision (slice 12), not this.
3719 /// Pure over the raw value so it is unit-testable without the environment.
3720 fn parse_org_scope_config(
3721 raw: Option<String>,
3722 ) -> Result<Option<(String, Vec<car_sync::OrgVerifyingKey>)>, String> {
3723 let Some(raw) = raw.filter(|s| !s.is_empty()) else {
3724 return Ok(None);
3725 };
3726 let (org_id, granters_csv) = raw.split_once(':').ok_or_else(|| {
3727 "PARSLEE_SYNC_ORG_SCOPE must be <orgId>:<granter_ed25519_pubkey_hex>[,...]".to_string()
3728 })?;
3729 // Honor the "loud failure" rule: a non-canonical org id (space, Unicode,
3730 // the platform's raw tenant form) must error here, not silently resolve
3731 // nothing — it would never match a real wrap's canonical `org` anyway.
3732 car_sync::require_canonical_org(org_id)
3733 .map_err(|e| format!("PARSLEE_SYNC_ORG_SCOPE: bad orgId: {e}"))?;
3734 let granters = granters_csv
3735 .split(',')
3736 .map(str::trim)
3737 .filter(|h| !h.is_empty())
3738 .map(|hex| {
3739 car_sync::parse_ed25519_verifying(hex)
3740 .map_err(|e| format!("PARSLEE_SYNC_ORG_SCOPE: bad granter key {hex:?}: {e}"))
3741 })
3742 .collect::<Result<Vec<_>, _>>()?;
3743 if granters.is_empty() {
3744 return Err(
3745 "PARSLEE_SYNC_ORG_SCOPE: at least one granter ed25519 pubkey is required".into(),
3746 );
3747 }
3748 Ok(Some((org_id.to_string(), granters)))
3749 }
3750
3751 /// Persist the daemon-wide standing chat-goal registry. The snapshot is
3752 /// taken under the async mutex, then serialized and atomically rewritten on
3753 /// the blocking pool so chat/event dispatch does not block a Tokio worker.
3754 pub async fn persist_chat_goals(&self) -> Result<(), String> {
3755 let path = chat_goals_path_from_journal_dir(&self.journal_dir);
3756 let goals = self.chat_goals.lock().await.clone();
3757 tokio::task::spawn_blocking(move || {
3758 write_chat_goals_atomic(&path, &goals)
3759 .map_err(|e| format!("write {}: {e}", path.display()))
3760 })
3761 .await
3762 .map_err(|e| format!("persist chat goals join: {e}"))?
3763 }
3764
3765 /// Constructor for embedders (e.g. `tokhn-daemon`). The shared
3766 /// memgine handle is **required**: every WS session created by
3767 /// this state will reuse the same engine, preventing the
3768 /// dual-memgine bug.
3769 ///
3770 /// For embedders that also want to inject a pre-warmed inference
3771 /// engine or other advanced wiring, build a [`ServerStateConfig`]
3772 /// directly and call [`ServerState::with_config`].
3773 pub fn embedded(
3774 journal_dir: PathBuf,
3775 shared_memgine: Arc<Mutex<car_memgine::MemgineEngine>>,
3776 ) -> Self {
3777 Self::with_config(ServerStateConfig::new(journal_dir).with_shared_memgine(shared_memgine))
3778 }
3779
3780 /// Build a `ServerState` from a [`ServerStateConfig`] — the path
3781 /// embedders use when they need to inject a shared memgine *and*
3782 /// a pre-warmed inference engine, or any other advanced wiring
3783 /// the convenience constructors don't cover.
3784 pub fn with_config(cfg: ServerStateConfig) -> Self {
3785 Self::try_with_config(cfg)
3786 .unwrap_or_else(|error| panic!("server state startup failed: {error}"))
3787 }
3788
3789 /// Build a state with test-owned feedback diagnostic roots and no ambient
3790 /// registry probes. This bypasses the production diagnostic default rather
3791 /// than constructing it and then overriding it, so state construction never
3792 /// resolves `HOME`/`USERPROFILE` for feedback. Public only because Cargo
3793 /// integration tests link `car-server-core` without `cfg(test)`.
3794 #[doc(hidden)]
3795 pub fn with_config_and_isolated_feedback_diagnostics(
3796 cfg: ServerStateConfig,
3797 models_dir: PathBuf,
3798 huggingface_hub_root: PathBuf,
3799 ) -> Self {
3800 Self::try_with_config_and_feedback_diagnostics(
3801 cfg,
3802 crate::feedback::FeedbackDiagnostics::isolated(models_dir, huggingface_hub_root),
3803 )
3804 .unwrap_or_else(|error| panic!("server state startup failed: {error}"))
3805 }
3806
3807 /// Fallible startup surface used by the daemon before it begins listening.
3808 /// Durable lifecycle outbox replay is acknowledgement-bounded; any
3809 /// durability-unknown result stops construction and leaves its RunStore
3810 /// preimage/quarantine intact for an exact later retry.
3811 pub fn try_with_config(cfg: ServerStateConfig) -> Result<Self, String> {
3812 Self::try_with_config_and_feedback_diagnostics(
3813 cfg,
3814 crate::feedback::FeedbackDiagnostics::production(),
3815 )
3816 }
3817
3818 fn try_with_config_and_feedback_diagnostics(
3819 cfg: ServerStateConfig,
3820 feedback_diagnostics: crate::feedback::FeedbackDiagnostics,
3821 ) -> Result<Self, String> {
3822 let startup_acknowledgement_timeout = cfg.startup_reconciliation_acknowledgement_timeout;
3823 if startup_acknowledgement_timeout.is_zero()
3824 || startup_acknowledgement_timeout > car_eventlog::MAX_CRITICAL_ACKNOWLEDGEMENT_TIMEOUT
3825 {
3826 return Err(format!(
3827 "startup reconciliation acknowledgement timeout must be between 1ns and {}ms, got {}ms",
3828 car_eventlog::MAX_CRITICAL_ACKNOWLEDGEMENT_TIMEOUT.as_millis(),
3829 startup_acknowledgement_timeout.as_millis()
3830 ));
3831 }
3832 let inference = std::sync::OnceLock::new();
3833 if let Some(eng) = cfg.inference {
3834 // OnceLock::set returns Err if already set — fresh OnceLock
3835 // means it's empty, so this is infallible here.
3836 let _ = inference.set(eng);
3837 }
3838 let voice_sessions = Arc::new(car_voice::VoiceSessionRegistry::new());
3839 // Reap sessions whose clients dropped without calling
3840 // voice.transcribe_stream.stop (WS disconnect, process exit,
3841 // etc.). Listener handles otherwise leak for the daemon's
3842 // lifetime. `with_config` is sync but always called from the
3843 // `#[tokio::main]` entry point, so `Handle::try_current()`
3844 // inside `start_sweeper` finds the runtime.
3845 voice_sessions.start_sweeper();
3846 // UI-improvement agent is pure decision logic — no I/O, no
3847 // persistence handle. Memgine ingest of strategy outcomes is
3848 // the caller's responsibility (handler.rs after a successful
3849 // Decision::Patch). Keeps the agent crate Mutex-flavor
3850 // agnostic so it can compose with std/tokio mutex callers.
3851 let ui_agent = Arc::new(car_ui_agent::UIImprovementAgent::with_default_strategies());
3852 let ui_agent_oscillation = Arc::new(crate::ui_agent_loop::OscillationDetector::new());
3853 let ui_agent_budget = Arc::new(crate::ui_agent_loop::IterationBudget::new());
3854 // Disk-backed run-trace store (U3). Rooted at the `runs/` sibling
3855 // of the journal dir (`~/.car/runs`), with retention read from
3856 // `~/.car/config.toml`. Boot-time GC enforces the retention cap
3857 // (R6) — best-effort; a failed GC must never block startup, so the
3858 // count is dropped. Never evicts an in-progress run.
3859 let mut run_store = crate::run_store::RunStore::from_journal_dir(&cfg.journal_dir);
3860 if let Some(failures) = cfg.run_store_failures.clone() {
3861 run_store = run_store.with_failure_injector(failures);
3862 }
3863 if let Some(gate) = cfg.run_store_summary_read_gate.clone() {
3864 run_store = run_store.with_summary_read_gate(gate);
3865 }
3866 if let Some(gate) = cfg.run_store_summary_write_gate.clone() {
3867 run_store = run_store.with_summary_write_gate(gate);
3868 }
3869 if let Some(gate) = cfg.run_store_append_gate.clone() {
3870 run_store = run_store.with_append_gate(gate);
3871 }
3872 if let Some(gate) = cfg.run_store_lookup_gate.clone() {
3873 run_store = run_store.with_lookup_gate(gate);
3874 }
3875 if let Some(failures) = cfg.run_store_private_path_failures.clone() {
3876 run_store = run_store.with_private_path_failure_injector(failures);
3877 }
3878 run_store
3879 .reconcile_proposal_retry_rollbacks()
3880 .map_err(|error| format!("proposal retry rollback reconciliation failed: {error}"))?;
3881 let chat_goals = load_chat_goals_from_disk(&cfg.journal_dir);
3882 let selfheal_ledger = cfg.selfheal_ledger.clone().unwrap_or_else(|| {
3883 cfg.journal_dir
3884 .parent()
3885 .unwrap_or(&cfg.journal_dir)
3886 .join("selfheal")
3887 .join("detections.jsonl")
3888 });
3889 // Config lives beside the daemon's other state. A missing or empty
3890 // `heal.toml` is a disabled loop, which is the default: there are no
3891 // compiled-in targets, because a loop that opened pull requests in a
3892 // repository nobody named would be a surprise in somebody else's
3893 // tracker.
3894 let heal_state_dir =
3895 crate::coder::rpc::coder_state_dir().unwrap_or_else(|_| cfg.journal_dir.join("coder"));
3896 let heal_config_dir = car_home::root_or_relative();
3897 let heal = Arc::new(crate::coder::heal_service::HealService::with_config_dir(
3898 crate::coder::heal_config::HealConfig::load(&heal_config_dir),
3899 heal_state_dir,
3900 heal_config_dir,
3901 ));
3902 let selfheal = Arc::new(crate::selfheal::SelfhealService::open(
3903 selfheal_ledger,
3904 cfg.selfheal_interval_secs,
3905 cfg.selfheal_evidence.clone(),
3906 cfg.selfheal_source_probe.clone(),
3907 cfg.selfheal_replay_verb_probe.clone(),
3908 )?);
3909 // Drop terminal history outside the configured retention window before
3910 // lifecycle replay. This keeps the reconciliation set bounded even
3911 // after years of newsroom runs; in-progress traces are never evicted.
3912 let _evicted_before_replay = run_store.gc();
3913 let mut startup_journals =
3914 StartupJournalCache::new(&cfg.journal_dir, cfg.journal_failures.as_ref());
3915 // Establish the durable RunStarted bracket before replaying a pending
3916 // proposal terminal. Then adopt the crash orphan and run lifecycle
3917 // reconciliation again to add RunCompleted. Both passes share the
3918 // per-client journal cache, are exact-row idempotent, and preserve:
3919 // RunStarted -> ProposalCompleted -> RunCompleted.
3920 reconcile_durable_run_journals(
3921 &mut startup_journals,
3922 &run_store,
3923 startup_acknowledgement_timeout,
3924 )?;
3925 reconcile_pending_proposal_journals(
3926 &mut startup_journals,
3927 &run_store,
3928 startup_acknowledgement_timeout,
3929 )?;
3930 reconcile_completed_proposal_guards(&run_store);
3931 // The in-memory map is empty at boot, so any remaining on-disk
3932 // InProgress run is a crashed prior process; mark it Incomplete so it
3933 // is terminal and age-GC-eligible (FIX 4).
3934 let _adopted = run_store.adopt_orphans();
3935 reconcile_durable_run_journals(
3936 &mut startup_journals,
3937 &run_store,
3938 startup_acknowledgement_timeout,
3939 )?;
3940 let evicted = run_store.gc();
3941 if evicted > 0 {
3942 // Said out loud, for the same reason the coder GC below says it: an
3943 // operator whose own config just ate part of their store has to be
3944 // able to see it happened. This was silent, which is what made a
3945 // zero cap a data-loss incident rather than a log line (car#1338) —
3946 // and `max_age_days = 1` where someone meant `10` still deletes nine
3947 // days of traces, which the age of the policy has nothing to do with.
3948 tracing::info!(
3949 evicted,
3950 max_per_agent = ?run_store.retention().count_cap(),
3951 max_age_days = ?run_store.retention().age_cap_days(),
3952 "pruned run traces (~/.car/config.toml [runs] retention)"
3953 );
3954 }
3955 // Adopt crash/restart-orphaned coder sessions the same way: the
3956 // in-memory `coder_sessions` map is empty at boot, so any non-terminal
3957 // snapshot under the coder state dir is a prior process's orphan. Mark
3958 // genuinely-stranded ones Failed ("daemon restarted mid-session") and
3959 // preserve a `needs_approval` orphan whose worktree still exists so the
3960 // user can inspect/approve it by hand. Best-effort — never blocks boot.
3961 if let Ok(coder_dir) = crate::coder::rpc::coder_state_dir() {
3962 let _coder_adopted = crate::coder::session::adopt_orphaned_sessions(&coder_dir);
3963 // Then prune, the same way `run_store.gc()` above prunes run
3964 // traces. Adoption runs first on purpose: it is what turns a
3965 // stranded orphan terminal, and only terminal sessions are
3966 // collectable. An orphan adopted just now carries a fresh
3967 // `updated_at`, so this boot never collects what this boot
3968 // rewrote — it ages out on a later one.
3969 let retention = crate::coder::config::CoderConfig::load().session_retention();
3970 let collected = crate::coder::session::gc_sessions(
3971 &coder_dir,
3972 &retention,
3973 crate::coder::session::SweepScope::Boot,
3974 );
3975 if collected > 0 {
3976 // Said out loud, unlike `run_store.gc()` above. That policy has
3977 // existed since the run store did; this one is NEW and applies
3978 // retroactively, so the first boot after an upgrade can delete
3979 // years of snapshots under caps the operator never chose. An
3980 // operator who did not want that has to be able to see it
3981 // happened.
3982 tracing::info!(
3983 collected,
3984 max_sessions = retention.max_sessions,
3985 max_age_days = retention.max_age_days,
3986 "pruned coder session snapshots (~/.car/coder.toml retention)"
3987 );
3988 }
3989 }
3990 // The daemon-wide shared approval ledger (kernel review C1): one
3991 // journal-backed store for every session's HITL records, so approvals
3992 // cross connections and survive restart. In-memory fallback is loud —
3993 // a silent downgrade here would fake durability.
3994 let peer_audit_journal = cfg
3995 .peer_audit_journal
3996 .clone()
3997 .unwrap_or_else(|| default_peer_audit_journal_path(&cfg.journal_dir));
3998 let approval_ledger = {
3999 let path = cfg.approval_journal.or_else(default_approval_journal_path);
4000 match path {
4001 Some(p) => {
4002 // The journal's parent must exist before the first
4003 // append-mode open (ApprovalLedger::record does not
4004 // create directories).
4005 if let Some(parent) = p.parent() {
4006 let _ = std::fs::create_dir_all(parent);
4007 }
4008 match car_policy::ApprovalLedger::with_journal(&p) {
4009 Ok(l) => {
4010 if l.skipped_on_load() > 0 {
4011 tracing::warn!(
4012 path = %p.display(),
4013 skipped = l.skipped_on_load(),
4014 "approval-ledger journal had unparseable lines; partial ledger loaded"
4015 );
4016 }
4017 l
4018 }
4019 Err(e) => {
4020 tracing::warn!(
4021 path = %p.display(), error = %e,
4022 "approval-ledger journal unopenable; falling back to IN-MEMORY \
4023 ledger — approvals will NOT survive restart"
4024 );
4025 car_policy::ApprovalLedger::new()
4026 }
4027 }
4028 }
4029 None => {
4030 tracing::warn!(
4031 "no CAR_HOME/HOME/USERPROFILE to place approvals.jsonl under the CAR \
4032 state root; approval ledger is IN-MEMORY — approvals will NOT survive \
4033 restart"
4034 );
4035 car_policy::ApprovalLedger::new()
4036 }
4037 }
4038 };
4039 // Built before the struct literal so the browser-view registry can be
4040 // pointed at the SAME `HostState` it will broadcast sign-in attention
4041 // on (`host.event`) — see `crate::browser_attention`.
4042 let host = Arc::new(crate::host::HostState::new());
4043 let browser_views = Arc::new(crate::browser_view::BrowserViewRegistry::default());
4044 browser_views.set_signin_attention(Arc::new(
4045 crate::browser_attention::HostSignInAttention::new(Arc::clone(&host)),
4046 ));
4047 Ok(Self {
4048 journal_dir: cfg.journal_dir,
4049 peer_audit_journal,
4050 sessions: Mutex::new(HashMap::new()),
4051 inference,
4052 host,
4053 shared_memgine: cfg.shared_memgine,
4054 feedback_diagnostics,
4055 trajectory_store: Arc::new(car_memgine::TrajectoryStore::new(
4056 &cfg.trajectory_dir
4057 .unwrap_or_else(car_memgine::TrajectoryStore::default_path),
4058 )),
4059 selfheal,
4060 heal,
4061 voice_sessions,
4062 meetings: Arc::new(car_meeting::MeetingRegistry::new()),
4063 a2ui: car_a2ui::A2uiSurfaceStore::new(),
4064 ui_agent,
4065 ui_agent_oscillation,
4066 ui_agent_budget,
4067 admission: Arc::new(crate::admission::InferenceAdmission::new()),
4068 a2ui_route_auth: Mutex::new(HashMap::new()),
4069 supervisor: std::sync::OnceLock::new(),
4070 declagents: std::sync::OnceLock::new(),
4071 routing: std::sync::OnceLock::new(),
4072 sync: std::sync::Mutex::new(None),
4073 org_sync: std::sync::Mutex::new(None),
4074 observer_manifest_path: std::sync::OnceLock::new(),
4075 a2a_dispatcher: std::sync::OnceLock::new(),
4076 a2a_runtime: std::sync::Mutex::new(cfg.a2a_runtime),
4077 a2a_store: std::sync::Mutex::new(cfg.a2a_store),
4078 a2a_card_source: std::sync::Mutex::new(cfg.a2a_card_source),
4079 a2ui_subscribers: Mutex::new(HashMap::new()),
4080 mcp_executor: Arc::new(car_engine::McpToolExecutor::new()),
4081 connectors: std::sync::OnceLock::new(),
4082 connectors_loaded: std::sync::atomic::AtomicBool::new(false),
4083 channel_supervisor: std::sync::OnceLock::new(),
4084 durable_tasks: Mutex::new(tokio::task::JoinSet::new()),
4085 auth_completion_owner_id: uuid::Uuid::new_v4().simple().to_string(),
4086 auth_token: std::sync::OnceLock::new(),
4087 host_token: std::sync::OnceLock::new(),
4088 mobile_runtime_url: std::sync::OnceLock::new(),
4089 mobile_registration_url: std::sync::OnceLock::new(),
4090 parslee_session: std::sync::OnceLock::new(),
4091 attached_agents: Mutex::new(HashMap::new()),
4092 agent_memgines: Mutex::new(HashMap::new()),
4093 namespace_memgines: Mutex::new(HashMap::new()),
4094 coder_sessions: Mutex::new(HashMap::new()),
4095 coder_disk_gc_base: std::time::Instant::now(),
4096 // `0` IS "just swept": boot ran the same sweep a few lines above,
4097 // and zero seconds have elapsed since the base taken on the line
4098 // above this one. The first `coder.start` is therefore a no-op
4099 // rather than an immediate re-scan.
4100 coder_disk_gc_at: AtomicU64::new(0),
4101 coder_subscribers: Mutex::new(HashMap::new()),
4102 coder_watchers: Mutex::new(HashMap::new()),
4103 coder_watch_notify: std::sync::OnceLock::new(),
4104 coder_discussions: Mutex::new(HashMap::new()),
4105 coder_discussion_slots: Arc::new(tokio::sync::Semaphore::new(
4106 crate::coder::discuss::MAX_OPEN_DISCUSSIONS,
4107 )),
4108 chat_sessions: Mutex::new(HashMap::new()),
4109 peer_guards: Mutex::new(HashMap::new()),
4110 peer_standing: Mutex::new(HashMap::new()),
4111 held_peer_messages: Mutex::new(std::collections::VecDeque::new()),
4112 lan_discovery: std::sync::Mutex::new(None),
4113 peer_identity: std::sync::Mutex::new(None),
4114 peer_trust: car_a2a::peer_auth::PeerTrust::new(),
4115 chat_collectors: Mutex::new(HashMap::new()),
4116 chat_goals: Mutex::new(chat_goals),
4117 runs: Mutex::new(HashMap::new()),
4118 run_resume_liveness: Mutex::new(()),
4119 run_resume_lease: cfg.run_resume_lease,
4120 run_completion_fence_gate: cfg.run_completion_fence_gate,
4121 run_durability_locks: Mutex::new(HashMap::new()),
4122 run_subscribers: Mutex::new(HashMap::new()),
4123 browser_views,
4124 run_store,
4125 journal_failures: cfg.journal_failures,
4126 mcp_url: std::sync::OnceLock::new(),
4127 approval_gate: cfg.approval_gate.unwrap_or_default(),
4128 supervision: Arc::new(crate::supervision::SupervisionRegistry::default()),
4129 approval_ledger: Arc::new(tokio::sync::RwLock::new(approval_ledger)),
4130 // Installed by the daemon binary at startup (see
4131 // `car-server/src/main.rs`), never here: the implementation lives
4132 // in `car-bench`, which depends on this crate.
4133 harness_measurer: std::sync::RwLock::new(None),
4134 })
4135 }
4136
4137 /// Install the in-process harness evaluator `evolution.run`'s
4138 /// `harness_measure` path grades candidates with. Called once at daemon
4139 /// startup; a later call replaces it.
4140 pub fn set_harness_measurer(&self, m: Arc<dyn crate::evolution::HarnessMeasurer>) {
4141 match self.harness_measurer.write() {
4142 Ok(mut guard) => *guard = Some(m),
4143 // Poisoned only if a panic happened while the lock was held.
4144 // Refusing loudly beats leaving `evolution.run` to believe an
4145 // evaluator is installed when none is.
4146 Err(e) => tracing::error!(
4147 error = %e,
4148 "harness_measurer lock poisoned; the in-process harness evaluator was NOT installed"
4149 ),
4150 }
4151 }
4152
4153 /// The installed harness evaluator, if any. Clones the `Arc` out and drops
4154 /// the guard so callers never hold a `std::sync` lock across an await.
4155 pub fn harness_measurer(&self) -> Option<Arc<dyn crate::evolution::HarnessMeasurer>> {
4156 match self.harness_measurer.read() {
4157 Ok(guard) => guard.clone(),
4158 Err(e) => {
4159 tracing::error!(error = %e, "harness_measurer lock poisoned; treating as absent");
4160 None
4161 }
4162 }
4163 }
4164
4165 /// Start one daemon-owned operation and return a connection-scoped result
4166 /// receiver. Dropping the receiver never cancels the operation.
4167 ///
4168 /// The caller awaits the receiver exactly as it would have awaited the
4169 /// operation, so response shape and timing are unchanged for a connection
4170 /// that stays alive. What changes is the failure mode: when the connection
4171 /// goes away, the per-connection `JoinSet` aborts only this waiter, and the
4172 /// operation itself runs to completion on the daemon.
4173 pub async fn spawn_durable_operation<F, E>(
4174 &self,
4175 operation_name: impl Into<String>,
4176 operation: F,
4177 ) -> tokio::sync::oneshot::Receiver<Result<serde_json::Value, E>>
4178 where
4179 F: std::future::Future<Output = Result<serde_json::Value, E>> + Send + 'static,
4180 E: Send + 'static,
4181 {
4182 let (result_tx, result_rx) = tokio::sync::oneshot::channel();
4183 let operation_name = operation_name.into();
4184 let mut tasks = self.durable_tasks.lock().await;
4185 while let Some(result) = tasks.try_join_next() {
4186 if let Err(error) = result {
4187 tracing::warn!(error = %error, "daemon-owned task failed to join");
4188 }
4189 }
4190 tasks.spawn(async move {
4191 let result = operation.await;
4192 if result_tx.send(result).is_err() {
4193 tracing::debug!(
4194 operation = %operation_name,
4195 "daemon-owned operation finished after its response waiter closed"
4196 );
4197 }
4198 operation_name
4199 });
4200 result_rx
4201 }
4202
4203 /// Start one fire-and-reconcile daemon-owned operation.
4204 pub async fn spawn_durable_task<F>(&self, operation_name: impl Into<String>, operation: F)
4205 where
4206 F: std::future::Future<Output = ()> + Send + 'static,
4207 {
4208 let operation_name = operation_name.into();
4209 let mut tasks = self.durable_tasks.lock().await;
4210 while let Some(result) = tasks.try_join_next() {
4211 if let Err(error) = result {
4212 tracing::warn!(error = %error, "daemon-owned task failed to join");
4213 }
4214 }
4215 tasks.spawn(async move {
4216 operation.await;
4217 operation_name
4218 });
4219 }
4220
4221 /// Enable the per-launch auth handshake. After this call, every
4222 /// new WS connection must call `session.auth` with `token` as
4223 /// the first frame; otherwise the connection is closed. Called
4224 /// by `car-server` at startup unless `--no-auth` is set
4225 /// (default flipped 2026-05); embedders supply their own token
4226 /// if they want the same posture. Returns `Err(token)` when
4227 /// auth was already installed.
4228 pub fn install_auth_token(&self, token: String) -> Result<(), String> {
4229 self.auth_token.set(token)
4230 }
4231
4232 /// Install the per-launch host token (Parslee-ai/car#254). A
4233 /// session that later presents this via `session.auth { host_token }`
4234 /// is granted the host-management role (`ClientSession::is_host`),
4235 /// which `authorize_run_access` requires for cross-agent run-trace
4236 /// reads. Set by `car-server` at startup (mints + writes the `0600`
4237 /// `host-token` file) unless `--no-auth` is set. Returns `Err(token)`
4238 /// when a host token was already installed.
4239 pub fn install_host_token(&self, token: String) -> Result<(), String> {
4240 self.host_token.set(token)
4241 }
4242
4243 pub fn install_mobile_runtime_url(&self, url: String) -> Result<(), String> {
4244 self.mobile_runtime_url.set(url)
4245 }
4246
4247 pub fn install_mobile_registration_url(&self, url: String) -> Result<(), String> {
4248 self.mobile_registration_url.set(url)
4249 }
4250
4251 pub fn install_parslee_session(
4252 &self,
4253 session: crate::parslee_auth::ParsleeSession,
4254 ) -> Result<(), crate::parslee_auth::ParsleeSession> {
4255 self.parslee_session.set(session)
4256 }
4257
4258 /// Install the runtime channel supervisor (Units 1/2/3). Called by
4259 /// `car-server` at boot right after `spawn_channel_pollers` builds it, so
4260 /// the host-gated `messaging.config.set` handler can reach it via
4261 /// `ServerState` to spawn a channel's watcher on enable. Idempotent on the
4262 /// first call; a second install is rejected (returns the supervisor back).
4263 pub fn install_channel_supervisor(
4264 &self,
4265 supervisor: Arc<crate::channel_supervisor::ChannelSupervisor>,
4266 ) -> Result<(), Arc<crate::channel_supervisor::ChannelSupervisor>> {
4267 self.channel_supervisor.set(supervisor)
4268 }
4269
4270 /// Install the bound MCP URL after car-server's listener is up.
4271 /// Idempotent on the first call; subsequent calls are accepted
4272 /// silently (matches the supervisor / a2a_dispatcher install
4273 /// idiom). Returns `Err(())` when an MCP URL was already
4274 /// installed — embedders should treat this as "another
4275 /// component beat us to it" and use whichever value is now set.
4276 pub fn install_mcp_url(&self, url: String) -> Result<(), String> {
4277 self.mcp_url.set(url)
4278 }
4279
4280 /// Lazy-initialize and return the remote MCP connector manager,
4281 /// backed by `~/.car/connectors.json` over the shared
4282 /// [`mcp_executor`](Self::mcp_executor). Construction is sync and
4283 /// touches no network — call [`ensure_connectors_loaded`](crate::session::ServerState::ensure_connectors_loaded) to dial
4284 /// persisted connectors.
4285 pub fn connectors(&self) -> Arc<car_connectors::ConnectorManager> {
4286 self.connectors
4287 .get_or_init(|| {
4288 let mgr = car_connectors::ConnectorManager::new(self.mcp_executor.clone())
4289 .unwrap_or_else(|_| {
4290 // HOME unresolved (rare): fall back to a CWD-relative
4291 // manifest so connectors still work this session.
4292 car_connectors::ConnectorManager::with_path(
4293 self.mcp_executor.clone(),
4294 std::path::PathBuf::from("connectors.json"),
4295 )
4296 });
4297 Arc::new(mgr)
4298 })
4299 .clone()
4300 }
4301
4302 /// Load persisted connectors and dial them, **once** per process.
4303 /// Idempotent: the first caller does the work; later callers return
4304 /// immediately. `car-server` calls this at boot so connector tools
4305 /// are live before clients connect; the `connectors.*` handlers call
4306 /// it too, so an embedder that skips the boot call still gets a
4307 /// lazy load on first use. Enabled tools discovered here are
4308 /// registered into any already-open sessions.
4309 pub async fn ensure_connectors_loaded(&self) {
4310 use std::sync::atomic::Ordering;
4311 if self.connectors_loaded.swap(true, Ordering::SeqCst) {
4312 return;
4313 }
4314 let mgr = self.connectors();
4315 match mgr.load_and_connect().await {
4316 Ok(entries) => self.register_connector_entries(&entries).await,
4317 Err(e) => tracing::warn!("connector load failed: {e}"),
4318 }
4319 }
4320
4321 /// Register connector [`car_engine::ToolEntry`]s into every currently-open
4322 /// session's runtime registry, so a tool enabled mid-session
4323 /// becomes visible without a reconnect. New sessions pick up the
4324 /// enabled set in [`create_session`](crate::session::ServerState::create_session).
4325 pub async fn register_connector_entries(&self, entries: &[car_engine::ToolEntry]) {
4326 if entries.is_empty() {
4327 return;
4328 }
4329 let sessions: Vec<Arc<ClientSession>> =
4330 self.sessions.lock().await.values().cloned().collect();
4331 for session in sessions {
4332 for entry in entries {
4333 session.runtime.register_tool_entry(entry.clone()).await;
4334 }
4335 }
4336 }
4337
4338 /// Unregister connector tools (by canonical name) from every open
4339 /// session's runtime, so a disabled or removed connector's tools
4340 /// stop being visible to the model and accepted by the validator.
4341 pub async fn unregister_connector_tools(&self, canonical_names: &[String]) {
4342 if canonical_names.is_empty() {
4343 return;
4344 }
4345 let sessions: Vec<Arc<ClientSession>> =
4346 self.sessions.lock().await.values().cloned().collect();
4347 for session in sessions {
4348 for name in canonical_names {
4349 session.runtime.unregister_tool(name).await;
4350 }
4351 }
4352 }
4353
4354 /// Whether any currently connected WS client holds the host-management
4355 /// role — authenticated via `session.auth { host_token }`
4356 /// (`ClientSession::is_host`), NOT the same thing as `host.subscribe`
4357 /// membership (see the doc comment on the auth check this mirrors,
4358 /// around `handle_session_auth`). In practice: is a CarHost / Command
4359 /// Deck app connected right now, so its browser drawer is a real
4360 /// visible surface?
4361 ///
4362 /// Scans the (small) live session set fresh on every call — no caching,
4363 /// so this is always current. The deciding signal for Task 7's
4364 /// headless/headed browser-launch default and its sign-in host-gone
4365 /// fallback (`assistant::browser_tools::HostConnectivity`).
4366 pub async fn any_host_connected(&self) -> bool {
4367 self.sessions
4368 .lock()
4369 .await
4370 .values()
4371 .any(|s| s.is_host.load(std::sync::atomic::Ordering::Acquire))
4372 }
4373
4374 /// Lazy-initialize and return the agent supervisor. The first
4375 /// call constructs a [`car_registry::supervisor::Supervisor`] backed by
4376 /// `~/.car/agents.json` + `~/.car/logs/`. Embedders that need a
4377 /// non-default location should call
4378 /// [`ServerState::install_supervisor`] before any handler runs.
4379 ///
4380 /// In observer mode (set via [`install_observer_manifest`](crate::session::ServerState::install_observer_manifest)),
4381 /// returns a clear error mentioning the manifest path the
4382 /// primary daemon owns. This prevents the second daemon from
4383 /// re-attempting `user_default()` (which would also fail with
4384 /// `AlreadyRunning`) on every WS call, and gives mutation
4385 /// handlers a stable refusal path. Read-only handlers
4386 /// (`agents.list`, `agents.health`) should call
4387 /// [`Self::observer_manifest_path`] first and fall back to
4388 /// [`car_registry::supervisor::Supervisor::list_from_manifest`] /
4389 /// `health_from_manifest` when set. Closes
4390 /// Parslee-ai/car-releases#44.
4391 /// The declarative-agent registry, lazy-initialized on first use.
4392 pub fn declagents(&self) -> Result<Arc<car_registry::declarative::DeclRegistry>, String> {
4393 if let Some(r) = self.declagents.get() {
4394 return Ok(r.clone());
4395 }
4396 let r = Arc::new(car_registry::declarative::DeclRegistry::user_default()?);
4397 let _ = self.declagents.set(r);
4398 Ok(self.declagents.get().expect("set or pre-existing").clone())
4399 }
4400
4401 pub fn routing(&self) -> Result<Arc<car_registry::routing::RoutingStore>, String> {
4402 if let Some(r) = self.routing.get() {
4403 return Ok(r.clone());
4404 }
4405 let r = Arc::new(car_registry::routing::RoutingStore::user_default()?);
4406 let _ = self.routing.set(r);
4407 Ok(self.routing.get().expect("set or pre-existing").clone())
4408 }
4409
4410 pub fn supervisor(&self) -> Result<Arc<car_registry::supervisor::Supervisor>, String> {
4411 if let Some(s) = self.supervisor.get() {
4412 return Ok(s.clone());
4413 }
4414 if let Some(p) = self.observer_manifest_path.get() {
4415 return Err(format!(
4416 "this car-server is observe-only — another car-server process \
4417 holds the supervisor lock for {}. Mutations refuse here; route \
4418 them to the primary daemon, or stop the other car-server first.",
4419 p.display()
4420 ));
4421 }
4422 let s = car_registry::supervisor::Supervisor::user_default()
4423 .map(Arc::new)
4424 .map_err(|e| e.to_string())?;
4425 // OnceLock::set returns the original arg back on collision —
4426 // a concurrent caller racing through user_default. Take
4427 // whichever wins.
4428 let _ = self.supervisor.set(s);
4429 Ok(self.supervisor.get().expect("set or pre-existing").clone())
4430 }
4431
4432 /// Replace the lazy default with a caller-supplied supervisor.
4433 /// Returns `Err(())` when a supervisor was already installed.
4434 /// Used by the standalone `car-server` binary to call
4435 /// `start_all()` on a known-good handle without paying the
4436 /// lazy-init lookup cost.
4437 pub fn install_supervisor(
4438 &self,
4439 supervisor: Arc<car_registry::supervisor::Supervisor>,
4440 ) -> Result<(), Arc<car_registry::supervisor::Supervisor>> {
4441 self.supervisor.set(supervisor)
4442 }
4443
4444 /// Non-acquiring read of the currently-installed supervisor.
4445 /// Unlike [`supervisor`](Self::supervisor), this does NOT lazy-
4446 /// init via `user_default()` — it returns `None` instead of
4447 /// constructing a fresh `Supervisor` and acquiring the
4448 /// `<manifest>.lock` as a side effect. Use this from read-only
4449 /// metadata paths (`host.subscribe` identity, status surfaces)
4450 /// where causing lock acquisition on observation would be a
4451 /// Heisenberg subscribe — the act of asking "do you own the
4452 /// lock?" must not be the act of taking it.
4453 pub fn supervisor_if_installed(&self) -> Option<Arc<car_registry::supervisor::Supervisor>> {
4454 self.supervisor.get().cloned()
4455 }
4456
4457 /// Mark this daemon as *observing* a manifest owned by another
4458 /// car-server process. After this call, `supervisor()` returns
4459 /// an "observe-only" error and read-only handlers
4460 /// (`agents.list`, `agents.health`) fall back to the static
4461 /// `Supervisor::list_from_manifest` / `health_from_manifest`
4462 /// paths. Idempotent — subsequent calls with the same path are
4463 /// no-ops; a different path returns `Err(())`. Closes
4464 /// Parslee-ai/car-releases#44.
4465 pub fn install_observer_manifest(&self, path: PathBuf) -> Result<(), PathBuf> {
4466 self.observer_manifest_path.set(path)
4467 }
4468
4469 /// Path of the manifest this daemon is observing but not
4470 /// supervising. `None` when this daemon owns the supervisor
4471 /// (the normal case) or when no manifest is configured at all
4472 /// (no `HOME`, embedder didn't install one).
4473 pub fn observer_manifest_path(&self) -> Option<&PathBuf> {
4474 self.observer_manifest_path.get()
4475 }
4476
4477 /// Lazy-initialize and return the in-core A2A dispatcher. The
4478 /// first call constructs an [`car_a2a::A2aDispatcher`] from
4479 /// either the embedder's overrides (set via
4480 /// [`ServerStateConfig::with_a2a_runtime`] / `with_a2a_store` /
4481 /// `with_a2a_card_source`) or sensible defaults: a fresh
4482 /// `Runtime` with `register_agent_basics` registered, an
4483 /// `InMemoryTaskStore`, and a card built from the runtime's
4484 /// tool schemas advertising `ws://127.0.0.1:9100/` as the
4485 /// public URL. Closes Parslee-ai/car-releases#28.
4486 pub async fn a2a_dispatcher(&self) -> Arc<car_a2a::A2aDispatcher> {
4487 if let Some(d) = self.a2a_dispatcher.get() {
4488 return d.clone();
4489 }
4490
4491 // Embedder overrides take precedence; fall back to defaults
4492 // for each slot independently (so an embedder that only
4493 // wants a custom card can leave the runtime + store at
4494 // defaults). `Mutex::take()` consumes the slot so the
4495 // defaults aren't reconstructed on a racing init that loses
4496 // the OnceLock::set call below.
4497 let runtime = self
4498 .a2a_runtime
4499 .lock()
4500 .expect("a2a_runtime mutex poisoned")
4501 .take();
4502 let runtime = match runtime {
4503 Some(r) => r,
4504 None => {
4505 let r = Arc::new(car_engine::Runtime::new());
4506 r.register_agent_basics().await;
4507 r
4508 }
4509 };
4510
4511 let store = self
4512 .a2a_store
4513 .lock()
4514 .expect("a2a_store mutex poisoned")
4515 .take()
4516 .unwrap_or_else(|| Arc::new(car_a2a::InMemoryTaskStore::new()));
4517
4518 let card_source = self
4519 .a2a_card_source
4520 .lock()
4521 .expect("a2a_card_source mutex poisoned")
4522 .take();
4523 let card_source = match card_source {
4524 Some(c) => c,
4525 None => {
4526 let card = car_a2a::build_default_agent_card(
4527 &runtime,
4528 car_a2a::AgentCardConfig::minimal(
4529 "Common Agent Runtime",
4530 "Embedded CAR daemon — A2A v1.0 reachable over WebSocket JSON-RPC.",
4531 "ws://127.0.0.1:9100/",
4532 car_a2a::AgentProvider {
4533 organization: "Parslee".into(),
4534 url: Some("https://github.com/Parslee-ai/car".into()),
4535 },
4536 ),
4537 )
4538 .await;
4539 Arc::new(move || card.clone()) as Arc<car_a2a::AgentCardSource>
4540 }
4541 };
4542
4543 let dispatcher = Arc::new(car_a2a::A2aDispatcher::new(runtime, store, card_source));
4544 // OnceLock::set returns Err on race — accept whichever
4545 // dispatcher won and clone-return that one.
4546 let _ = self.a2a_dispatcher.set(dispatcher);
4547 self.a2a_dispatcher
4548 .get()
4549 .expect("a2a_dispatcher set or pre-existing")
4550 .clone()
4551 }
4552
4553 /// Record the start of a run and return its [`RunMeta`]. The
4554 /// `run_id` is minted by the caller (the `runs.start` handler)
4555 /// so it can set `session.current_run_id` to the same value
4556 /// **before** responding (KTD3). Idempotent collision on an
4557 /// already-present `run_id` is treated as a fresh insert (uuids
4558 /// don't collide in practice; if one did, the latest start wins).
4559 ///
4560 /// U3: also writes the `RunStarted` line to the disk store + creates
4561 /// the run file, so the run is on disk from the first record (REPLAY
4562 /// survives a restart even before any turn lands). Disk failures are
4563 /// logged, never fatal — the in-memory registry is the live path.
4564 /// Fan one `runs.trace.event` out to every live subscriber of
4565 /// `run_id` (agent run tracing, U4). Called by the lifecycle methods
4566 /// **while they hold the `runs` lock** so append-and-notify is
4567 /// serialized with subscribe's snapshot-and-register (invariant #1).
4568 ///
4569 /// Each push is a non-blocking `try_send` onto the subscriber's
4570 /// bounded channel — the WS socket is written only by that
4571 /// subscriber's dedicated drain task, never here (invariant #2). A
4572 /// full channel drops the event (the slow-subscriber case); the
4573 /// client detects the cursor gap and re-subscribes (R8). Takes the
4574 /// already-acquired subscribers guard so the caller controls the lock
4575 /// scope and the `runs` → `run_subscribers` order.
4576 fn fanout_locked(
4577 subscribers: &HashMap<(String, String), crate::host::RunTraceSubscriber>,
4578 run_id: &str,
4579 agent_id: &str,
4580 record: car_proto::RunRecord,
4581 cursor: usize,
4582 status: car_proto::RunLiveStatus,
4583 ) {
4584 for ((sub_run, _client), sub) in subscribers.iter() {
4585 if sub_run != run_id {
4586 continue;
4587 }
4588 let event = car_proto::RunTraceEvent {
4589 run_id: run_id.to_string(),
4590 agent_id: agent_id.to_string(),
4591 record: record.clone(),
4592 cursor,
4593 status,
4594 };
4595 // Best-effort: a wedged subscriber's full channel drops the
4596 // event rather than stalling the producer. Logged at debug —
4597 // the client backfills via re-subscribe.
4598 if !sub.push(event) {
4599 tracing::debug!(
4600 run_id,
4601 "run-trace: dropped event for slow subscriber (channel full)"
4602 );
4603 }
4604 }
4605 }
4606
4607 /// Atomically reserve a run id/idempotency key across every live session.
4608 /// A per-run durability lock serializes the disk absence check with the
4609 /// final process-wide map insert. The global `runs` lock is held only for
4610 /// the two bounded map checks, never for filesystem enumeration/replay.
4611 pub async fn reserve_run(&self, meta: RunMeta) -> Result<RunReservation, String> {
4612 fn existing_reservation(
4613 existing: &RunMeta,
4614 requested: &RunMeta,
4615 ) -> Result<RunReservation, String> {
4616 if existing.client_id != requested.client_id {
4617 return Err(format!(
4618 "{} run `{}` belongs to client_id `{}`",
4619 car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX,
4620 requested.run_id,
4621 existing.client_id
4622 ));
4623 }
4624 if existing.agent_id != requested.agent_id {
4625 return Err(format!(
4626 "{} run `{}` was retried under a different agent",
4627 car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX,
4628 requested.run_id
4629 ));
4630 }
4631 let changed_fields = [
4632 (existing.intent != requested.intent).then_some("intent"),
4633 (existing.outcome_description != requested.outcome_description)
4634 .then_some("outcome_description"),
4635 ]
4636 .into_iter()
4637 .flatten()
4638 .collect::<Vec<_>>();
4639 if !changed_fields.is_empty() {
4640 return Err(format!(
4641 "{} idempotency conflict for run `{}`: retry changed occurrence-defining {}",
4642 car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX,
4643 requested.run_id,
4644 changed_fields.join(" and ")
4645 ));
4646 }
4647 Ok(RunReservation::Existing(existing.clone()))
4648 }
4649
4650 let durability_lock = self.run_durability_lock(&meta.run_id).await;
4651 let _durability_guard = durability_lock.lock().await;
4652 {
4653 let runs = self.runs.lock().await;
4654 if let Some(existing) = runs.get(&meta.run_id) {
4655 return existing_reservation(existing, &meta);
4656 }
4657 }
4658 let store = self.run_store.clone();
4659 let durable_run_id = meta.run_id.clone();
4660 let durable = tokio::task::spawn_blocking(move || store.run_started(&durable_run_id))
4661 .await
4662 .map_err(|error| format!("RunStarted reservation read task failed: {error}"))?
4663 .map_err(|error| format!("RunStarted reservation read failed: {error}"))?;
4664 if let Some(started) = durable {
4665 return Err(format!(
4666 "{} persisted run `{}` belongs to client_id `{}` and cannot be adopted",
4667 car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX,
4668 meta.run_id,
4669 started.client_id.as_deref().unwrap_or("historical/unknown")
4670 ));
4671 }
4672 let mut runs = self.runs.lock().await;
4673 if let Some(existing) = runs.get(&meta.run_id) {
4674 return existing_reservation(existing, &meta);
4675 }
4676 runs.insert(meta.run_id.clone(), meta);
4677 Ok(RunReservation::New)
4678 }
4679
4680 /// Release a newly reserved candidate when `runs.start` rejects before it
4681 /// binds or persists that candidate. The exact pristine metadata and the
4682 /// absent RunStore boundary are both re-checked under the per-run durability
4683 /// lock. This deliberately does not touch `current_run_id` or the journal
4684 /// binding, which may still belong to an older live run on this session.
4685 pub(crate) async fn release_unpersisted_run_reservation(
4686 &self,
4687 session: &ClientSession,
4688 expected: &RunMeta,
4689 ) -> Result<(), String> {
4690 if expected.client_id != session.client_id
4691 || expected.start_committed
4692 || expected.termination.is_some()
4693 || expected.ended_at.is_some()
4694 || !expected.turns.is_empty()
4695 || expected.pending_terminal.is_some()
4696 {
4697 return Err(format!(
4698 "run `{}` is not a pristine reservation owned by this session",
4699 expected.run_id
4700 ));
4701 }
4702 let still_expected = |existing: &RunMeta| {
4703 existing.run_id == expected.run_id
4704 && existing.client_id == expected.client_id
4705 && existing.agent_id == expected.agent_id
4706 && existing.intent == expected.intent
4707 && existing.outcome_description == expected.outcome_description
4708 && existing.started_at == expected.started_at
4709 && existing.durability_generation == expected.durability_generation
4710 && !existing.start_committed
4711 && existing.termination.is_none()
4712 && existing.ended_at.is_none()
4713 && existing.turns.is_empty()
4714 && existing.pending_terminal.is_none()
4715 };
4716 let durability_lock = self.run_durability_lock(&expected.run_id).await;
4717 let _durability_guard = durability_lock.lock().await;
4718 {
4719 let runs = self.runs.lock().await;
4720 let existing = runs
4721 .get(&expected.run_id)
4722 .ok_or_else(|| format!("reserved run `{}` is absent", expected.run_id))?;
4723 if !still_expected(existing) {
4724 return Err(format!(
4725 "reserved run `{}` changed before pre-start release",
4726 expected.run_id
4727 ));
4728 }
4729 }
4730 let store = self.run_store.clone();
4731 let durable_run_id = expected.run_id.clone();
4732 if tokio::task::spawn_blocking(move || store.run_started(&durable_run_id))
4733 .await
4734 .map_err(|error| format!("RunStarted release read task failed: {error}"))?
4735 .map_err(|error| format!("RunStarted release read failed: {error}"))?
4736 .is_some()
4737 {
4738 return Err(format!(
4739 "reserved run `{}` reached durable storage and cannot be released",
4740 expected.run_id
4741 ));
4742 }
4743 let mut runs = self.runs.lock().await;
4744 let existing = runs
4745 .get(&expected.run_id)
4746 .ok_or_else(|| format!("reserved run `{}` is absent", expected.run_id))?;
4747 if !still_expected(existing) {
4748 return Err(format!(
4749 "reserved run `{}` changed during pre-start release",
4750 expected.run_id
4751 ));
4752 }
4753 runs.remove(&expected.run_id);
4754 Ok(())
4755 }
4756
4757 pub async fn persist_run_start(&self, run_id: &str) -> Result<car_proto::RunStarted, String> {
4758 let durability_lock = self.run_durability_lock(run_id).await;
4759 let _durability_guard = durability_lock.lock().await;
4760 let started = {
4761 let runs = self.runs.lock().await;
4762 let meta = runs
4763 .get(run_id)
4764 .ok_or_else(|| format!("unknown reserved run_id `{run_id}`"))?;
4765 car_proto::RunStarted {
4766 run_id: meta.run_id.clone(),
4767 client_id: Some(meta.client_id.clone()),
4768 agent_id: meta.agent_id.clone(),
4769 intent: meta.intent.clone(),
4770 outcome_description: meta.outcome_description.clone(),
4771 started_at: meta.started_at,
4772 }
4773 };
4774 let store = self.run_store.clone();
4775 let durable_started = started.clone();
4776 tokio::task::spawn_blocking(move || store.write_started(&durable_started))
4777 .await
4778 .map_err(|error| format!("RunStarted durability task failed: {error}"))?
4779 .map_err(|error| format!("RunStarted durability failed: {error}"))?;
4780 Ok(started)
4781 }
4782
4783 pub async fn commit_run_start(&self, run_id: &str) -> Result<(), String> {
4784 let mut runs = self.runs.lock().await;
4785 let meta = runs
4786 .get_mut(run_id)
4787 .ok_or_else(|| format!("unknown reserved run_id `{run_id}`"))?;
4788 if meta.start_committed {
4789 return Ok(());
4790 }
4791 meta.start_committed = true;
4792 meta.durability_generation = meta.durability_generation.wrapping_add(1);
4793 let started = car_proto::RunStarted {
4794 run_id: meta.run_id.clone(),
4795 client_id: Some(meta.client_id.clone()),
4796 agent_id: meta.agent_id.clone(),
4797 intent: meta.intent.clone(),
4798 outcome_description: meta.outcome_description.clone(),
4799 started_at: meta.started_at,
4800 };
4801 let subs = self.run_subscribers.lock().await;
4802 Self::fanout_locked(
4803 &subs,
4804 &started.run_id,
4805 &started.agent_id,
4806 car_proto::RunRecord::Started(started.clone()),
4807 0,
4808 car_proto::RunLiveStatus::InProgress,
4809 );
4810 Ok(())
4811 }
4812
4813 /// Resolve a `runs.start` transaction whose acknowledgement failed before
4814 /// the owning socket disconnected. Returns `true` when no durable start
4815 /// boundary existed and the reservation was released. Returns `false`
4816 /// after reconciling an exact durable `RunStarted`, which the disconnect
4817 /// sweep must then close as one historical `Incomplete` occurrence.
4818 pub(crate) async fn reconcile_or_release_unacknowledged_start(
4819 &self,
4820 session: &ClientSession,
4821 run_id: &str,
4822 ) -> Result<bool, String> {
4823 let durability_lock = self.run_durability_lock(run_id).await;
4824 let _durability_guard = durability_lock.lock().await;
4825 let expected = {
4826 let runs = self.runs.lock().await;
4827 let meta = runs
4828 .get(run_id)
4829 .ok_or_else(|| format!("unknown reserved run_id `{run_id}`"))?;
4830 if meta.client_id != session.client_id {
4831 return Err(format!(
4832 "{} run `{run_id}` belongs to client_id `{}`",
4833 car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX,
4834 meta.client_id
4835 ));
4836 }
4837 if meta.start_committed {
4838 return Ok(false);
4839 }
4840 if meta.is_terminal() || meta.pending_terminal.is_some() {
4841 return Err(format!(
4842 "uncommitted run `{run_id}` unexpectedly has terminal state"
4843 ));
4844 }
4845 car_proto::RunStarted {
4846 run_id: meta.run_id.clone(),
4847 client_id: Some(meta.client_id.clone()),
4848 agent_id: meta.agent_id.clone(),
4849 intent: meta.intent.clone(),
4850 outcome_description: meta.outcome_description.clone(),
4851 started_at: meta.started_at,
4852 }
4853 };
4854
4855 let store = self.run_store.clone();
4856 let durable_run_id = run_id.to_string();
4857 let durable = tokio::task::spawn_blocking(move || store.run_started(&durable_run_id))
4858 .await
4859 .map_err(|error| format!("RunStarted recovery read task failed: {error}"))?
4860 .map_err(|error| format!("RunStarted recovery read failed: {error}"))?;
4861 match durable {
4862 Some(durable) => {
4863 if durable != expected {
4864 return Err(format!(
4865 "durable RunStarted identity for `{run_id}` does not match its live reservation"
4866 ));
4867 }
4868 // Re-run both exact idempotent durability barriers. The first
4869 // retry turns a prior write-without-fsync into an acknowledged
4870 // RunStore boundary; the second either appends RunStarted or
4871 // fsyncs the exact journal row already pending in memory.
4872 let store = self.run_store.clone();
4873 let durable_expected = expected.clone();
4874 tokio::task::spawn_blocking(move || store.write_started(&durable_expected))
4875 .await
4876 .map_err(|error| {
4877 format!("RunStarted recovery durability task failed: {error}")
4878 })?
4879 .map_err(|error| format!("RunStarted recovery durability failed: {error}"))?;
4880 session.append_run_started_event(&expected).await?;
4881 self.commit_run_start(run_id).await?;
4882 Ok(false)
4883 }
4884 None => {
4885 // `runs.start` orders RunStore before journal append, so an
4886 // absent RunStore boundary proves no lifecycle row was ever
4887 // admitted to the journal. Remove only the zero-byte file an
4888 // open-before-write failure may have left behind.
4889 let store = self.run_store.clone();
4890 let durable_expected = expected.clone();
4891 tokio::task::spawn_blocking(move || {
4892 store.rollback_empty_run_start(&durable_expected)
4893 })
4894 .await
4895 .map_err(|error| format!("empty RunStarted rollback task failed: {error}"))?
4896 .map_err(|error| format!("empty RunStarted rollback failed: {error}"))?;
4897 {
4898 let mut runs = self.runs.lock().await;
4899 let meta = runs.get(run_id).ok_or_else(|| {
4900 format!("reserved run `{run_id}` disappeared during rollback")
4901 })?;
4902 let still_expected = meta.client_id == session.client_id
4903 && !meta.start_committed
4904 && !meta.is_terminal()
4905 && meta.pending_terminal.is_none()
4906 && meta.agent_id == expected.agent_id
4907 && meta.intent == expected.intent
4908 && meta.outcome_description == expected.outcome_description
4909 && meta.started_at == expected.started_at;
4910 if !still_expected {
4911 return Err(format!(
4912 "reserved run `{run_id}` changed during empty-start rollback"
4913 ));
4914 }
4915 runs.remove(run_id);
4916 }
4917 session.clear_run_journal_binding(run_id).await?;
4918 let mut current = session.current_run_id.lock().await;
4919 if current.as_deref() == Some(run_id) {
4920 *current = None;
4921 }
4922 Ok(true)
4923 }
4924 }
4925 }
4926
4927 /// Compatibility/internal path for CAR-owned in-process callers that do
4928 /// not expose `runs.start`. The public RPC uses reserve/persist/journal/
4929 /// commit explicitly so its acknowledgement includes both durable stores.
4930 pub async fn start_run(&self, meta: RunMeta) -> Result<(), String> {
4931 let run_id = meta.run_id.clone();
4932 self.reserve_run(meta).await?;
4933 self.persist_run_start(&run_id).await?;
4934 self.commit_run_start(&run_id).await
4935 }
4936
4937 /// Make a run terminal with a harness-reported outcome
4938 /// (`runs.complete`). Returns `Err` if the `run_id` is unknown or
4939 /// already terminal — the handler maps that to a JSON-RPC error so
4940 /// a double-complete or stale id is visible, not silently swallowed.
4941 ///
4942 /// U3: also appends the terminal `RunEnded` line to the run's JSONL
4943 /// file so REPLAY (U5) reports the right status (Completed) after a
4944 /// restart. The disk write happens after the lock is released; disk
4945 /// failures are logged, never fatal.
4946 pub async fn prepare_run_completion(
4947 &self,
4948 run_id: &str,
4949 termination: car_proto::RunTermination,
4950 ) -> Result<car_proto::RunEnded, String> {
4951 self.prepare_run_completion_for_active_owner(run_id, termination, None)
4952 .await
4953 }
4954
4955 pub(crate) async fn prepare_run_completion_for_active_owner(
4956 &self,
4957 run_id: &str,
4958 termination: car_proto::RunTermination,
4959 expected_active_client_id: Option<&str>,
4960 ) -> Result<car_proto::RunEnded, String> {
4961 let completion_digest = run_completion_digest(&termination)?;
4962 if expected_active_client_id.is_some() {
4963 if let Some(gate) = &self.run_completion_fence_gate {
4964 gate.wait_if_armed().await;
4965 }
4966 }
4967 let durability_lock = self.run_durability_lock(run_id).await;
4968 let _durability_guard = durability_lock.lock().await;
4969 let ended = {
4970 let mut runs = self.runs.lock().await;
4971 let meta = runs
4972 .get_mut(run_id)
4973 .ok_or_else(|| format!("unknown run_id `{run_id}`"))?;
4974 if let Some(error) = &meta.trace_corruption {
4975 return Err(error.clone());
4976 }
4977 if !meta.start_committed {
4978 return Err(format!("run `{run_id}` start is not durably committed"));
4979 }
4980 if expected_active_client_id.is_some_and(|expected| meta.active_client_id != expected) {
4981 return Err(format!(
4982 "run `{run_id}` active owner changed before terminalization"
4983 ));
4984 }
4985 if let Some(requested) = &meta.cancellation_pending {
4986 let matches_cancel = matches!(
4987 &termination,
4988 car_proto::RunTermination::Cancelled { cancellation }
4989 if cancellation.run_id == requested.run_id
4990 && cancellation.idempotency_key == requested.idempotency_key
4991 && cancellation.reason_digest == requested.reason_digest
4992 && cancellation.principal == requested.principal
4993 && cancellation.action_id == requested.action_id
4994 && cancellation.request_id == requested.request_id
4995 );
4996 if !matches_cancel {
4997 return Err(format!(
4998 "run `{run_id}` has a durable cancellation request and is quarantined"
4999 ));
5000 }
5001 }
5002 if let Some(existing) = &meta.termination {
5003 if !same_run_termination(existing, &termination) {
5004 return Err(format!("run `{run_id}` already has a different terminal"));
5005 }
5006 meta.resume_lease = None;
5007 car_proto::RunEnded {
5008 run_id: run_id.to_string(),
5009 client_id: Some(meta.client_id.clone()),
5010 agent_id: meta.agent_id.clone(),
5011 termination,
5012 completion_digest: Some(completion_digest),
5013 ended_at: meta
5014 .ended_at
5015 .ok_or_else(|| "terminal run is missing ended_at".to_string())?,
5016 }
5017 } else if let Some(pending) = &meta.pending_terminal {
5018 if !same_run_termination(&pending.termination, &termination)
5019 || pending.completion_digest.as_deref() != Some(completion_digest.as_str())
5020 {
5021 return Err(format!(
5022 "run `{run_id}` has a different pending terminal transaction"
5023 ));
5024 }
5025 meta.resume_lease = None;
5026 pending.clone()
5027 } else {
5028 let ended = car_proto::RunEnded {
5029 run_id: run_id.to_string(),
5030 client_id: Some(meta.client_id.clone()),
5031 agent_id: meta.agent_id.clone(),
5032 termination,
5033 completion_digest: Some(completion_digest),
5034 ended_at: chrono::Utc::now(),
5035 };
5036 meta.pending_terminal = Some(ended.clone());
5037 meta.resume_lease = None;
5038 meta.durability_generation = meta.durability_generation.wrapping_add(1);
5039 ended
5040 }
5041 };
5042 let store = self.run_store.clone();
5043 let durable_ended = ended.clone();
5044 let persisted = tokio::task::spawn_blocking(move || store.write_ended(&durable_ended))
5045 .await
5046 .map_err(|error| format!("RunEnded durability task failed: {error}"))?;
5047 if let Err(error) = persisted {
5048 if crate::run_store::is_trace_corruption_error(&error) {
5049 let mut runs = self.runs.lock().await;
5050 let message = self
5051 .quarantine_run_trace_locked(&mut runs, run_id, error.to_string())
5052 .await;
5053 return Err(message);
5054 }
5055 return Err(format!("RunEnded durability failed: {error}"));
5056 }
5057 Ok(ended)
5058 }
5059
5060 pub async fn commit_run_completion(&self, ended: &car_proto::RunEnded) -> Result<(), String> {
5061 {
5062 let mut runs = self.runs.lock().await;
5063 let meta = runs
5064 .get_mut(&ended.run_id)
5065 .ok_or_else(|| format!("unknown run_id `{}`", ended.run_id))?;
5066 if let Some(existing) = &meta.termination {
5067 if same_run_termination(existing, &ended.termination)
5068 && meta.ended_at == Some(ended.ended_at)
5069 {
5070 return Ok(());
5071 }
5072 return Err(format!(
5073 "run `{}` already has a different terminal",
5074 ended.run_id
5075 ));
5076 }
5077 if !meta
5078 .pending_terminal
5079 .as_ref()
5080 .is_some_and(|pending| same_run_ended(pending, ended))
5081 {
5082 return Err(format!(
5083 "run `{}` terminal commit does not match its prepared transaction",
5084 ended.run_id
5085 ));
5086 }
5087 meta.termination = Some(ended.termination.clone());
5088 meta.ended_at = Some(ended.ended_at);
5089 meta.pending_terminal = None;
5090 meta.resume_lease = None;
5091 meta.cancellation_pending = None;
5092 meta.cancellation_receipt = None;
5093 meta.durability_generation = meta.durability_generation.wrapping_add(1);
5094 let cursor = meta.turn_cursor();
5095 let status = meta.live_status();
5096 let agent_id = meta.agent_id.clone();
5097 let subs = self.run_subscribers.lock().await;
5098 Self::fanout_locked(
5099 &subs,
5100 &ended.run_id,
5101 &agent_id,
5102 car_proto::RunRecord::Ended(ended.clone()),
5103 cursor,
5104 status,
5105 );
5106 }
5107 // Heap hygiene: the run is now terminal and fully flushed to disk,
5108 // and the terminal `runs.trace.event` has already fanned out to any
5109 // live subscriber. Drop the in-memory per-turn buffer (each entry
5110 // holds a full prompt + CLI output, potentially MBs) so completed
5111 // runs don't pin the heaviest payloads for the daemon's lifetime.
5112 // The lightweight `RunMeta` header (agent_id, status, started_at,
5113 // termination) stays resident so `run_meta`/status lookups still
5114 // work. A late `runs.subscribe` to this terminal run re-sources its
5115 // snapshot from disk (see `subscribe_run`). Only terminal runs are
5116 // cleared; an in-progress run keeps its turns (the live snapshot
5117 // source). `Vec::new()` frees the buffer's capacity, not just its
5118 // length.
5119 self.clear_terminal_run_turns(&ended.run_id).await;
5120 Ok(())
5121 }
5122
5123 pub async fn complete_run(
5124 &self,
5125 run_id: &str,
5126 termination: car_proto::RunTermination,
5127 ) -> Result<car_proto::RunEnded, String> {
5128 let ended = self.prepare_run_completion(run_id, termination).await?;
5129 self.commit_run_completion(&ended).await?;
5130 Ok(ended)
5131 }
5132
5133 /// Commit the body-free request to RunStore and the authenticated opening
5134 /// client's journal under the same per-run durability lock used by
5135 /// terminal completion. Memory changes only after both receipts exist.
5136 pub async fn persist_run_cancellation_requested(
5137 &self,
5138 owner_session: &ClientSession,
5139 requested: &car_proto::RunCancellationRequested,
5140 ) -> Result<(), String> {
5141 let run_id = requested.run_id.as_str();
5142 let durability_lock = self.run_durability_lock(run_id).await;
5143 let _durability_guard = durability_lock.lock().await;
5144 let agent_id = {
5145 let runs = self.runs.lock().await;
5146 let meta = runs
5147 .get(run_id)
5148 .ok_or_else(|| format!("unknown run_id `{run_id}`"))?;
5149 if meta.is_terminal() || meta.pending_terminal.is_some() {
5150 return Err(format!("run `{run_id}` is already terminal"));
5151 }
5152 if meta.client_id != owner_session.client_id || !meta.start_committed {
5153 return Err("run cancellation client/run binding mismatch".into());
5154 }
5155 if let Some(existing) = &meta.cancellation_pending {
5156 if existing != requested {
5157 return Err(format!(
5158 "run `{run_id}` already has a different cancellation request"
5159 ));
5160 }
5161 return Ok(());
5162 }
5163 meta.agent_id.clone()
5164 };
5165 let store = self.run_store.clone();
5166 let durable_agent = agent_id.clone();
5167 let durable_requested = requested.clone();
5168 tokio::task::spawn_blocking(move || {
5169 store.write_cancellation_requested(&durable_agent, &durable_requested)
5170 })
5171 .await
5172 .map_err(|error| format!("cancellation request durability task failed: {error}"))?
5173 .map_err(|error| format!("cancellation request durability failed: {error}"))?;
5174 owner_session
5175 .append_run_cancellation_requested_event(requested)
5176 .await?;
5177 let mut runs = self.runs.lock().await;
5178 let meta = runs
5179 .get_mut(run_id)
5180 .ok_or_else(|| format!("unknown run_id `{run_id}`"))?;
5181 meta.cancellation_pending = Some(requested.clone());
5182 meta.durability_generation = meta.durability_generation.wrapping_add(1);
5183 let cursor = meta.turn_cursor();
5184 let agent_id = meta.agent_id.clone();
5185 let subs = self.run_subscribers.lock().await;
5186 Self::fanout_locked(
5187 &subs,
5188 run_id,
5189 &agent_id,
5190 car_proto::RunRecord::CancellationRequested(requested.clone()),
5191 cursor,
5192 car_proto::RunLiveStatus::CancellationPending,
5193 );
5194 Ok(())
5195 }
5196
5197 pub async fn persist_run_cancellation_result(
5198 &self,
5199 owner_session: &ClientSession,
5200 result: &car_proto::RunCancelResponse,
5201 ) -> Result<(), String> {
5202 let run_id = result.run_id.as_str();
5203 let durability_lock = self.run_durability_lock(run_id).await;
5204 let _durability_guard = durability_lock.lock().await;
5205 let agent_id = {
5206 let runs = self.runs.lock().await;
5207 let meta = runs
5208 .get(run_id)
5209 .ok_or_else(|| format!("unknown run_id `{run_id}`"))?;
5210 if meta.is_terminal() {
5211 return Err(format!("run `{run_id}` is already terminal"));
5212 }
5213 let requested = meta
5214 .cancellation_pending
5215 .as_ref()
5216 .ok_or_else(|| format!("run `{run_id}` has no cancellation request"))?;
5217 if requested.idempotency_key != result.idempotency_key
5218 || requested.reason_digest != result.reason_digest
5219 || requested.principal != result.principal
5220 {
5221 return Err(format!("run `{run_id}` cancellation receipt mismatch"));
5222 }
5223 if let Some(existing) = &meta.cancellation_receipt {
5224 if existing != result {
5225 return Err(format!(
5226 "run `{run_id}` has a different cancellation receipt"
5227 ));
5228 }
5229 return Ok(());
5230 }
5231 meta.agent_id.clone()
5232 };
5233 let store = self.run_store.clone();
5234 let durable_agent = agent_id.clone();
5235 let durable_result = result.clone();
5236 tokio::task::spawn_blocking(move || {
5237 store.write_cancellation_result(&durable_agent, &durable_result)
5238 })
5239 .await
5240 .map_err(|error| format!("cancellation result durability task failed: {error}"))?
5241 .map_err(|error| format!("cancellation result durability failed: {error}"))?;
5242 owner_session
5243 .append_run_cancellation_result_event(result)
5244 .await?;
5245 let mut runs = self.runs.lock().await;
5246 let meta = runs
5247 .get_mut(run_id)
5248 .ok_or_else(|| format!("unknown run_id `{run_id}`"))?;
5249 meta.cancellation_receipt = Some(result.clone());
5250 meta.durability_generation = meta.durability_generation.wrapping_add(1);
5251 let cursor = meta.turn_cursor();
5252 let agent_id = meta.agent_id.clone();
5253 let subs = self.run_subscribers.lock().await;
5254 Self::fanout_locked(
5255 &subs,
5256 run_id,
5257 &agent_id,
5258 car_proto::RunRecord::CancellationResult(result.clone()),
5259 cursor,
5260 car_proto::RunLiveStatus::CancellationPending,
5261 );
5262 Ok(())
5263 }
5264
5265 /// Finish an unconfirmed cancellation after the opening socket has gone
5266 /// away. The durable `RunStarted` client id is the journal authority: a
5267 /// reconnect may request recovery, but it may not redirect the receipt to
5268 /// its own journal. RunStore, that authenticated journal, live state, and
5269 /// subscribers advance as one retryable transaction before success is
5270 /// returned.
5271 pub async fn persist_recovered_run_cancellation_result(
5272 &self,
5273 agent_id: &str,
5274 result: &car_proto::RunCancelResponse,
5275 ) -> Result<(), String> {
5276 if result.status != car_proto::RunCancellationStatus::TerminationUnconfirmed
5277 || result.terminal_digest.is_some()
5278 {
5279 return Err("recovered cancellation receipt must be termination_unconfirmed".into());
5280 }
5281 let run_id = result.run_id.as_str();
5282 let durability_lock = self.run_durability_lock(run_id).await;
5283 let _durability_guard = durability_lock.lock().await;
5284 let store = self.run_store.clone();
5285 let durable_agent = agent_id.to_string();
5286 let durable_run = run_id.to_string();
5287 let records = tokio::task::spawn_blocking(move || {
5288 store.get_run_trace_for_checked(&durable_agent, &durable_run)
5289 })
5290 .await
5291 .map_err(|error| format!("recovered cancellation trace task failed: {error}"))?
5292 .map_err(|error| format!("recovered cancellation trace read failed: {error}"))?
5293 .ok_or_else(|| "recovered cancellation durable owner is unavailable".to_string())?;
5294 let started = records
5295 .iter()
5296 .find_map(|record| match record {
5297 car_proto::RunRecord::Started(started) => Some(started),
5298 _ => None,
5299 })
5300 .ok_or_else(|| "recovered cancellation is missing durable RunStarted".to_string())?;
5301 if started.agent_id != agent_id {
5302 return Err("recovered cancellation durable agent mismatch".into());
5303 }
5304 let client_id = started
5305 .client_id
5306 .as_deref()
5307 .filter(|client_id| {
5308 !client_id.is_empty()
5309 && !client_id.contains('/')
5310 && !client_id.contains('\\')
5311 && *client_id != "."
5312 && *client_id != ".."
5313 })
5314 .ok_or_else(|| {
5315 "recovered cancellation has no authenticated durable journal owner".to_string()
5316 })?
5317 .to_string();
5318 if records
5319 .iter()
5320 .any(|record| matches!(record, car_proto::RunRecord::Ended(_)))
5321 {
5322 return Err(format!("run `{run_id}` is already terminal"));
5323 }
5324 let requested = records
5325 .iter()
5326 .find_map(|record| match record {
5327 car_proto::RunRecord::CancellationRequested(requested) => Some(requested),
5328 _ => None,
5329 })
5330 .ok_or_else(|| format!("run `{run_id}` has no durable cancellation request"))?;
5331 if requested.receipt_version != result.receipt_version
5332 || requested.run_id != result.run_id
5333 || requested.idempotency_key != result.idempotency_key
5334 || requested.reason_digest != result.reason_digest
5335 || requested.principal != result.principal
5336 || requested.action_id != result.action_id
5337 || requested.request_id != result.request_id
5338 {
5339 return Err(format!(
5340 "run `{run_id}` recovered cancellation receipt mismatch"
5341 ));
5342 }
5343 if let Some(existing) = records.iter().find_map(|record| match record {
5344 car_proto::RunRecord::CancellationResult(existing) => Some(existing),
5345 _ => None,
5346 }) {
5347 if existing != result {
5348 return Err(format!(
5349 "run `{run_id}` has a different durable cancellation receipt"
5350 ));
5351 }
5352 }
5353 {
5354 let runs = self.runs.lock().await;
5355 if let Some(meta) = runs.get(run_id) {
5356 if meta.agent_id != agent_id
5357 || meta.client_id != client_id
5358 || !meta.start_committed
5359 || meta.is_terminal()
5360 || meta.cancellation_pending.as_ref() != Some(requested)
5361 {
5362 return Err(format!(
5363 "run `{run_id}` live state does not authenticate recovered cancellation"
5364 ));
5365 }
5366 if meta
5367 .cancellation_receipt
5368 .as_ref()
5369 .is_some_and(|existing| existing != result)
5370 {
5371 return Err(format!(
5372 "run `{run_id}` has a different live cancellation receipt"
5373 ));
5374 }
5375 }
5376 }
5377
5378 let store = self.run_store.clone();
5379 let durable_agent = agent_id.to_string();
5380 let durable_result = result.clone();
5381 tokio::task::spawn_blocking(move || {
5382 store.write_cancellation_result(&durable_agent, &durable_result)
5383 })
5384 .await
5385 .map_err(|error| format!("recovered cancellation durability task failed: {error}"))?
5386 .map_err(|error| format!("recovered cancellation durability failed: {error}"))?;
5387
5388 let journal_dir = self.journal_dir.clone();
5389 let journal_failures = self.journal_failures.clone();
5390 let journal_client = client_id.clone();
5391 let journal_result = result.clone();
5392 let journal_was_new = tokio::task::spawn_blocking(move || {
5393 append_recovered_cancellation_result_journal(
5394 journal_dir,
5395 journal_failures,
5396 journal_client,
5397 journal_result,
5398 )
5399 })
5400 .await
5401 .map_err(|error| format!("recovered cancellation journal task failed: {error}"))??;
5402
5403 let mut runs = self.runs.lock().await;
5404 let (cursor, fanout) = match runs.get_mut(run_id) {
5405 Some(meta) => {
5406 let fanout = meta.cancellation_receipt.is_none();
5407 meta.cancellation_receipt = Some(result.clone());
5408 if fanout {
5409 meta.durability_generation = meta.durability_generation.wrapping_add(1);
5410 }
5411 (meta.turn_cursor(), fanout)
5412 }
5413 None => (
5414 records
5415 .iter()
5416 .filter(|record| matches!(record, car_proto::RunRecord::Turn(_)))
5417 .count(),
5418 journal_was_new,
5419 ),
5420 };
5421 if fanout {
5422 let subscribers = self.run_subscribers.lock().await;
5423 Self::fanout_locked(
5424 &subscribers,
5425 run_id,
5426 agent_id,
5427 car_proto::RunRecord::CancellationResult(result.clone()),
5428 cursor,
5429 car_proto::RunLiveStatus::CancellationPending,
5430 );
5431 }
5432 Ok(())
5433 }
5434
5435 /// Mark a run `Incomplete` (R5) — used by disconnect cleanup when a
5436 /// harness drops without `runs.complete`. No-op if the run is
5437 /// already terminal (the common healthy-close case where
5438 /// `runs.complete` won the race). Returns `true` if it actually
5439 /// wrote the `Incomplete` marker.
5440 ///
5441 /// U3: on the transition to `Incomplete`, appends the terminal
5442 /// `RunEnded { Incomplete }` line to disk so an orphaned run reports
5443 /// `Incomplete` (not `InProgress`) on REPLAY after a restart. Disk
5444 /// failures are logged, never fatal.
5445 pub async fn prepare_run_incomplete(&self, run_id: &str) -> Option<car_proto::RunEnded> {
5446 self.prepare_run_incomplete_for_active_owner(run_id, None)
5447 .await
5448 }
5449
5450 async fn prepare_run_incomplete_for_active_owner(
5451 &self,
5452 run_id: &str,
5453 expected_active_client_id: Option<&str>,
5454 ) -> Option<car_proto::RunEnded> {
5455 // If a `runs.complete` transaction already prepared its exact outcome,
5456 // disconnect cleanup must finish that transaction, never overwrite it
5457 // with Incomplete.
5458 let termination = self
5459 .runs
5460 .lock()
5461 .await
5462 .get(run_id)
5463 .and_then(|meta| meta.pending_terminal.as_ref())
5464 .map(|ended| ended.termination.clone())
5465 .unwrap_or(car_proto::RunTermination::Incomplete);
5466 match self
5467 .prepare_run_completion_for_active_owner(run_id, termination, expected_active_client_id)
5468 .await
5469 {
5470 Ok(ended) => Some(ended),
5471 Err(error) => {
5472 tracing::error!(run_id, %error, "failed to prepare durable incomplete terminal");
5473 None
5474 }
5475 }
5476 }
5477
5478 pub async fn mark_run_incomplete(&self, run_id: &str) -> Option<car_proto::RunEnded> {
5479 let ended = self.prepare_run_incomplete(run_id).await?;
5480 if let Err(error) = self.commit_run_completion(&ended).await {
5481 tracing::error!(run_id, %error, "failed to commit incomplete terminal");
5482 return None;
5483 }
5484 Some(ended)
5485 }
5486
5487 /// Free the in-memory per-turn buffer of a TERMINAL run, keeping the
5488 /// lightweight `RunMeta` header resident (agent run tracing, heap
5489 /// hygiene follow-up). Called at the end of `complete_run` /
5490 /// `mark_run_incomplete`, AFTER the disk flush and terminal fanout, so
5491 /// the disk store remains the durable source of truth and any late
5492 /// `runs.subscribe` re-sources the snapshot from disk
5493 /// ([`Self::subscribe_run`]).
5494 ///
5495 /// Guards on `is_terminal()` so a same-id reuse that somehow re-opened
5496 /// the run (it shouldn't — ids are uuids) never has live turns dropped.
5497 /// `Vec::new()` releases the buffer's capacity, not just its length —
5498 /// the per-turn payloads (full prompt + CLI output) are the heavy part.
5499 async fn clear_terminal_run_turns(&self, run_id: &str) {
5500 let mut runs = self.runs.lock().await;
5501 if let Some(meta) = runs.get_mut(run_id) {
5502 if meta.is_terminal() {
5503 meta.turns = Vec::new();
5504 }
5505 }
5506 }
5507
5508 /// Non-acquiring read of a run's current metadata (clone). Used by
5509 /// tests and by the U2 recorder to learn a run's owning `agent_id`.
5510 ///
5511 /// NOTE: this clones the ENTIRE `RunMeta`, including its `turns` buffer
5512 /// (full prompts + CLI output — up to the per-run caps). Hot callers
5513 /// that need only the run's header facts must use [`Self::run_header`]
5514 /// instead; cloning the whole buffer per batch RPC is a ~512 MB
5515 /// worst-case copy at this PR's caps (ADV-2).
5516 pub async fn run_meta(&self, run_id: &str) -> Option<RunMeta> {
5517 self.runs.lock().await.get(run_id).cloned()
5518 }
5519
5520 /// Lightweight, non-cloning read of a run's header facts under the
5521 /// `runs` lock: `(agent_id, terminal, turn_count, trace_corruption)`.
5522 /// The owning agent id and optional error are the only heap allocations;
5523 /// This is what `handle_runs_record_turns` needs — owner for the write
5524 /// authz check, terminality for the distinguishable `run_terminal` drop,
5525 /// and the current turn count for the (fast-path) ceiling pre-check —
5526 /// without the `run_meta` deep clone of every recorded turn's payload.
5527 pub async fn run_header(&self, run_id: &str) -> Option<(String, bool, usize, Option<String>)> {
5528 self.runs.lock().await.get(run_id).map(|m| {
5529 (
5530 m.agent_id.clone(),
5531 m.is_terminal(),
5532 m.turns.len(),
5533 m.trace_corruption.clone(),
5534 )
5535 })
5536 }
5537
5538 /// Lightweight authenticated producer binding for run lifecycle handlers:
5539 /// `(client_id, terminal)`. Never falls back to disk because an on-disk row
5540 /// has no live WebSocket producer to authorize new writes.
5541 pub async fn run_lifecycle_binding(&self, run_id: &str) -> Option<(String, bool)> {
5542 self.runs.lock().await.get(run_id).map(|meta| {
5543 (
5544 meta.active_client_id.clone(),
5545 meta.is_terminal() || meta.trace_corruption.is_some(),
5546 )
5547 })
5548 }
5549
5550 /// `(client_id, terminal, start_committed, completion_pending)` for the
5551 /// authenticated producer path. Proposal writes require the exact owner,
5552 /// a committed start, and no prepared terminal transaction.
5553 pub async fn run_lifecycle_state(&self, run_id: &str) -> Option<(String, bool, bool, bool)> {
5554 self.run_lifecycle_state_with_corruption(run_id).await.map(
5555 |(client_id, terminal, committed, pending, _)| {
5556 (client_id, terminal, committed, pending)
5557 },
5558 )
5559 }
5560
5561 pub async fn run_lifecycle_state_with_corruption(
5562 &self,
5563 run_id: &str,
5564 ) -> Option<(String, bool, bool, bool, Option<String>)> {
5565 self.runs.lock().await.get(run_id).map(|meta| {
5566 (
5567 meta.active_client_id.clone(),
5568 meta.is_terminal(),
5569 meta.start_committed,
5570 meta.pending_terminal.is_some() || meta.cancellation_pending.is_some(),
5571 meta.trace_corruption.clone(),
5572 )
5573 })
5574 }
5575
5576 /// Atomically fence an orphaned live run to a replacement socket that has
5577 /// already authenticated as the run's agent. The immutable durable owner
5578 /// never changes: proposal and terminal artifacts continue to validate
5579 /// against the original `RunStarted` identity.
5580 pub(crate) async fn resume_run(
5581 &self,
5582 run_id: &str,
5583 agent_id: &str,
5584 replacement_client_id: &str,
5585 ) -> Result<RunResumeBinding, String> {
5586 // Serialize the ownership swap with every durable append/finalization
5587 // for this run. A task that already crossed this fence commits before
5588 // resume; one that has only captured the stale socket identity loses
5589 // after resume and cannot write any bytes.
5590 let durability_lock = self.run_durability_lock(run_id).await;
5591 let _durability_guard = durability_lock.lock().await;
5592 // Session creation/removal takes this same short fence around the map
5593 // mutation. Keep both guards through the run-owner compare-and-swap so
5594 // neither a late contender nor a disconnect can invalidate liveness
5595 // between validation and commit.
5596 let _liveness_guard = self.run_resume_liveness.lock().await;
5597 let live_clients = self
5598 .sessions
5599 .lock()
5600 .await
5601 .keys()
5602 .cloned()
5603 .collect::<std::collections::HashSet<_>>();
5604 if !live_clients.contains(replacement_client_id) {
5605 return Err("runs.resume replacement socket is no longer active".into());
5606 }
5607 let mut runs = self.runs.lock().await;
5608 let meta = runs
5609 .get_mut(run_id)
5610 .ok_or_else(|| "run not found or not authorized".to_string())?;
5611 if meta.agent_id != agent_id {
5612 return Err("run not found or not authorized".into());
5613 }
5614 if meta.is_terminal() {
5615 return Err(format!(
5616 "{} run `{run_id}` is already terminal",
5617 car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX
5618 ));
5619 }
5620 if !meta.start_committed {
5621 return Err(format!(
5622 "{} run `{run_id}` start is not durably committed",
5623 car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX
5624 ));
5625 }
5626 if meta.pending_terminal.is_some()
5627 || meta.cancellation_pending.is_some()
5628 || meta.trace_corruption.is_some()
5629 {
5630 return Err(format!(
5631 "{} run `{run_id}` is quarantined and cannot be resumed",
5632 car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX
5633 ));
5634 }
5635
5636 if meta.active_client_id == replacement_client_id {
5637 let resumed_from_client_id =
5638 meta.resume_predecessor_client_id.clone().ok_or_else(|| {
5639 format!(
5640 "{} run `{run_id}` is already active on this socket",
5641 car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX
5642 )
5643 })?;
5644 return Ok(RunResumeBinding {
5645 run_id: meta.run_id.clone(),
5646 agent_id: meta.agent_id.clone(),
5647 active_client_id: meta.active_client_id.clone(),
5648 resumed_from_client_id,
5649 });
5650 }
5651 if live_clients.contains(&meta.active_client_id) {
5652 return Err(format!(
5653 "{} run `{run_id}` still has a live owner",
5654 car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX
5655 ));
5656 }
5657 let lease_is_current = meta.resume_lease.as_ref().is_some_and(|lease| {
5658 lease.disconnected_client_id == meta.active_client_id
5659 && tokio::time::Instant::now() < lease.expires_at
5660 });
5661 if !lease_is_current {
5662 return Err(format!(
5663 "{} run `{run_id}` resume lease is unavailable or expired",
5664 car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX
5665 ));
5666 }
5667
5668 let resumed_from_client_id = std::mem::replace(
5669 &mut meta.active_client_id,
5670 replacement_client_id.to_string(),
5671 );
5672 meta.resume_predecessor_client_id = Some(resumed_from_client_id.clone());
5673 meta.resume_lease = None;
5674 meta.durability_generation = meta.durability_generation.wrapping_add(1);
5675 Ok(RunResumeBinding {
5676 run_id: meta.run_id.clone(),
5677 agent_id: meta.agent_id.clone(),
5678 active_client_id: meta.active_client_id.clone(),
5679 resumed_from_client_id,
5680 })
5681 }
5682
5683 /// Returns the immutable durable owner and the currently fenced live owner.
5684 pub(crate) async fn run_owner_binding(&self, run_id: &str) -> Option<(String, String)> {
5685 self.runs
5686 .lock()
5687 .await
5688 .get(run_id)
5689 .map(|meta| (meta.client_id.clone(), meta.active_client_id.clone()))
5690 }
5691
5692 /// Append per-turn trace records to a run's in-memory buffer (agent
5693 /// run tracing, U2). The recorder calls this after every
5694 /// `proposal.submit` on a session with a current run, passing the
5695 /// `RunRecord::Turn`s the recorder produced for that proposal.
5696 ///
5697 /// Returns a [`RecordRunTurnsOutcome`]:
5698 /// - `Appended { new_total }` — the batch landed; `new_total` is the
5699 /// run's new total turn count (the caller computes its next
5700 /// `start_index` from it).
5701 /// - `RefusedCeiling` — the batch would take the run past
5702 /// [`RECORD_TURNS_RUN_CEILING`] and was refused WHOLE (the under-lock
5703 /// runaway backstop, ADV-1). Nothing was appended.
5704 /// - `UnknownOrTerminal` — the `run_id` is unknown (the bracket was
5705 /// never opened) or already terminal (a turn arriving after
5706 /// `runs.complete` is dropped — the run is closed). Nothing was
5707 /// appended.
5708 ///
5709 /// U3 flushes the in-memory buffer to disk; U4 broadcasts it. Both read
5710 /// via [`Self::run_turns`].
5711 ///
5712 /// U3: the same turn records are appended to the run's JSONL file so
5713 /// REPLAY (U5) sees them after a restart. Persistence is the commit point:
5714 /// only after the exact JSONL batch is durable do memory and subscribers
5715 /// advance. The operation runs in a daemon-owned task, so dropping a
5716 /// request waiter cannot strand a persisted batch between disk and memory.
5717 pub async fn record_run_turns(
5718 self: &Arc<Self>,
5719 run_id: &str,
5720 records: Vec<car_proto::RunRecord>,
5721 ) -> RecordRunTurnsOutcome {
5722 self.record_run_turns_for_owner(run_id, None, records).await
5723 }
5724
5725 /// Append client-narrated turns only while `active_client_id` still owns
5726 /// the run. The socket id is daemon-minted and captured by the handler
5727 /// before it detaches the work into a daemon-owned task; a later resume
5728 /// therefore fences delayed work from the predecessor without trusting a
5729 /// caller-supplied credential.
5730 #[doc(hidden)]
5731 pub async fn record_run_turns_for_active_owner(
5732 self: &Arc<Self>,
5733 run_id: &str,
5734 active_client_id: &str,
5735 records: Vec<car_proto::RunRecord>,
5736 ) -> RecordRunTurnsOutcome {
5737 self.record_run_turns_for_owner(run_id, Some(active_client_id.to_string()), records)
5738 .await
5739 }
5740
5741 async fn record_run_turns_for_owner(
5742 self: &Arc<Self>,
5743 run_id: &str,
5744 expected_active_client_id: Option<String>,
5745 records: Vec<car_proto::RunRecord>,
5746 ) -> RecordRunTurnsOutcome {
5747 let state = Arc::clone(self);
5748 let run_id = run_id.to_string();
5749 match tokio::spawn(async move {
5750 state
5751 .record_run_turns_owned(&run_id, expected_active_client_id.as_deref(), records)
5752 .await
5753 })
5754 .await
5755 {
5756 Ok(outcome) => outcome,
5757 Err(error) => RecordRunTurnsOutcome::PersistenceFailed(format!(
5758 "durable turn append task failed: {error}"
5759 )),
5760 }
5761 }
5762
5763 /// Make a proposal's authenticated final trace durable exactly once and
5764 /// reflect a newly-appended batch into the live registry/subscription
5765 /// stream before proposal finalization can acknowledge success.
5766 pub async fn ensure_proposal_run_turns(
5767 self: &Arc<Self>,
5768 pending: &crate::run_store::PendingProposalFinalization,
5769 ) -> Result<(), String> {
5770 let state = Arc::clone(self);
5771 let pending = pending.clone();
5772 tokio::spawn(async move { state.ensure_proposal_run_turns_owned(&pending).await })
5773 .await
5774 .map_err(|error| format!("proposal trace durability task failed: {error}"))?
5775 }
5776
5777 async fn ensure_proposal_run_turns_owned(
5778 &self,
5779 pending: &crate::run_store::PendingProposalFinalization,
5780 ) -> Result<(), String> {
5781 let durability_lock = self.run_durability_lock(&pending.run_id).await;
5782 let _durability_guard = durability_lock.lock().await;
5783 let (agent_id, status, generation) = {
5784 let runs = self.runs.lock().await;
5785 let meta = runs
5786 .get(&pending.run_id)
5787 .ok_or_else(|| format!("active run `{}` is absent", pending.run_id))?;
5788 if let Some(error) = &meta.trace_corruption {
5789 return Err(error.clone());
5790 }
5791 if !meta.accepts_proposals() {
5792 return Err(format!("active run `{}` is not writable", pending.run_id));
5793 }
5794 (
5795 meta.agent_id.clone(),
5796 meta.live_status(),
5797 meta.durability_generation,
5798 )
5799 };
5800 let store = self.run_store.clone();
5801 let durable = pending.clone();
5802 let durable_result =
5803 tokio::task::spawn_blocking(move || store.ensure_proposal_turns(&durable))
5804 .await
5805 .map_err(|error| format!("proposal trace durability task failed: {error}"))?;
5806 let mut runs = self.runs.lock().await;
5807 let ensured = match durable_result {
5808 Ok(ensured) => ensured,
5809 Err(error) if crate::run_store::is_trace_corruption_error(&error) => {
5810 let message = self
5811 .quarantine_run_trace_locked(&mut runs, &pending.run_id, error.to_string())
5812 .await;
5813 return Err(message);
5814 }
5815 Err(error) => return Err(format!("proposal trace durability failed: {error}")),
5816 };
5817 let meta = runs.get(&pending.run_id).ok_or_else(|| {
5818 format!(
5819 "active run `{}` disappeared after durability",
5820 pending.run_id
5821 )
5822 })?;
5823 if let Some(error) = &meta.trace_corruption {
5824 return Err(error.clone());
5825 }
5826 if meta.durability_generation != generation || !meta.accepts_proposals() {
5827 return Err(format!(
5828 "active run `{}` changed during proposal trace durability",
5829 pending.run_id
5830 ));
5831 }
5832 let already_live = !ensured.appended
5833 && meta.turns.iter().any(|record| match record {
5834 car_proto::RunRecord::Turn(turn) => {
5835 turn.proposal_id.as_deref() == Some(pending.final_proposal_id.as_str())
5836 }
5837 _ => false,
5838 });
5839 if ensured.appended || !already_live {
5840 let meta = runs
5841 .get_mut(&pending.run_id)
5842 .expect("run registry remains locked during proposal trace commit");
5843 let base = ensured
5844 .records
5845 .first()
5846 .and_then(|record| match record {
5847 car_proto::RunRecord::Turn(turn) => Some(turn.index),
5848 _ => None,
5849 })
5850 .unwrap_or(meta.turns.len());
5851 meta.turns.extend(ensured.records.iter().cloned());
5852 meta.durability_generation = meta.durability_generation.wrapping_add(1);
5853 let subs = self.run_subscribers.lock().await;
5854 for (offset, record) in ensured.records.into_iter().enumerate() {
5855 Self::fanout_locked(
5856 &subs,
5857 &pending.run_id,
5858 &agent_id,
5859 record,
5860 base + offset + 1,
5861 status,
5862 );
5863 }
5864 }
5865 Ok(())
5866 }
5867
5868 async fn record_run_turns_owned(
5869 &self,
5870 run_id: &str,
5871 expected_active_client_id: Option<&str>,
5872 mut records: Vec<car_proto::RunRecord>,
5873 ) -> RecordRunTurnsOutcome {
5874 // Serialize durability only with this run. The global registry lock is
5875 // held for the admission snapshot and memory/fanout commit, never for
5876 // trace scanning, append, flush, or fsync.
5877 let durability_lock = self.run_durability_lock(run_id).await;
5878 let _durability_guard = durability_lock.lock().await;
5879 let (agent_id, base, new_total, status, generation) = {
5880 let mut runs = self.runs.lock().await;
5881 match runs.get_mut(run_id) {
5882 Some(meta) => {
5883 if let Some(error) = &meta.trace_corruption {
5884 return RecordRunTurnsOutcome::PersistenceFailed(error.clone());
5885 }
5886 if expected_active_client_id
5887 .is_some_and(|expected| meta.active_client_id != expected)
5888 {
5889 return RecordRunTurnsOutcome::UnknownOrTerminal;
5890 }
5891 if !meta.accepts_proposals() {
5892 return RecordRunTurnsOutcome::UnknownOrTerminal;
5893 }
5894 // ADV-1: hard ceiling, under the lock. Count only the
5895 // `Turn` records — Started/Ended are not buffered here,
5896 // and the ceiling is a turn-count cap. Refuse the whole
5897 // batch if appending it would cross the ceiling.
5898 let incoming_turns = records
5899 .iter()
5900 .filter(|r| matches!(r, car_proto::RunRecord::Turn(_)))
5901 .count();
5902 if meta.turns.len() + incoming_turns > RECORD_TURNS_RUN_CEILING {
5903 return RecordRunTurnsOutcome::RefusedCeiling;
5904 } else {
5905 // Position of the first record about to be appended —
5906 // its post-append cursor is `base + 1`.
5907 let base = meta.turns.len();
5908 // FIX 6: re-stamp each turn's `index` from the LIVE
5909 // append position under the `runs` lock. The caller
5910 // computes a provisional `start_index` from a pre-read
5911 // `run_turn_count` OUTSIDE this lock — a TOCTOU: two
5912 // concurrent proposals on one connection can read the
5913 // same `start_index` and bake overlapping indices.
5914 // Authoritatively assigning the index here, from
5915 // `base + offset`, makes it contiguous and correct
5916 // regardless of the racing reads upstream.
5917 for (offset, record) in records.iter_mut().enumerate() {
5918 if let car_proto::RunRecord::Turn(turn) = record {
5919 turn.index = base + offset;
5920 }
5921 }
5922 let agent_id = meta.agent_id.clone();
5923 let status = meta.live_status();
5924 (
5925 agent_id,
5926 base,
5927 base + records.len(),
5928 status,
5929 meta.durability_generation,
5930 )
5931 }
5932 }
5933 None => return RecordRunTurnsOutcome::UnknownOrTerminal,
5934 }
5935 };
5936
5937 let store = self.run_store.clone();
5938 let durable_agent = agent_id.clone();
5939 let durable_run = run_id.to_string();
5940 let durable_records = records.clone();
5941 let append = tokio::task::spawn_blocking(move || {
5942 store.append_turns(&durable_agent, &durable_run, &durable_records)
5943 })
5944 .await;
5945 let mut runs = self.runs.lock().await;
5946 match append {
5947 Ok(Ok(())) => {}
5948 Ok(Err(error)) => {
5949 tracing::warn!(run_id, %error, "run-store: failed to persist turns");
5950 if crate::run_store::is_trace_corruption_error(&error) {
5951 let message = self
5952 .quarantine_run_trace_locked(&mut runs, run_id, error.to_string())
5953 .await;
5954 return RecordRunTurnsOutcome::PersistenceFailed(message);
5955 }
5956 return RecordRunTurnsOutcome::PersistenceFailed(error.to_string());
5957 }
5958 Err(error) => {
5959 tracing::warn!(run_id, %error, "run-store: durable turn task failed");
5960 return RecordRunTurnsOutcome::PersistenceFailed(error.to_string());
5961 }
5962 }
5963
5964 let meta = runs
5965 .get_mut(run_id)
5966 .expect("run registry entry survives a committed turn append");
5967 if let Some(error) = &meta.trace_corruption {
5968 return RecordRunTurnsOutcome::PersistenceFailed(error.clone());
5969 }
5970 if meta.durability_generation != generation || !meta.accepts_proposals() {
5971 return RecordRunTurnsOutcome::PersistenceFailed(format!(
5972 "run `{run_id}` changed during durable turn append"
5973 ));
5974 }
5975 meta.turns.extend(records.iter().cloned());
5976 meta.durability_generation = meta.durability_generation.wrapping_add(1);
5977 let subs = self.run_subscribers.lock().await;
5978 for (offset, record) in records.into_iter().enumerate() {
5979 Self::fanout_locked(&subs, run_id, &agent_id, record, base + offset + 1, status);
5980 }
5981 RecordRunTurnsOutcome::Appended { new_total }
5982 }
5983
5984 /// Number of turns already recorded for a run — the `start_index` the
5985 /// recorder passes so per-turn `index` stays monotonic across the
5986 /// run's proposals. `0` for an unknown run.
5987 pub async fn run_turn_count(&self, run_id: &str) -> usize {
5988 self.runs
5989 .lock()
5990 .await
5991 .get(run_id)
5992 .map(|m| m.turns.len())
5993 .unwrap_or(0)
5994 }
5995
5996 /// Clone of a run's ordered per-turn trace (agent run tracing, U2).
5997 /// This is the accessor U3 reads to flush turns to disk and U4 reads
5998 /// to broadcast them. Empty Vec for an unknown run or a run with no
5999 /// turns yet.
6000 pub async fn run_turns(&self, run_id: &str) -> Vec<car_proto::RunRecord> {
6001 self.runs
6002 .lock()
6003 .await
6004 .get(run_id)
6005 .map(|m| m.turns.clone())
6006 .unwrap_or_default()
6007 }
6008
6009 /// Atomically snapshot a run's turns AND register a live
6010 /// `runs.trace.event` subscriber for `(run_id, host_client_id)` —
6011 /// the load-bearing invariant #1 (agent run tracing, U4).
6012 ///
6013 /// Holds the `runs` lock across BOTH (a) reading the run's current
6014 /// turn buffer as the snapshot at cursor `C = turns.len()` and (b)
6015 /// inserting the subscriber into `run_subscribers`. Because the
6016 /// lifecycle path (`record_run_turns` / `complete_run` /
6017 /// `mark_run_incomplete`) appends-and-notifies under the SAME lock,
6018 /// the snapshot contains exactly the turns ≤ C and every turn appended
6019 /// after registration is delivered — no turn in the snapshot/register
6020 /// window is dropped (gap) or double-delivered (dup).
6021 ///
6022 /// Spawns the subscriber's drain task (bounded channel → WS) so the
6023 /// producer never writes the socket directly (invariant #2). Returns
6024 /// the snapshot response shape. `None` if the `run_id` is unknown (the
6025 /// handler maps that to a not-found error). A re-subscribe by the same
6026 /// `(run_id, host_client_id)` replaces the prior subscriber (its drain
6027 /// task ends when the old sender drops) and re-snapshots — the R8
6028 /// reconnect path: the fresh snapshot covers any turns emitted during
6029 /// the gap with no dup.
6030 ///
6031 /// Terminal-run disk fallback: a completed/incomplete run has its
6032 /// in-memory `turns` evicted for heap hygiene (see
6033 /// [`Self::clear_terminal_run_turns`]). When the run is terminal AND its
6034 /// in-memory buffer is empty, the snapshot is re-sourced from the disk
6035 /// store ([`crate::run_store::RunStore::get_run_trace`], the same source
6036 /// `runs.get_trace` reads), filtered to `RunRecord::Turn` so the shape
6037 /// matches the live in-memory snapshot. A terminal run takes no further
6038 /// appends, so the disk trail is final and there is no gap/dup concern —
6039 /// the snapshot/register atomicity the `runs` lock provides is only
6040 /// load-bearing for the in-progress path, which is unchanged.
6041 pub(crate) async fn subscribe_run_page(
6042 &self,
6043 run_id: &str,
6044 host_client_id: &str,
6045 channel: Arc<WsChannel>,
6046 cursor: usize,
6047 limit: usize,
6048 ) -> Result<Option<RunSubscribePageResult>, String> {
6049 // Build the subscriber (and its drain task) OUTSIDE the runs lock
6050 // so spawning never happens under the hot lock; registration
6051 // itself is the only thing serialized.
6052 let subscriber =
6053 crate::host::RunTraceSubscriber::spawn(host_client_id.to_string(), channel);
6054
6055 let (agent_id, mut expected_state) = {
6056 let runs = self.runs.lock().await;
6057 let Some(meta) = runs.get(run_id) else {
6058 return Ok(None);
6059 };
6060 if let Some(error) = &meta.trace_corruption {
6061 return Err(error.clone());
6062 }
6063 if meta.is_terminal() {
6064 return Ok(Some(RunSubscribePageResult::Durable {
6065 agent_id: meta.agent_id.clone(),
6066 status: meta.live_status(),
6067 }));
6068 }
6069 (
6070 meta.agent_id.clone(),
6071 (meta.turns.len(), meta.pending_terminal.is_some()),
6072 )
6073 };
6074
6075 for attempt in 0..=RUN_SUBSCRIBE_SUMMARY_STATE_RETRY_LIMIT {
6076 // Keep disk I/O off the global run-registry lock. A concurrent
6077 // mutation advances the captured state under that lock; retrying
6078 // the summary read then gives snapshot/registration one coherent
6079 // durable/live boundary.
6080 let store = self.run_store.clone();
6081 let durable_agent = agent_id.clone();
6082 let durable_run = run_id.to_string();
6083 let durable_corruption = tokio::task::spawn_blocking(move || {
6084 store.run_trace_corruption_for(&durable_agent, &durable_run)
6085 })
6086 .await
6087 .map_err(|error| format!("run trace corruption check task failed: {error}"))?
6088 .map_err(|error| format!("run trace corruption check failed: {error}"))?;
6089 let mut runs = self.runs.lock().await;
6090 let Some(meta) = runs.get(run_id) else {
6091 return Ok(None);
6092 };
6093 if let Some(error) = &meta.trace_corruption {
6094 return Err(error.clone());
6095 }
6096 if meta.agent_id != agent_id {
6097 return Err(format!("run `{run_id}` changed ownership during subscribe"));
6098 }
6099 if let Some(corruption) = durable_corruption {
6100 let message = self
6101 .quarantine_run_trace_locked(
6102 &mut runs,
6103 run_id,
6104 format!("malformed run trace record at line {}", corruption.line),
6105 )
6106 .await;
6107 return Err(message);
6108 }
6109 let status = meta.live_status();
6110 if meta.is_terminal() {
6111 return Ok(Some(RunSubscribePageResult::Durable { agent_id, status }));
6112 }
6113 let current_state = (meta.turns.len(), meta.pending_terminal.is_some());
6114 if current_state != expected_state {
6115 if attempt == RUN_SUBSCRIBE_SUMMARY_STATE_RETRY_LIMIT {
6116 return Err(format!(
6117 "run `{run_id}` changed repeatedly during durable subscribe validation; retry"
6118 ));
6119 }
6120 expected_state = current_state;
6121 drop(runs);
6122 continue;
6123 }
6124 let live_cursor = meta.turns.len();
6125 if cursor > live_cursor {
6126 return Err(format!(
6127 "runs.subscribe cursor {cursor} exceeds live_cursor {live_cursor}"
6128 ));
6129 }
6130 let end = cursor.saturating_add(limit).min(live_cursor);
6131 let turns = meta.turns[cursor..end].to_vec();
6132 let next_cursor = (end < live_cursor).then_some(end);
6133 let subscribed = next_cursor.is_none();
6134 if subscribed {
6135 let mut subs = self.run_subscribers.lock().await;
6136 subs.insert((run_id.to_string(), host_client_id.to_string()), subscriber);
6137 }
6138 drop(runs);
6139
6140 return Ok(Some(RunSubscribePageResult::Ready(
6141 car_proto::RunSubscribeResponse {
6142 run_id: run_id.to_string(),
6143 agent_id,
6144 turns,
6145 cursor,
6146 limit,
6147 next_cursor,
6148 live_cursor,
6149 subscribed,
6150 status,
6151 },
6152 )));
6153 }
6154 unreachable!("bounded subscribe validation loop always returns")
6155 }
6156
6157 /// Remove a live run-trace subscriber for `(run_id, host_client_id)`
6158 /// (agent run tracing, U4). Returns `true` if a subscription existed.
6159 /// Dropping the [`crate::host::RunTraceSubscriber`] drops its channel
6160 /// sender, which ends the drain task.
6161 pub async fn unsubscribe_run(&self, run_id: &str, host_client_id: &str) -> bool {
6162 self.run_subscribers
6163 .lock()
6164 .await
6165 .remove(&(run_id.to_string(), host_client_id.to_string()))
6166 .is_some()
6167 }
6168
6169 /// Drop every live run-trace subscription owned by `host_client_id`
6170 /// (agent run tracing, U4 — R8 cleanup). Called from
6171 /// [`remove_session`](crate::session::ServerState::remove_session) on disconnect so a CarHost that drops doesn't
6172 /// leave dangling drain tasks. Reconnect-durability is client-side:
6173 /// the run stays subscribable while it lives, and the CarHost re-issues
6174 /// `runs.subscribe {run_id}` on its new connection — the server never
6175 /// synthesizes a failure on subscriber drop.
6176 pub async fn drop_run_subscribers_for_client(&self, host_client_id: &str) {
6177 self.run_subscribers
6178 .lock()
6179 .await
6180 .retain(|(_run, client), _| client != host_client_id);
6181 }
6182
6183 /// Disconnect cleanup for agent runs (R5). Called from
6184 /// [`remove_session`] with the `client_id` of the dropping
6185 /// connection. For every run this connection owns that is **not**
6186 /// yet terminal, wait a short grace window then re-check: if a
6187 /// concurrently-dispatched `runs.complete` made it terminal in the
6188 /// meantime, leave it; otherwise mark it `Incomplete`. The grace
6189 /// window is the fix for the serve-mode false-positive where a
6190 /// healthy `runs.complete` is still in flight (in a spawned
6191 /// dispatch task) when the close frame arrives (Risk: run identity
6192 /// races).
6193 async fn sweep_runs_for_disconnect(&self, session: &ClientSession) {
6194 let client_id = session.client_id.as_str();
6195 // Snapshot the still-open runs owned by this connection.
6196 let pending: Vec<(String, Option<tokio::time::Instant>)> = {
6197 let runs = self.runs.lock().await;
6198 runs.values()
6199 .filter(|m| m.active_client_id == client_id && !m.is_terminal())
6200 .map(|m| {
6201 let resume_deadline = m.resume_lease.as_ref().and_then(|lease| {
6202 (lease.disconnected_client_id == client_id).then_some(lease.expires_at)
6203 });
6204 (m.run_id.clone(), resume_deadline)
6205 })
6206 .collect()
6207 };
6208 if pending.is_empty() {
6209 return;
6210 }
6211 for (run_id, resume_deadline) in pending {
6212 // The lease deadline was installed atomically with removal from
6213 // `sessions`; sleeping to that absolute instant keeps teardown
6214 // work before this point from silently extending the contract.
6215 match resume_deadline {
6216 Some(deadline) => tokio::time::sleep_until(deadline).await,
6217 None => tokio::time::sleep(RUN_COMPLETE_GRACE).await,
6218 }
6219 match self.run_lifecycle_state(&run_id).await {
6220 None | Some((_, true, _, _)) => continue,
6221 Some((owner, false, _, _)) if owner != client_id => {
6222 // An authenticated replacement claimed this run during the
6223 // disconnect grace window. The old owner is fenced and
6224 // must not be allowed to close the replacement's run.
6225 continue;
6226 }
6227 Some((_, false, true, _)) => {}
6228 Some((_, false, false, _)) => {
6229 match self
6230 .reconcile_or_release_unacknowledged_start(session, &run_id)
6231 .await
6232 {
6233 Ok(true) => continue,
6234 Ok(false) => {}
6235 Err(error) => {
6236 tracing::error!(run_id, client_id, %error, "failed to reconcile or release an unacknowledged RunStarted transaction");
6237 continue;
6238 }
6239 }
6240 }
6241 }
6242 match self.run_store.pending_proposal(&run_id) {
6243 Ok(Some(_)) => {
6244 tracing::warn!(
6245 run_id,
6246 client_id,
6247 "leaving run open because proposal finalization is pending durable replay"
6248 );
6249 continue;
6250 }
6251 Err(error) => {
6252 tracing::error!(run_id, client_id, %error, "leaving run open because proposal finalization state is unreadable and outcome is unknown");
6253 continue;
6254 }
6255 Ok(None) => {}
6256 }
6257 match self.run_store.execution_marker(&run_id) {
6258 Ok(Some(marker)) => {
6259 tracing::warn!(
6260 run_id,
6261 client_id,
6262 original_proposal_id = marker.original_proposal_id,
6263 "leaving run open because proposal execution outcome is unknown"
6264 );
6265 continue;
6266 }
6267 Err(error) => {
6268 tracing::error!(run_id, client_id, %error, "leaving run open because proposal execution marker is unreadable and outcome is unknown");
6269 continue;
6270 }
6271 Ok(None) => {}
6272 }
6273 if let Some(ended) = self
6274 .prepare_run_incomplete_for_active_owner(&run_id, Some(client_id))
6275 .await
6276 {
6277 let durable_client_id = ended.client_id.as_deref().unwrap_or(client_id);
6278 let append_result = match session
6279 .append_run_terminal_event_once(&ended, durable_client_id)
6280 .await
6281 {
6282 Err(error) if error.is_retry_safe() => {
6283 tracing::warn!(
6284 run_id,
6285 client_id,
6286 %error,
6287 "disconnect terminal durability is unknown; retrying the exact journal row once"
6288 );
6289 session
6290 .append_run_terminal_event_once(&ended, durable_client_id)
6291 .await
6292 }
6293 result => result,
6294 };
6295 if let Err(error) = append_result {
6296 tracing::error!(run_id, client_id, %error, "failed to bind disconnect terminal journal event");
6297 continue;
6298 }
6299 if let Err(error) = self.commit_run_completion(&ended).await {
6300 tracing::error!(run_id, client_id, %error, "failed to commit disconnect terminal");
6301 continue;
6302 }
6303 if let Err(error) = session.clear_run_journal_binding(&run_id).await {
6304 tracing::error!(run_id, client_id, %error, "failed to clear durable disconnect journal binding");
6305 continue;
6306 }
6307 let mut current = session.current_run_id.lock().await;
6308 if current.as_deref() == Some(run_id.as_str()) {
6309 *current = None;
6310 }
6311 }
6312 }
6313 }
6314
6315 /// Fails only when `<home>/.car/policies/` holds a malformed rule file —
6316 /// see [`apply_project_policies`] for why that is fatal rather than a
6317 /// warning.
6318 pub async fn create_session(
6319 &self,
6320 client_id: &str,
6321 channel: Arc<WsChannel>,
6322 ) -> Result<Arc<ClientSession>, String> {
6323 let journal_path = self.journal_dir.join(format!("{}.jsonl", client_id));
6324 let event_log = match self.journal_failures.clone() {
6325 Some(failures) => EventLog::with_journal_failure_injector(journal_path, failures),
6326 None => EventLog::with_journal(journal_path),
6327 };
6328
6329 let negotiated_capabilities =
6330 Arc::new(std::sync::RwLock::new(std::collections::BTreeSet::new()));
6331 let halted = Arc::new(AtomicBool::new(false));
6332 let ws_executor = Arc::new(WsToolExecutor::new(
6333 channel.clone(),
6334 negotiated_capabilities.clone(),
6335 halted.clone(),
6336 ));
6337
6338 // Compose: connector (remote MCP) tools route to the shared
6339 // process-wide executor; everything else falls back to this
6340 // session's WS tool executor. Both views share the same routing
6341 // table, so a connector enabled in any session is callable here.
6342 let executor: Arc<dyn ToolExecutor> =
6343 Arc::new(self.mcp_executor.share_with_fallback(ws_executor));
6344
6345 // Outbound human messaging. Built-in iMessage adapter over the same
6346 // `RealMessageSender` + `~/.car` config store the daemon's
6347 // `messaging.*` handlers use, plus a host fallback so a channel CAR has
6348 // no built-in transport for (Teams, per Parslee-ai/car#885) is
6349 // delivered by the host rather than compiled into the runtime.
6350 //
6351 // NO APPROVAL LOOP. `RealMessageSender` deliberately calls the *un-gated*
6352 // `car_ffi_common::integrations::messages_send` so that the approval
6353 // transport's own prompt does not raise an approval for itself. That
6354 // choice is safe to reuse here, and the recursion this new path might
6355 // look like it opens does not exist, because the two directions never
6356 // meet: `messaging.send` → registry → adapter → `messages_send` is a
6357 // straight line into the JXA bridge with no re-entry into the runtime,
6358 // and an approval raised *for* `messaging.send` is delivered by the
6359 // orchestrator/fanout path, which calls `MessageSender` directly and
6360 // never dispatches a runtime tool. Neither leg can call the other, so
6361 // there is no cycle to bound. Keep it that way: routing approval
6362 // delivery through `messaging.send` WOULD create one.
6363 let outbound = Arc::new(car_messaging::outbound::OutboundRegistry::new());
6364 outbound.register(Arc::new(
6365 car_messaging::outbound::ImessageOutboundAdapter::new(
6366 Arc::new(crate::messaging_orchestrator::RealMessageSender),
6367 crate::messaging_config::MessagingConfigStore::from_home(),
6368 ),
6369 ));
6370 outbound.set_fallback(Arc::new(crate::host_channel::HostChannelAdapter::new(
6371 executor.clone(),
6372 )));
6373
6374 // The trajectory store is the daemon-wide one, so traces from every
6375 // session accumulate in a single history. Without this the executor's
6376 // `persist_trajectory` short-circuits on its `self.trajectory_store
6377 // .as_ref()?` and nothing is ever recorded — which is why the
6378 // per-tool success rates had no data behind them.
6379 let runtime = Runtime::new()
6380 .with_event_log(event_log)
6381 .with_executor(executor)
6382 .with_trajectory_store(self.trajectory_store.clone())
6383 .with_message_sink(outbound);
6384
6385 // Declarative rules from `~/.car/policies/*.toml`. Same `~/.car`
6386 // convention the messaging config and the approval journal use — the
6387 // daemon serves whatever project a client happens to be working in, so
6388 // its policy set is the operator's, not a per-project one. (The
6389 // assistant, which IS rooted in a project, loads `<root>/.car`.)
6390 if let Some(car_dir) = car_home_dir() {
6391 apply_project_policies(&runtime, &car_dir).await?;
6392 }
6393
6394 // The session's permission-tier gate. Built here rather than inline in
6395 // the `ClientSession` literal below because the admission gate
6396 // registered a few lines down has to hold the SAME object the
6397 // `permission.*` RPCs mutate — see the field doc on
6398 // `ClientSession::permission_gate`.
6399 //
6400 // `SandboxEdit` is the default `docs/websocket-protocol.md` publishes,
6401 // and the code had drifted to `ReadOnly` while nothing enforced the
6402 // tier. `RiskClassifier::baseline` maps both `StateWrite` and
6403 // `ToolCall` to `SandboxEdit`, so keeping the drifted `ReadOnly`
6404 // default the moment enforcement lands would escalate essentially
6405 // every action every existing binding client submits. `full_access`
6406 // actions still escalate — mandatory HITL, exactly as documented.
6407 let permission_gate = Arc::new(tokio::sync::RwLock::new(car_policy::PermissionGate::new(
6408 car_policy::PermissionTier::SandboxEdit,
6409 )));
6410 // These handles are shared with the permission admission gate so it
6411 // observes the identity established later by `session.auth`; taking a
6412 // snapshot here would leave every newly-created session unbound.
6413 let authenticated = Arc::new(std::sync::atomic::AtomicBool::new(false));
6414 let agent_id = Arc::new(tokio::sync::Mutex::new(None));
6415 let callback_tool_schema_digests = Arc::new(tokio::sync::RwLock::new(HashMap::new()));
6416
6417 // If the embedder supplied a shared memgine, every session uses it.
6418 // Otherwise each session gets its own — matches pre-extraction behavior.
6419 // Create it before the admission gate so proposal.submit and the
6420 // permission advisory RPCs consult the exact same live skill ceilings.
6421 let memgine = match &self.shared_memgine {
6422 Some(eng) => eng.clone(),
6423 None => Arc::new(Mutex::new(car_memgine::MemgineEngine::new(None))),
6424 };
6425
6426 // A terminal callback poisons this WebSocket session after the current
6427 // proposal's engine-level abort and rollback. Register the latch as an
6428 // ordinary admission gate so a later proposal gets the same complete
6429 // rejected-result shape as every other pre-execution refusal.
6430 runtime
6431 .register_admission_gate(Arc::new(SessionHaltAdmissionGate {
6432 halted: halted.clone(),
6433 }))
6434 .await;
6435
6436 // Statically verify submitted proposals before any action dispatches.
6437 //
6438 // This is the surface that matters for it: `proposal.submit` accepts
6439 // caller-authored, **multi-action** proposals, and per-action validation
6440 // only rejects an action once execution reaches it — so a bad tool name
6441 // in action 5 is caught after actions 1–4 have already had their side
6442 // effects. The gate refuses the whole proposal up front, so nothing
6443 // partial happens. (On a runtime that submits one action at a time, like
6444 // the assistant loop, it adds nothing the validator wasn't already
6445 // doing; see `car_engine::verify_gate` for what blocks and why.)
6446 runtime
6447 .register_admission_gate(Arc::new(car_engine::StaticVerificationGate::new(
6448 runtime.tools.clone(),
6449 )))
6450 .await;
6451
6452 // Out-of-process supervision (proposal item 5). Registered
6453 // unconditionally: with no supervisor subscribed the gate short-circuits
6454 // to Allow on an empty-map read, so leaving it on costs nothing and
6455 // means a supervisor can attach to a session that was already running
6456 // rather than only to sessions created after it connected.
6457 runtime
6458 .register_admission_gate(Arc::new(crate::supervision::SupervisionGate::new(
6459 self.supervision.clone(),
6460 )))
6461 .await;
6462
6463 // Enforce the session's granted permission tier before any action
6464 // dispatches (Parslee-ai/car#890).
6465 //
6466 // Everything this needs already existed — the per-session tier gate,
6467 // the durable daemon-wide approval ledger, `permission.approve` — but
6468 // nothing consulted any of it on the way to the executor, so an action
6469 // the daemon's own `permission.evaluate` called `needs_approval`
6470 // executed anyway and its state write persisted. This is the missing
6471 // call site: it asks the SAME gate the same question `permission.
6472 // evaluate` asks, against the SAME shared ledger, so the advisory
6473 // answer and the enforced one cannot diverge. An escalation blocks the
6474 // whole proposal until an operator approves the named fingerprint;
6475 // approving it makes the next submit come back `Allow` with no
6476 // escalation raised at all.
6477 runtime
6478 .register_admission_gate(Arc::new(
6479 crate::permission_gate::PermissionAdmissionGate::new(
6480 permission_gate.clone(),
6481 self.approval_ledger.clone(),
6482 )
6483 .with_authenticated_agent(authenticated.clone(), agent_id.clone())
6484 .with_callback_tools(
6485 callback_tool_schema_digests.clone(),
6486 runtime.registry.clone(),
6487 )
6488 .with_skill_memgine(memgine.clone())
6489 .with_event_log(runtime.log.clone()),
6490 ))
6491 .await;
6492
6493 let session = Arc::new(ClientSession {
6494 client_id: client_id.to_string(),
6495 runtime: Arc::new(runtime),
6496 channel,
6497 host: self.host.clone(),
6498 memgine,
6499 browser: car_ffi_common::browser::BrowserSessionSlot::new(),
6500 // When auth is disabled (no token installed), every
6501 // session is "authenticated" by default — preserves the
6502 // pre-#32 behaviour. When auth is enabled, the value is
6503 // ignored on creation; the dispatcher's gate checks
6504 // `ServerState::auth_token.is_some()` to decide whether
6505 // to enforce.
6506 authenticated,
6507 negotiated_protocol_version: std::sync::atomic::AtomicU32::new(0),
6508 negotiated_capabilities,
6509 inference_control: Arc::new(crate::inference_control::InferenceRegistry::default()),
6510 is_host: std::sync::atomic::AtomicBool::new(false),
6511 agent_id,
6512 agent_method_allowlist: Arc::new(std::sync::RwLock::new(None)),
6513 callback_tool_schema_digests,
6514 memory_namespace: tokio::sync::Mutex::new(None),
6515 bound_memgine: tokio::sync::Mutex::new(None),
6516 current_run_id: tokio::sync::Mutex::new(None),
6517 run_lifecycle_guard: Arc::new(tokio::sync::Mutex::new(())),
6518 permission_gate,
6519 halted,
6520 evolution_guard: crate::evolution::CycleGuard::default(),
6521 last_chat_turn: tokio::sync::Mutex::new(None),
6522 chat_inflight: std::sync::atomic::AtomicUsize::new(0),
6523 tenant: tokio::sync::Mutex::new(None),
6524 tool_stream_subscribed: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)),
6525 });
6526
6527 // Seed connector tools already enabled (in any session) into this
6528 // new session's runtime registry so the model sees them at once.
6529 // Only touches the manager if connectors have been initialized —
6530 // embedders that never use connectors pay nothing here.
6531 if self.connectors.get().is_some() {
6532 for entry in self.connectors().enabled_tool_entries().await {
6533 session.runtime.register_tool_entry(entry).await;
6534 }
6535 }
6536
6537 {
6538 let _liveness_guard = self.run_resume_liveness.lock().await;
6539 self.sessions
6540 .lock()
6541 .await
6542 .insert(client_id.to_string(), session.clone());
6543 }
6544
6545 Ok(session)
6546 }
6547
6548 /// Bind a session's runtime to the [`car_engine::McpSubstrate`] backed by an
6549 /// already-connected MCP connector, making that session **coherent**:
6550 /// its commodity built-ins (`read_file`/`write_file`/…/`run_command`)
6551 /// act on the *same* environment as its `mcp_{slug}_*` connector tools,
6552 /// instead of being split between the host/WS client and the remote MCP
6553 /// server (`docs/execution-substrate.md` §3, phase 3).
6554 ///
6555 /// This is **opt-in and explicit** — it is reached only via the
6556 /// `session.bindSubstrate` JSON-RPC method, never inferred from a
6557 /// connector merely being enabled. Sessions that never call it keep the
6558 /// exact historic composition (`share_with_fallback(ws_executor)` over a
6559 /// default `LocalSubstrate`), so GUI / A2A-over-WS / connectors-as-tools
6560 /// / coder behavior is unchanged.
6561 ///
6562 /// On bind we:
6563 /// 1. Wrap the live connector session (shared with the
6564 /// `mcp_{slug}_*` routes — no second dial) as an [`car_engine::McpSubstrate`] and
6565 /// `set_substrate` it on the session runtime.
6566 /// 2. `register_agent_basics()` so the model sees portable names
6567 /// (`read_file`/`run_command`/…) that the engine routes to the bound
6568 /// substrate.
6569 /// 3. Swap the session executor to a [`SubstrateShadowExecutor`] over the
6570 /// same `share_with_fallback(ws_executor)` composition, so the bare
6571 /// substrate-owned names fall through to the substrate while connector
6572 /// routes and host tool callbacks keep working unchanged (option (a)).
6573 ///
6574 /// Returns the substrate's environment name (the connector slug) on
6575 /// success, or an error string if the connector is not connected.
6576 pub async fn bind_substrate_to_connector(
6577 &self,
6578 session: &Arc<ClientSession>,
6579 slug: &str,
6580 ) -> Result<String, String> {
6581 self.ensure_connectors_loaded().await;
6582
6583 // The connector's live MCP session is the same handle the
6584 // `mcp_{slug}_*` routes dispatch through — wrap it, don't re-dial.
6585 let mcp_session = self
6586 .mcp_executor
6587 .session(slug)
6588 .await
6589 .ok_or_else(|| format!("connector '{slug}' is not connected"))?;
6590
6591 let substrate: Arc<dyn car_engine::Substrate> =
6592 Arc::new(car_engine::McpSubstrate::new(mcp_session, slug.to_string()));
6593 let name = substrate.name().to_string();
6594
6595 // 1. Bind the substrate; 2. register substrate-backed built-ins.
6596 //
6597 // These three steps (set_substrate → register_agent_basics →
6598 // set_executor below) are not one critical section, so a tool `execute`
6599 // racing on this same session mid-swap could observe a half-applied
6600 // state. That's benign here: a WS connection processes its JSON-RPC
6601 // requests sequentially, and `bindSubstrate` is a client control call
6602 // issued before tool work — so no in-flight built-in overlaps the swap.
6603 session.runtime.set_substrate(substrate).await;
6604 session.runtime.register_agent_basics().await;
6605
6606 // 3. Shadow the substrate-owned names so they fall through to the
6607 // bound substrate, keeping connector routes + WS host callbacks.
6608 let ws_executor = Arc::new(WsToolExecutor::new(
6609 session.channel.clone(),
6610 session.negotiated_capabilities.clone(),
6611 session.halted.clone(),
6612 ));
6613 let composed: Arc<dyn ToolExecutor> =
6614 Arc::new(self.mcp_executor.share_with_fallback(ws_executor));
6615 let shadowed: Arc<dyn ToolExecutor> = Arc::new(SubstrateShadowExecutor::new(composed));
6616 session.runtime.set_executor(shadowed).await;
6617
6618 Ok(name)
6619 }
6620
6621 /// Remove a per-client session from the registry on disconnect.
6622 /// Returns the removed session if present so callers can drop any
6623 /// remaining strong refs (e.g. drain pending tool callbacks). Fix
6624 /// for MULTI-4 / WS-3 — without this, `state.sessions` retains
6625 /// `Arc<ClientSession>` for every connection that ever existed.
6626 pub async fn remove_session(&self, client_id: &str) -> Option<Arc<ClientSession>> {
6627 // Cancel any live detached tool invocations on this session's
6628 // runtime (linus review D3): after disconnect the per-session
6629 // registry is unreachable, so a still-running tool would be
6630 // uncancellable until daemon restart. Same teardown discipline
6631 // as run subscribers / chat sessions below.
6632 if let Some(session) = self.sessions.lock().await.get(client_id).cloned() {
6633 let n = session.runtime.tool_handles.cancel_all().await;
6634 if n > 0 {
6635 tracing::info!(
6636 client_id,
6637 cancelled = n,
6638 "cancelled detached tools at session teardown"
6639 );
6640 }
6641 }
6642
6643 let removed = {
6644 let _liveness_guard = self.run_resume_liveness.lock().await;
6645 let removed = self.sessions.lock().await.remove(client_id);
6646 let resume_negotiated = removed.as_ref().is_some_and(|session| {
6647 session
6648 .negotiated_capabilities
6649 .read()
6650 .map(|caps| caps.contains(car_proto::RUNS_RESUME_CAPABILITY))
6651 .unwrap_or(false)
6652 });
6653 if resume_negotiated {
6654 let expires_at = tokio::time::Instant::now() + self.run_resume_lease;
6655 let mut runs = self.runs.lock().await;
6656 for meta in runs
6657 .values_mut()
6658 .filter(|meta| meta.active_client_id == client_id && !meta.is_terminal())
6659 {
6660 meta.resume_lease = Some(RunResumeLease {
6661 disconnected_client_id: client_id.to_string(),
6662 expires_at,
6663 });
6664 }
6665 }
6666 removed
6667 };
6668 if let Some(session) = &removed {
6669 // #169: drop the agent_id → client_id binding so a
6670 // disconnected lifecycle agent can reconnect (or its
6671 // supervisor-respawned replacement can take the slot)
6672 // without colliding with the stale claim.
6673 let bound = session.agent_id.lock().await.clone();
6674 if let Some(id) = bound {
6675 let mut attached = self.attached_agents.lock().await;
6676 if attached.get(&id).map(String::as_str) == Some(client_id) {
6677 attached.remove(&id);
6678 }
6679 }
6680 // Drop the peer-messaging guard with the binding. The guard holds a
6681 // rate budget and a dedupe window keyed by agent *name*, so a
6682 // supervisor-respawned replacement taking the same name would
6683 // otherwise inherit the dead process's history — arriving
6684 // pre-throttled, or silently deduped against messages it never saw.
6685 let bound_for_guards = session.agent_id.lock().await.clone();
6686 if let Some(id) = bound_for_guards {
6687 self.peer_guards.lock().await.remove(&id);
6688 }
6689 // Drop any in-flight `agents.chat` sessions bound to this
6690 // client — either side disconnecting orphans the stream,
6691 // and a respawned agent's stray `agent.chat.event`
6692 // notifications must not race into a stale routing entry.
6693 // See `docs/proposals/agent-chat-surface.md`.
6694 let bound_agent = session.agent_id.lock().await.clone();
6695 let mut chats = self.chat_sessions.lock().await;
6696 chats.retain(|_, s| {
6697 if s.host_client_id == client_id {
6698 return false;
6699 }
6700 if let Some(agent_id) = &bound_agent {
6701 if &s.agent_id == agent_id {
6702 return false;
6703 }
6704 }
6705 true
6706 });
6707 // Drop the chat lock before the run sweep, which awaits the
6708 // grace window and re-locks `runs` — keeping locks disjoint
6709 // avoids holding `chat_sessions` across the sleep.
6710 drop(chats);
6711 // Same for in-process A2A chat collectors owned by this client.
6712 // Dropping the entry drops the sole sender, so the collecting task's
6713 // `recv()` returns `None` and it falls back immediately instead of
6714 // waiting out its 180s cap. Done after `drop(chats)` so we never
6715 // hold `chat_sessions` while locking `chat_collectors` (the
6716 // interceptor locks them collectors→sessions; staying disjoint here
6717 // avoids inverting that order).
6718 self.chat_collectors
6719 .lock()
6720 .await
6721 .retain(|_, c| c.host_client_id != client_id);
6722 // Agent run tracing (U4): drop this connection's live
6723 // run-trace subscriptions so its drain tasks end and the
6724 // registry doesn't accumulate stale `(run_id, client)` entries
6725 // across reconnects. Reconnect-durability is client-side: the
6726 // run stays subscribable; CarHost re-subscribes by `run_id` on
6727 // its new connection (R8). This is exempt from the chat
6728 // drain-and-synthesize-error path — a dropped trace subscriber
6729 // never marks the underlying run failed.
6730 self.drop_run_subscribers_for_client(client_id).await;
6731 // Browser-drawer subscriptions are per-connection too. Dropping
6732 // them also starts the control grace period on any view this
6733 // connection was DRIVING, so an agent is never parked forever
6734 // behind a controller whose app just quit (see
6735 // `browser_view::CONTROL_GRACE`).
6736 self.browser_views
6737 .drop_subscriptions_for_client(client_id)
6738 .await;
6739 // Only a host-client removal can change host connectivity. This
6740 // used to fan a redundant reverse call to every producer for every
6741 // agent/CLI disconnect too. The session is already out of
6742 // `sessions`, so a real host removal reads the post-removal truth.
6743 if session.is_host.load(std::sync::atomic::Ordering::Acquire) {
6744 let host_connected = self.any_host_connected().await;
6745 self.browser_views
6746 .broadcast_host_connected(host_connected)
6747 .await;
6748 }
6749 // And if this connection was a supervised agent PUBLISHING a
6750 // browser, that browser went with the process. Its views stay
6751 // registered reporting an empty browser, so the drawer sees the
6752 // change and a restarted process can replace them in place (see
6753 // `browser_relay::RelayProducer::note_disconnected`).
6754 self.browser_views
6755 .note_producer_disconnected(client_id)
6756 .await;
6757 // Coder event subscriptions are per-connection too; the session
6758 // itself keeps running (reconnect + `coder.subscribe { from_seq }`
6759 // replays what was missed).
6760 // Board watchers (`coder.watch`) and discussion streams are
6761 // per-connection too, and are dropped on the same boundary.
6762 crate::coder::rpc::drop_subscriptions_for_client(self, client_id).await;
6763 crate::coder::discuss::drop_subscriptions_for_client(self, client_id).await;
6764 // Agent run tracing (R5): any run this connection owns that
6765 // has no terminal record yet is swept to `Incomplete` after
6766 // a short grace window — long enough for an in-flight
6767 // `runs.complete` to land first so a healthy close is never
6768 // mislabeled.
6769 let _run_guard = session.run_lifecycle_guard.lock().await;
6770 self.sweep_runs_for_disconnect(session).await;
6771 }
6772 removed
6773 }
6774}
6775
6776#[cfg(test)]
6777mod durable_task_tests {
6778 use super::*;
6779
6780 #[tokio::test]
6781 async fn dropping_response_waiter_does_not_cancel_operation_or_release_its_guard() {
6782 let temp = tempfile::tempdir().unwrap();
6783 let state = Arc::new(ServerState::with_config(ServerStateConfig::new(
6784 temp.path().to_path_buf(),
6785 )));
6786 let overlap_guard = Arc::new(tokio::sync::Mutex::new(()));
6787 let operation_guard = overlap_guard.clone();
6788 let (started_tx, started_rx) = tokio::sync::oneshot::channel();
6789 let (release_tx, release_rx) = tokio::sync::oneshot::channel();
6790
6791 let response = state
6792 .spawn_durable_operation("auth.test", async move {
6793 let _held = operation_guard.lock().await;
6794 let _ = started_tx.send(());
6795 let _ = release_rx.await;
6796 Ok::<_, String>(serde_json::json!({ "ok": true }))
6797 })
6798 .await;
6799 started_rx.await.unwrap();
6800 drop(response);
6801
6802 assert!(
6803 overlap_guard.try_lock().is_err(),
6804 "dropping only the connection waiter must not release the operation guard"
6805 );
6806 release_tx.send(()).unwrap();
6807 let joined = tokio::time::timeout(
6808 std::time::Duration::from_secs(1),
6809 state.durable_tasks.lock().await.join_next(),
6810 )
6811 .await
6812 .expect("daemon-owned operation should finish")
6813 .expect("task should exist")
6814 .expect("task should join");
6815 assert_eq!(joined, "auth.test");
6816 assert!(
6817 overlap_guard.try_lock().is_ok(),
6818 "the guard releases only after the daemon-owned operation finishes"
6819 );
6820 }
6821}
6822
6823#[cfg(test)]
6824mod tool_timeout_tests {
6825 use super::*;
6826
6827 #[test]
6828 fn honors_action_timeout_over_default() {
6829 // The action's budget bounds the wait (+grace so the executor's own
6830 // deadline reaps first) — no 60s ceiling, no env read.
6831 assert_eq!(
6832 tool_callback_timeout(Some(180_000)),
6833 std::time::Duration::from_millis(180_000 + TOOL_TIMEOUT_GRACE_MS)
6834 );
6835 // A budget far above the old 60s ceiling is honored (the #259 bug),
6836 // and the wait always exceeds the budget (so the executor wins).
6837 assert!(tool_callback_timeout(Some(600_000)) > std::time::Duration::from_secs(600));
6838 assert!(tool_callback_timeout(Some(180_000)) >= std::time::Duration::from_millis(180_000));
6839 }
6840
6841 #[test]
6842 fn default_is_not_the_old_60s() {
6843 // With no action budget and no env override, the fallback is the
6844 // raised default — not the hardcoded 60s that reaped real tools.
6845 // (Asserted only when CAR_TOOL_TIMEOUT is unset, which it is in CI.)
6846 if std::env::var_os("CAR_TOOL_TIMEOUT").is_none() {
6847 assert_eq!(
6848 tool_callback_timeout(None),
6849 std::time::Duration::from_millis(DEFAULT_TOOL_TIMEOUT_MS)
6850 );
6851 assert!(
6852 DEFAULT_TOOL_TIMEOUT_MS > 60_000,
6853 "default must exceed the old 60s"
6854 );
6855 }
6856 }
6857}
6858
6859#[cfg(test)]
6860mod observer_mode_tests {
6861 use super::*;
6862
6863 fn journal_dir() -> PathBuf {
6864 let target = std::env::var_os("CARGO_TARGET_DIR")
6865 .map(std::path::PathBuf::from)
6866 .unwrap_or_else(|| {
6867 std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
6868 .join("..")
6869 .join("..")
6870 .join("target")
6871 });
6872 std::fs::create_dir_all(&target).ok();
6873 let target = std::fs::canonicalize(&target).unwrap_or(target);
6874 let tmp = tempfile::TempDir::new_in(&target).unwrap();
6875 let p = tmp.path().to_path_buf();
6876 std::mem::forget(tmp); // keep the dir alive for the test
6877 p
6878 }
6879
6880 #[test]
6881 fn supervisor_returns_observer_error_when_marker_set() {
6882 // Closes Parslee-ai/car-releases#44: the second car-server on
6883 // a host installs the observer marker after `with_paths`
6884 // returns AlreadyRunning. Subsequent `state.supervisor()`
6885 // calls must return a clear "observe-only" error mentioning
6886 // the manifest path — they must NOT retry user_default()
6887 // (which would re-acquire the lock and likely also fail).
6888 let state = ServerState::standalone(journal_dir());
6889 let fake_manifest = PathBuf::from("/tmp/fake-manifest-for-test.json");
6890 state
6891 .install_observer_manifest(fake_manifest.clone())
6892 .expect("install_observer_manifest succeeds on fresh state");
6893 assert_eq!(state.observer_manifest_path(), Some(&fake_manifest));
6894
6895 let err = state.supervisor().map(|_| ()).unwrap_err();
6896 assert!(
6897 err.contains("observe-only"),
6898 "error must mention observe-only mode: {err}"
6899 );
6900 assert!(
6901 err.contains("fake-manifest-for-test.json"),
6902 "error must surface the manifest path so operators know which daemon owns it: {err}"
6903 );
6904 }
6905
6906 #[test]
6907 fn install_observer_manifest_is_idempotent_per_path_collision() {
6908 let state = ServerState::standalone(journal_dir());
6909 let p = PathBuf::from("/tmp/manifest-a.json");
6910 let q = PathBuf::from("/tmp/manifest-b.json");
6911 state.install_observer_manifest(p.clone()).unwrap();
6912 // OnceLock::set returns the value back on collision.
6913 let err = state.install_observer_manifest(q.clone()).unwrap_err();
6914 assert_eq!(err, q);
6915 assert_eq!(state.observer_manifest_path(), Some(&p));
6916 }
6917
6918 #[test]
6919 fn supervisor_if_installed_does_not_lazy_init() {
6920 // The Heisenberg-subscribe guard: `host.subscribe`'s
6921 // identity path must use the non-acquiring read so a
6922 // purely observational client can't cause the daemon to
6923 // claim `<manifest>.lock` as a side effect of asking
6924 // about it. Fresh state has no supervisor installed.
6925 let state = ServerState::standalone(journal_dir());
6926 assert!(state.supervisor_if_installed().is_none());
6927 // observer_manifest_path should remain unset too — no
6928 // implicit init.
6929 assert!(state.observer_manifest_path().is_none());
6930 }
6931}
6932
6933#[cfg(test)]
6934mod substrate_binding_tests {
6935 use super::*;
6936 use car_engine::{McpSession, McpToolInfo};
6937 use car_ir::ActionProposal;
6938 use serde_json::json;
6939 use std::sync::Mutex as StdMutex;
6940
6941 fn journal_dir() -> PathBuf {
6942 let target = std::env::var_os("CARGO_TARGET_DIR")
6943 .map(std::path::PathBuf::from)
6944 .unwrap_or_else(|| {
6945 std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
6946 .join("..")
6947 .join("..")
6948 .join("target")
6949 });
6950 std::fs::create_dir_all(&target).ok();
6951 let target = std::fs::canonicalize(&target).unwrap_or(target);
6952 let tmp = tempfile::TempDir::new_in(&target).unwrap();
6953 let p = tmp.path().to_path_buf();
6954 std::mem::forget(tmp);
6955 p
6956 }
6957
6958 /// A minimal in-memory MCP session standing in for a connected `vm`
6959 /// connector. `read_text`/`write_text`/`run_command` behave like the
6960 /// `vm` bridge so we can assert a substrate-bound session's built-ins
6961 /// land here rather than on the WS client (which has no responder in a
6962 /// test and would hang/fail).
6963 struct FakeVmSession {
6964 name: String,
6965 files: Arc<StdMutex<std::collections::HashMap<String, String>>>,
6966 }
6967
6968 #[async_trait::async_trait]
6969 impl McpSession for FakeVmSession {
6970 async fn list_tools(&mut self) -> Result<Vec<McpToolInfo>, String> {
6971 Ok(vec![])
6972 }
6973 async fn call_tool(
6974 &mut self,
6975 name: &str,
6976 args: serde_json::Value,
6977 ) -> Result<serde_json::Value, String> {
6978 match name {
6979 "write_text" => {
6980 let p = args["path"].as_str().unwrap().to_string();
6981 let c = args["content"].as_str().unwrap().to_string();
6982 self.files.lock().unwrap().insert(p, c);
6983 Ok(json!("ok"))
6984 }
6985 "read_text" => {
6986 let p = args["path"].as_str().unwrap();
6987 let c = self
6988 .files
6989 .lock()
6990 .unwrap()
6991 .get(p)
6992 .cloned()
6993 .ok_or_else(|| "not found".to_string())?;
6994 Ok(json!(c))
6995 }
6996 "run_command" => {
6997 let command = args["command"].as_str().unwrap_or_default();
6998 if let Some(path) = command
6999 .split_once("[ -e '")
7000 .map(|(_, path)| path)
7001 .and_then(|path| path.split_once("' ]").map(|(path, _)| path))
7002 {
7003 let exists = self.files.lock().unwrap().contains_key(path);
7004 return Ok(json!({
7005 "stdout": "",
7006 "stderr": "",
7007 "exit_code": if exists { 0 } else { 1 }
7008 }));
7009 }
7010 Ok(json!({
7011 "stdout": "from-vm",
7012 "stderr": "",
7013 "exit_code": 0
7014 }))
7015 }
7016 other => Err(format!("unknown tool {other}")),
7017 }
7018 }
7019 fn name(&self) -> &str {
7020 &self.name
7021 }
7022 }
7023
7024 fn read_file_proposal(path: &str) -> ActionProposal {
7025 serde_json::from_value(json!({
7026 "source": "test",
7027 "actions": [{
7028 "id": "r0",
7029 "type": "tool_call",
7030 "tool": "read_file",
7031 "parameters": { "path": path },
7032 "dependencies": [],
7033 }],
7034 }))
7035 .expect("proposal deserializes")
7036 }
7037
7038 /// The outbound message sink is attached to every session runtime, so
7039 /// `messaging.send` is a real, executable tool rather than a schema the
7040 /// runtime would refuse. `with_message_sink` registers the sink and the
7041 /// schema together, so seeing the schema proves the sink is there.
7042 #[tokio::test]
7043 async fn session_runtime_has_the_messaging_send_tool() {
7044 let state = Arc::new(ServerState::standalone(journal_dir()));
7045 let session = state
7046 .create_session("c-messaging", Arc::new(WsChannel::test_stub()))
7047 .await
7048 .unwrap();
7049
7050 assert!(
7051 session
7052 .runtime
7053 .tools
7054 .read()
7055 .await
7056 .contains_key("messaging.send"),
7057 "every session must be able to execute messaging.send"
7058 );
7059 }
7060
7061 /// Default (no bind): the session runtime keeps a `LocalSubstrate`.
7062 /// This is the backward-compat invariant — every existing consumer
7063 /// (GUI, A2A-over-WS, connectors-as-tools, coder) sees no change.
7064 #[tokio::test]
7065 async fn default_session_substrate_is_local() {
7066 let state = Arc::new(ServerState::standalone(journal_dir()));
7067 let channel = Arc::new(WsChannel::test_stub());
7068 let session = state.create_session("c-default", channel).await.unwrap();
7069
7070 let sub = session.runtime.substrate().await;
7071 assert_eq!(
7072 sub.name(),
7073 "local",
7074 "an un-bound session must keep the default LocalSubstrate"
7075 );
7076 assert!(sub.is_local(), "default substrate must be the host");
7077 }
7078
7079 /// Binding fails cleanly when the named connector is not connected —
7080 /// no panic, a descriptive error, and the substrate stays Local.
7081 #[tokio::test]
7082 async fn bind_unknown_connector_errors_and_keeps_local() {
7083 let state = Arc::new(ServerState::standalone(journal_dir()));
7084 let channel = Arc::new(WsChannel::test_stub());
7085 let session = state.create_session("c-missing", channel).await.unwrap();
7086
7087 let err = state
7088 .bind_substrate_to_connector(&session, "nope")
7089 .await
7090 .unwrap_err();
7091 assert!(err.contains("not connected"), "got: {err}");
7092 assert_eq!(session.runtime.substrate().await.name(), "local");
7093 }
7094
7095 /// A connector-driven session that binds its substrate routes the bare
7096 /// commodity built-ins (`read_file`/`write_file`) to that connector's
7097 /// environment via the engine fall-through, NOT to the WS client. We
7098 /// prove it end-to-end: write+read through the runtime land on the fake
7099 /// VM session's file map.
7100 #[tokio::test]
7101 async fn bound_session_routes_builtins_to_substrate() {
7102 let state = Arc::new(ServerState::standalone(journal_dir()));
7103 let channel = Arc::new(WsChannel::test_stub());
7104 let session = state.create_session("c-vm", channel).await.unwrap();
7105
7106 // Register a connected "vm" connector session + its routes the way
7107 // the connector manager does on dial.
7108 let files = Arc::new(StdMutex::new(std::collections::HashMap::new()));
7109 let fake: Arc<Mutex<dyn McpSession>> = Arc::new(Mutex::new(FakeVmSession {
7110 name: "vm".into(),
7111 files: files.clone(),
7112 }));
7113 state.mcp_executor.add_session("vm", fake).await;
7114
7115 // Bind the session to the vm connector's substrate.
7116 let bound = state
7117 .bind_substrate_to_connector(&session, "vm")
7118 .await
7119 .expect("bind succeeds for a connected connector");
7120 assert_eq!(bound, "vm");
7121 assert_eq!(session.runtime.substrate().await.name(), "vm");
7122
7123 // write_file through the runtime → must hit the fake VM session,
7124 // not the WS client (which would hang/fail with no responder).
7125 let write = serde_json::from_value::<ActionProposal>(json!({
7126 "source": "test",
7127 "actions": [{
7128 "id": "w0",
7129 "type": "tool_call",
7130 "tool": "write_file",
7131 "parameters": { "path": "/vm/a.txt", "content": "vm-bytes" },
7132 "dependencies": [],
7133 }],
7134 }))
7135 .unwrap();
7136 let wres = session.runtime.execute(&write).await;
7137 assert!(
7138 wres.results[0].error.is_none(),
7139 "write_file via substrate failed: {:?}",
7140 wres.results[0].error
7141 );
7142 assert_eq!(
7143 files.lock().unwrap().get("/vm/a.txt").map(String::as_str),
7144 Some("vm-bytes"),
7145 "write_file must land on the bound VM substrate"
7146 );
7147
7148 // read_file back must return the VM's content.
7149 let rres = session
7150 .runtime
7151 .execute(&read_file_proposal("/vm/a.txt"))
7152 .await;
7153 assert!(
7154 rres.results[0].error.is_none(),
7155 "read_file via substrate failed: {:?}",
7156 rres.results[0].error
7157 );
7158 let out = rres.results[0].output.clone().unwrap_or(Value::Null);
7159 let content = out.get("content").and_then(|v| v.as_str()).unwrap_or("");
7160 // read_file output is line-numbered `cat -n` style (H1/F4-remainder,
7161 // audit 2026-07-06); the VM's single-line content becomes " 1\t<line>".
7162 assert_eq!(
7163 content, " 1\tvm-bytes",
7164 "read_file must come from the VM substrate"
7165 );
7166 }
7167
7168 /// The shadow executor: substrate-owned built-in names return the
7169 /// `"unknown tool"` prefix (so the engine falls through to the bound
7170 /// substrate), while any other tool name is delegated to the inner
7171 /// executor unchanged (connector `mcp_*` routes + host WS callbacks).
7172 #[tokio::test]
7173 async fn shadow_executor_only_shadows_builtin_names() {
7174 struct Inner;
7175 #[async_trait::async_trait]
7176 impl ToolExecutor for Inner {
7177 async fn execute(&self, tool: &str, _p: &Value) -> Result<Value, String> {
7178 Ok(json!({ "delegated": tool }))
7179 }
7180 }
7181 let shadow = SubstrateShadowExecutor::new(Arc::new(Inner));
7182
7183 for name in SUBSTRATE_OWNED_TOOLS {
7184 let err = shadow.execute(name, &json!({})).await.unwrap_err();
7185 assert!(
7186 err.starts_with("unknown tool"),
7187 "{name} must be shadowed so the engine falls through: {err}"
7188 );
7189 }
7190
7191 // A connector tool name and a pure tool pass through untouched.
7192 for passthrough in ["mcp_vm_run_command", "calculate", "browser.run"] {
7193 let out = shadow.execute(passthrough, &json!({})).await.unwrap();
7194 assert_eq!(out["delegated"], json!(passthrough));
7195 }
7196 }
7197}
7198
7199/// `.car/policies/*.toml` reaching the runtime at all — the whole point of
7200/// wiring `load_project_policies` to a real call site. Exercised against a
7201/// temp directory rather than through `create_session`, which would read the
7202/// developer's real `$HOME`.
7203#[cfg(test)]
7204mod project_policy_loading_tests {
7205 use super::*;
7206
7207 fn runtime() -> Runtime {
7208 Runtime::new()
7209 }
7210
7211 #[tokio::test]
7212 async fn a_missing_policies_directory_is_silent() {
7213 let dir = tempfile::tempdir().unwrap();
7214 // No `.car` at all, let alone `.car/policies`.
7215 apply_project_policies(&runtime(), &dir.path().join(".car"))
7216 .await
7217 .expect("a project with no rules must start normally");
7218 }
7219
7220 #[tokio::test]
7221 async fn an_empty_policies_directory_is_silent() {
7222 let dir = tempfile::tempdir().unwrap();
7223 let car = dir.path().join(".car");
7224 std::fs::create_dir_all(car.join("policies")).unwrap();
7225 apply_project_policies(&runtime(), &car).await.unwrap();
7226 }
7227
7228 #[tokio::test]
7229 async fn a_well_formed_rule_file_loads_and_takes_effect() {
7230 let dir = tempfile::tempdir().unwrap();
7231 let car = dir.path().join(".car");
7232 std::fs::create_dir_all(car.join("policies")).unwrap();
7233 // Exactly the shape `car_policy::rules`' module docs advertise: a
7234 // top-level list for `deny_tool`, a table array for the per-parameter
7235 // rules.
7236 std::fs::write(
7237 car.join("policies").join("messaging.toml"),
7238 "deny_tool = [\"messaging.send\"]\n\n\
7239 [[deny_tool_param]]\n\
7240 tool = \"messaging.send\"\n\
7241 param = \"channel\"\n\
7242 equals = \"slack\"\n",
7243 )
7244 .unwrap();
7245
7246 let rt = runtime();
7247 apply_project_policies(&rt, &car).await.unwrap();
7248
7249 // Registered on the runtime's engine, not merely parsed.
7250 let names: Vec<String> = rt
7251 .list_policies(None)
7252 .await
7253 .unwrap()
7254 .into_iter()
7255 .map(|(name, _)| name)
7256 .collect();
7257 assert!(
7258 names.len() >= 2,
7259 "both declarative rules must reach the policy engine, got {names:?}"
7260 );
7261 }
7262
7263 #[tokio::test]
7264 async fn a_malformed_rule_file_fails_loudly_and_names_the_file() {
7265 let dir = tempfile::tempdir().unwrap();
7266 let car = dir.path().join(".car");
7267 std::fs::create_dir_all(car.join("policies")).unwrap();
7268 std::fs::write(
7269 car.join("policies").join("broken.toml"),
7270 "[[deny_tool]\ntool = \"shell\"",
7271 )
7272 .unwrap();
7273
7274 let err = apply_project_policies(&runtime(), &car)
7275 .await
7276 .expect_err("a malformed policy file must NOT be swallowed");
7277 assert!(
7278 err.contains("broken.toml"),
7279 "the operator has to know which file to fix: {err}"
7280 );
7281 assert!(err.contains("refusing to start"), "{err}");
7282 }
7283}
7284
7285#[cfg(test)]
7286mod org_scope_wiring_tests {
7287 use super::*;
7288
7289 fn granter_hex_for(secret: &[u8], user: &str) -> String {
7290 let master = car_sync::StretchedMaster::from_issued_high_entropy(secret, user);
7291 let vk = car_sync::ed25519_verifying(&car_sync::derive_ed25519_identity(&master, user));
7292 vk.to_bytes().iter().map(|b| format!("{b:02x}")).collect()
7293 }
7294 fn granter_hex() -> String {
7295 granter_hex_for(b"granter-login", "acc_granter")
7296 }
7297
7298 #[test]
7299 fn unset_env_is_the_personal_only_path() {
7300 // The real-tenants-OFF invariant: no value → Ok(None) → base provider.
7301 assert!(ServerState::parse_org_scope_config(None).unwrap().is_none());
7302 assert!(ServerState::parse_org_scope_config(Some(String::new()))
7303 .unwrap()
7304 .is_none());
7305 }
7306
7307 #[test]
7308 fn wellformed_env_parses_org_and_granter() {
7309 let hex = granter_hex();
7310 let (org, granters) = ServerState::parse_org_scope_config(Some(format!("orgtest:{hex}")))
7311 .unwrap()
7312 .expect("well-formed opt-in parses");
7313 assert_eq!(org, "orgtest");
7314 assert_eq!(granters.len(), 1);
7315 let expected_master =
7316 car_sync::StretchedMaster::from_issued_high_entropy(b"granter-login", "acc_granter");
7317 let expected = car_sync::ed25519_verifying(&car_sync::derive_ed25519_identity(
7318 &expected_master,
7319 "acc_granter",
7320 ));
7321 assert_eq!(granters[0].to_bytes(), expected.to_bytes());
7322 }
7323
7324 #[test]
7325 fn multiple_granters_parse_in_order() {
7326 // The trusted set is plural: an org can designate several granters.
7327 let g1 = granter_hex_for(b"g1", "acc_g1");
7328 let g2 = granter_hex_for(b"g2", "acc_g2");
7329 let (org, granters) =
7330 ServerState::parse_org_scope_config(Some(format!("orgtest:{g1},{g2}")))
7331 .unwrap()
7332 .expect("well-formed multi-granter opt-in parses");
7333 assert_eq!(org, "orgtest");
7334 assert_eq!(granters.len(), 2);
7335 assert_ne!(granters[0].to_bytes(), granters[1].to_bytes());
7336 // Surrounding whitespace on a granter entry is tolerated.
7337 let spaced = ServerState::parse_org_scope_config(Some(format!("orgtest:{g1} , {g2}")))
7338 .unwrap()
7339 .unwrap();
7340 assert_eq!(spaced.1.len(), 2);
7341 }
7342
7343 #[test]
7344 fn one_bad_granter_in_the_list_fails_the_whole_value() {
7345 // A typo in ANY granter is a loud failure — never silently drop it and
7346 // proceed with a partial trusted set.
7347 let good = granter_hex();
7348 assert!(
7349 ServerState::parse_org_scope_config(Some(format!("orgtest:{good},not-hex"))).is_err(),
7350 "a malformed granter among valid ones must be a hard error"
7351 );
7352 // An org id with no granters at all is rejected.
7353 assert!(ServerState::parse_org_scope_config(Some("orgtest:".into())).is_err());
7354 assert!(ServerState::parse_org_scope_config(Some("orgtest: , ".into())).is_err());
7355 }
7356
7357 #[test]
7358 fn malformed_env_is_a_hard_error_never_silent() {
7359 // A typo must NOT silently fall through to a wrong/unauthenticated state.
7360 let hex = granter_hex();
7361 let bads = [
7362 "no-colon-here".to_string(), // missing ':'
7363 ":deadbeef".to_string(), // empty org
7364 "orgtest:not-hex".to_string(), // bad hex
7365 "orgtest:aabb".to_string(), // wrong length
7366 format!("bad org:{hex}"), // non-canonical org (space)
7367 format!("org/evil:{hex}"), // non-canonical org (delimiter)
7368 ];
7369 for bad in &bads {
7370 assert!(
7371 ServerState::parse_org_scope_config(Some(bad.clone())).is_err(),
7372 "malformed value {bad:?} must be a hard error"
7373 );
7374 }
7375 // NOTE: an all-zero (identity/small-order) key PARSES — from_bytes accepts
7376 // the encoding; verify_strict rejects it at verification, so such a granter
7377 // simply grants nothing (fail-closed), which is correct, not a parse error.
7378 }
7379
7380 #[test]
7381 fn scope_routing_sends_only_matching_org_to_the_org_subsystem() {
7382 use car_sync::Scope;
7383 let acme = Scope::Shared { org: "acme".into() };
7384 let globex = Scope::Shared {
7385 org: "globex".into(),
7386 };
7387
7388 // org-scope OFF (no holder): EVERYTHING routes to the personal subsystem —
7389 // the byte-identical invariant.
7390 assert_eq!(route_for_scope(&Scope::Personal, None), SyncRoute::User);
7391 assert_eq!(route_for_scope(&acme, None), SyncRoute::User);
7392
7393 // opted into "acme": only Scope::Shared{acme} routes to the org subsystem.
7394 assert_eq!(route_for_scope(&acme, Some("acme")), SyncRoute::Org);
7395 assert_eq!(
7396 route_for_scope(&Scope::Personal, Some("acme")),
7397 SyncRoute::User
7398 );
7399 assert_eq!(
7400 route_for_scope(&globex, Some("acme")),
7401 SyncRoute::User,
7402 "a different org must NOT go to acme's delivery subsystem"
7403 );
7404 }
7405}