car_server_core/session.rs
1//! Server-side session state — shared across all connections.
2
3use car_engine::{AdmissionGate, GateContext, GateOutcome, Runtime, ToolExecutor};
4use car_eventlog::EventLog;
5use car_ir::ToolFailure;
6use car_proto::{ToolCancelRequest, ToolExecuteRequest, ToolExecuteResponse};
7use futures::Sink;
8use serde::{Deserialize, Serialize};
9use serde_json::Value;
10use sha2::{Digest, Sha256};
11use std::collections::HashMap;
12use std::path::{Path, PathBuf};
13use std::pin::Pin;
14use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
15use std::sync::Arc;
16use tokio::sync::{mpsc, oneshot, Mutex};
17use tokio_tungstenite::tungstenite::{Error as WsError, Message};
18
19/// Type-erased WebSocket sink. The dispatch loop accepts either a
20/// `WebSocketStream<TcpStream>` (the legacy car-server TCP listener)
21/// or a `WebSocketStream<UnixStream>` (the daemon-as-default UDS
22/// listener) — both implement `Sink<Message, Error = WsError>` after
23/// the tungstenite handshake. Erasing the type here avoids cascading
24/// a generic parameter through every WsChannel / Session / ServerState
25/// touchpoint in the dispatcher.
26pub type WsSink = Pin<Box<dyn Sink<Message, Error = WsError> + Send + Unpin + 'static>>;
27
28/// Grace window applied on disconnect before a still-open run is marked
29/// `Incomplete` (agent run tracing, U1 — R5). Short on purpose: it only
30/// has to cover the gap between a healthy `runs.complete` being
31/// dispatched on a spawned task and its terminal record landing, not any
32/// real work. Long enough to absorb that scheduling jitter, short enough
33/// that a genuinely abandoned run is reported `Incomplete` promptly.
34pub const RUN_COMPLETE_GRACE: std::time::Duration = std::time::Duration::from_millis(250);
35
36/// Bounded in-process lease for a disconnected run whose producer negotiated
37/// `runs.resume.v1`. Ten seconds leaves practical headroom for a loaded Mac's
38/// Python controller to establish a fresh local socket, authenticate, and
39/// negotiate capabilities without turning this into restart recovery. Runs
40/// without the capability keep the original 250 ms behavior; an unclaimed
41/// resumable run still becomes `Incomplete`.
42pub const RUN_RESUME_LEASE: std::time::Duration = std::time::Duration::from_secs(10);
43
44/// Maximum time an async run-lifecycle mutation waits for the journal writer's
45/// durability acknowledgement. Five seconds matches the server's existing
46/// acknowledgement/grace budget and remains well inside the default mutating
47/// handler deadline. If an operator override cancels earlier, the async append
48/// still marks the exact row pending before its first yield.
49const CRITICAL_JOURNAL_ACKNOWLEDGEMENT_TIMEOUT: std::time::Duration =
50 std::time::Duration::from_secs(5);
51
52/// Startup uses the same durability budget as live lifecycle mutations. A
53/// stalled journal must fail the constructor before the daemon listens rather
54/// than occupying the startup thread forever.
55const STARTUP_RECONCILIATION_ACKNOWLEDGEMENT_TIMEOUT: std::time::Duration =
56 CRITICAL_JOURNAL_ACKNOWLEDGEMENT_TIMEOUT;
57
58/// Per-run observation bound for disconnect cleanup: the ten-second resume
59/// lease, two bounded five-second journal acknowledgement attempts (the second
60/// is an exact retry after durability-unknown), and scheduler headroom. Cleanup
61/// never clears the run or binding unless an acknowledgement arrives.
62pub const RUN_DISCONNECT_CLEANUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(22);
63
64const RUN_SUBSCRIBE_SUMMARY_STATE_RETRY_LIMIT: usize = 3;
65
66/// Stable admission reason returned after a callback terminally halts its
67/// WebSocket session. The halt is intentionally connection-local and dies on
68/// reconnect; durable run outcomes remain the responsibility of `runs.complete`.
69pub const SESSION_HALTED_REASON: &str =
70 "session halted by a terminal tool failure; reconnect or ask a host to call session.clear_halt";
71
72struct SessionHaltAdmissionGate {
73 halted: Arc<AtomicBool>,
74}
75
76#[async_trait::async_trait]
77impl AdmissionGate for SessionHaltAdmissionGate {
78 fn name(&self) -> &str {
79 "session_halt"
80 }
81
82 async fn check(
83 &self,
84 _proposal: &car_ir::ActionProposal,
85 _ctx: &GateContext<'_>,
86 ) -> GateOutcome {
87 if self.halted.load(Ordering::Acquire) {
88 GateOutcome::reject_all(SESSION_HALTED_REASON)
89 } else {
90 GateOutcome::Allow
91 }
92 }
93}
94
95/// Server-side credentials for continuing an A2A-owned A2UI surface.
96///
97/// This intentionally lives outside `car_a2ui::A2uiSurfaceOwner` so
98/// renderers can inspect surface ownership without receiving secrets.
99#[derive(Debug, Clone, Serialize, Deserialize)]
100#[serde(rename_all = "camelCase", tag = "type")]
101pub enum A2aRouteAuth {
102 None,
103 Bearer { token: String },
104 Header { name: String, value: String },
105}
106
107#[cfg(test)]
108mod run_reservation_liveness_tests {
109 use super::*;
110 use crate::run_store::RunStoreLookupGate;
111 use std::time::Duration;
112
113 fn pristine_run(run_id: &str, client_id: &str) -> RunMeta {
114 RunMeta {
115 run_id: run_id.to_string(),
116 agent_id: "agent-release".to_string(),
117 client_id: client_id.to_string(),
118 active_client_id: client_id.to_string(),
119 resume_predecessor_client_id: None,
120 resume_lease: None,
121 intent: "release without blocking the registry".to_string(),
122 outcome_description: None,
123 started_at: chrono::Utc::now(),
124 termination: None,
125 ended_at: None,
126 turns: Vec::new(),
127 start_committed: false,
128 pending_terminal: None,
129 cancellation_pending: None,
130 cancellation_receipt: None,
131 trace_corruption: None,
132 durability_generation: 0,
133 }
134 }
135
136 #[tokio::test]
137 async fn blocked_release_lookup_does_not_hold_global_runs_lock() {
138 let tmp = tempfile::TempDir::new().unwrap();
139 let gate = RunStoreLookupGate::default();
140 let state = Arc::new(ServerState::with_config(
141 ServerStateConfig::new(tmp.path().join("journals"))
142 .with_run_store_lookup_gate(gate.clone()),
143 ));
144 let session = state
145 .create_session("release-client", Arc::new(WsChannel::test_stub()))
146 .await
147 .unwrap();
148 let expected = pristine_run("release-candidate", &session.client_id);
149 assert!(matches!(
150 state.reserve_run(expected.clone()).await,
151 Ok(RunReservation::New)
152 ));
153
154 gate.block_next();
155 let release_state = state.clone();
156 let release_session = session.clone();
157 let release_expected = expected.clone();
158 let release = tokio::spawn(async move {
159 release_state
160 .release_unpersisted_run_reservation(&release_session, &release_expected)
161 .await
162 });
163 let wait_gate = gate.clone();
164 assert!(
165 tokio::task::spawn_blocking(move || {
166 wait_gate.wait_until_entered(Duration::from_secs(3))
167 })
168 .await
169 .unwrap(),
170 "release must reach the blocked durable lookup"
171 );
172
173 let registry = tokio::time::timeout(Duration::from_secs(1), state.runs.lock())
174 .await
175 .expect("blocked release lookup must not hold the global runs lock");
176 drop(registry);
177 gate.release();
178 assert_eq!(release.await.unwrap(), Ok(()));
179 assert!(state.run_meta(&expected.run_id).await.is_none());
180 }
181}
182
183/// Shared write half of the WebSocket, plus pending callback channels.
184/// `write` is type-erased via [`WsSink`] so the dispatcher can run
185/// against any transport-specific WebSocketStream (TCP or UDS today;
186/// axum-bridged in future) without templatizing every consumer.
187pub struct WsChannel {
188 pub write: Mutex<WsSink>,
189 /// Pending tool execution callbacks: request_id → oneshot sender
190 pub pending: Mutex<HashMap<String, oneshot::Sender<ToolExecuteResponse>>>,
191 /// Body-free correlation for active callbacks. A session owns at most one
192 /// active run, so every entry belongs to that run while its lifecycle
193 /// guard is held by proposal execution.
194 pub active_actions: Mutex<HashMap<String, String>>,
195 pub next_id: AtomicU64,
196}
197
198/// A WS connection is a host event sink (#418): `HostState` broadcasts
199/// `host.event` frames to subscribers through this trait, so it never has to
200/// know about the concrete `WsChannel` / tungstenite types.
201#[async_trait::async_trait]
202impl crate::host::EventSubscriber for WsChannel {
203 async fn send_text(&self, json: String) {
204 use futures::SinkExt;
205 // Bound both lock acquisition and the socket write. Host events are
206 // emitted from reducer transitions; a suspended client must not hold
207 // that reducer's announcement lock (and browser controls behind it)
208 // indefinitely through TCP backpressure.
209 let _ = tokio::time::timeout(std::time::Duration::from_secs(10), async {
210 self.write
211 .lock()
212 .await
213 .send(Message::Text(json.into()))
214 .await
215 })
216 .await;
217 }
218}
219
220impl WsChannel {
221 pub fn next_request_id(&self) -> String {
222 let id = self.next_id.fetch_add(1, Ordering::SeqCst);
223 format!("cb-{}", id)
224 }
225
226 /// Test-only stub that returns a WsChannel whose write sink drains
227 /// to nowhere. Used by `host.rs` tests that need a real
228 /// `Arc<WsChannel>` in the subscribers map (to exercise membership
229 /// checks like the cross-session resolve fan-out) without
230 /// constructing a tungstenite handshake. Never writes are
231 /// performed against this stub; if anything tries, the drain sink
232 /// quietly absorbs.
233 #[cfg(test)]
234 pub fn test_stub() -> Self {
235 use futures::sink::SinkExt;
236 let sink: WsSink = Box::pin(
237 futures::sink::drain()
238 .sink_map_err(|_| tokio_tungstenite::tungstenite::Error::ConnectionClosed),
239 );
240 WsChannel {
241 write: Mutex::new(sink),
242 pending: Mutex::new(HashMap::new()),
243 active_actions: Mutex::new(HashMap::new()),
244 next_id: AtomicU64::new(0),
245 }
246 }
247
248 /// Test-only channel whose write sink is *observable*: every
249 /// `Message::Text` payload written to it is appended to the returned
250 /// shared Vec. `test_stub` drains to nowhere, which makes it useless
251 /// for asserting that a specific frame — e.g. the `tools.cancel`
252 /// notification — actually reached the wire.
253 ///
254 /// The Vec is behind a `std::sync::Mutex` (not tokio's) because the
255 /// sink closure is synchronous; the lock is taken and released inside
256 /// the closure before the returned future is handed back, so a test
257 /// holding the lock to assert can never deadlock a concurrent write.
258 #[cfg(test)]
259 pub fn test_capture() -> (Self, std::sync::Arc<std::sync::Mutex<Vec<String>>>) {
260 let frames = std::sync::Arc::new(std::sync::Mutex::new(Vec::<String>::new()));
261 let sink: WsSink = Box::pin(futures::sink::unfold(
262 frames.clone(),
263 |frames: std::sync::Arc<std::sync::Mutex<Vec<String>>>, msg: Message| {
264 if let Message::Text(text) = &msg {
265 // Scoped so the lock is released before we yield the
266 // (already-ready) future back to the sink machinery.
267 if let Ok(mut held) = frames.lock() {
268 held.push(text.as_str().to_string());
269 }
270 }
271 futures::future::ready(Ok::<_, WsError>(frames))
272 },
273 ));
274 (
275 WsChannel {
276 write: Mutex::new(sink),
277 pending: Mutex::new(HashMap::new()),
278 active_actions: Mutex::new(HashMap::new()),
279 next_id: AtomicU64::new(0),
280 },
281 frames,
282 )
283 }
284}
285
286/// In-flight `agents.chat` session bookkeeping. Created when a host
287/// client calls `agents.chat`, removed when the agent emits a terminal
288/// `agent.chat.event` (`kind: "done"` or `"error"`), when either side
289/// disconnects, or when the host cancels via `agents.chat.cancel`.
290///
291/// The session_id is host-supplied (or server-generated when omitted)
292/// and threads through every `agent.chat.event` notification so the
293/// server can route streamed deltas back to the originating host
294/// without needing per-session subscriptions. See
295/// `docs/proposals/agent-chat-surface.md` for the wire contract.
296#[derive(Debug, Clone)]
297pub struct ChatSession {
298 /// Agent that owns this chat — populated from
299 /// `attached_agents` at `agents.chat` dispatch time.
300 pub agent_id: String,
301 /// Client id of the host that issued `agents.chat`. The server
302 /// forwards `agent.chat.event` notifications back to *this* host
303 /// only, so two CarHost windows chatting with the same agent are
304 /// independent streams.
305 pub host_client_id: String,
306 /// Unix-seconds creation time — used by the future stale-session
307 /// sweeper to drop sessions whose agent died without emitting a
308 /// terminal event.
309 pub created_at: u64,
310 /// In-process cancellation flag for daemon-owned chat runs. External
311 /// attached agents receive `agent.chat.cancel` instead.
312 pub local_cancel: Option<Arc<AtomicBool>>,
313}
314
315/// Host-visible standing goal for a chat session. `goal.set` stores this keyed
316/// by `session_id`; `agents.chat` reuses it for that session when the turn does
317/// not pass an inline `goal`.
318#[derive(Debug, Clone, Serialize, Deserialize)]
319pub struct ChatGoalState {
320 pub session_id: String,
321 pub check: String,
322 pub max_iterations: u32,
323 pub status: String,
324 pub last_iteration: Option<u32>,
325 pub last_met: Option<bool>,
326 pub last_grounded: Option<bool>,
327 pub last_reason: Option<String>,
328 pub terminal_kind: Option<String>,
329 pub terminal_message: Option<String>,
330 pub updated_at: u64,
331}
332
333fn chat_goals_path_from_journal_dir(journal_dir: &Path) -> PathBuf {
334 let car_dir = journal_dir
335 .parent()
336 .map(PathBuf::from)
337 .unwrap_or_else(|| PathBuf::from("."));
338 car_dir.join("chat-goals.json")
339}
340
341fn load_chat_goals_from_disk(journal_dir: &Path) -> HashMap<String, ChatGoalState> {
342 let path = chat_goals_path_from_journal_dir(journal_dir);
343 let Ok(text) = std::fs::read_to_string(&path) else {
344 return HashMap::new();
345 };
346 match serde_json::from_str::<HashMap<String, ChatGoalState>>(&text) {
347 Ok(mut goals) => {
348 for goal in goals.values_mut() {
349 if goal.status == "running" {
350 goal.status = "active".to_string();
351 }
352 }
353 goals
354 }
355 Err(e) => {
356 tracing::warn!(
357 path = %path.display(),
358 error = %e,
359 "chat-goals store was unreadable; starting with an empty in-memory goal registry"
360 );
361 HashMap::new()
362 }
363 }
364}
365
366fn write_chat_goals_atomic(
367 path: &std::path::Path,
368 goals: &HashMap<String, ChatGoalState>,
369) -> std::io::Result<()> {
370 use std::sync::atomic::{AtomicU64, Ordering};
371 static SEQ: AtomicU64 = AtomicU64::new(0);
372 if let Some(parent) = path.parent() {
373 std::fs::create_dir_all(parent)?;
374 }
375 let json = serde_json::to_vec_pretty(goals)?;
376 let seq = SEQ.fetch_add(1, Ordering::Relaxed);
377 let mut tmp_os = path.as_os_str().to_owned();
378 tmp_os.push(format!(".tmp.{}.{}", std::process::id(), seq));
379 let tmp = PathBuf::from(tmp_os);
380 std::fs::write(&tmp, json)?;
381 std::fs::rename(&tmp, path)
382}
383
384/// One normalized chunk of an `agent.chat` stream, delivered to an in-process
385/// [`chat_collectors`](ServerState::chat_collectors) sink (the A2A bridge). A
386/// `token` chunk carries a `delta`; a terminal chunk has `kind` `"done"`
387/// (success), `"error"` (with `error` text), or `"auth_required"` (with
388/// `message` text — the turn was refused because of the Parslee account, and
389/// nothing follows it on the wire).
390#[derive(Debug, Clone)]
391pub struct ChatStreamChunk {
392 pub kind: String,
393 pub delta: Option<String>,
394 pub error: Option<String>,
395 /// The `message` field of an `auth_required` terminal frame. Carried
396 /// separately from `error` because the two are different things to a
397 /// caller: one reports a failure, the other names a step a person can
398 /// take.
399 pub message: Option<String>,
400}
401
402/// An in-process `agent.chat` collector entry: the sink that
403/// [`try_forward_agent_chat_event`](crate::handler::try_forward_agent_chat_event)
404/// feeds, plus the host client whose disconnect should abandon it. Holding the
405/// sole sender here means dropping the entry closes the stream, so the
406/// collecting task wakes immediately instead of waiting out its timeout.
407pub struct ChatCollector {
408 pub tx: tokio::sync::mpsc::UnboundedSender<ChatStreamChunk>,
409 pub host_client_id: String,
410}
411
412#[derive(Debug, Clone)]
413#[doc(hidden)]
414pub struct RunResumeLease {
415 pub disconnected_client_id: String,
416 pub expires_at: tokio::time::Instant,
417}
418
419/// Deterministic test seam at the authenticated completion boundary. It is
420/// inert unless explicitly armed through [`ServerStateConfig`].
421#[derive(Debug, Clone, Default)]
422#[doc(hidden)]
423pub struct RunCompletionFenceGate {
424 armed: Arc<std::sync::atomic::AtomicBool>,
425 entered: Arc<tokio::sync::Notify>,
426 release: Arc<tokio::sync::Notify>,
427}
428
429impl RunCompletionFenceGate {
430 pub fn block_next(&self) {
431 self.armed.store(true, std::sync::atomic::Ordering::Release);
432 }
433
434 pub async fn wait_until_entered(&self, timeout: std::time::Duration) -> bool {
435 tokio::time::timeout(timeout, self.entered.notified())
436 .await
437 .is_ok()
438 }
439
440 pub fn release(&self) {
441 self.armed
442 .store(false, std::sync::atomic::Ordering::Release);
443 self.release.notify_one();
444 }
445
446 async fn wait_if_armed(&self) {
447 if !self.armed.load(std::sync::atomic::Ordering::Acquire) {
448 return;
449 }
450 self.entered.notify_one();
451 if self.armed.load(std::sync::atomic::Ordering::Acquire) {
452 self.release.notified().await;
453 }
454 }
455}
456
457/// Daemon-side record of a single agent run (agent run tracing, U1).
458///
459/// Keyed by `run_id` in the process-wide [`ServerState::runs`] registry
460/// so the record survives the WS connection that produced it — needed
461/// both for the disconnect grace window (R5) and so the disk store (U3)
462/// can flush a run even after its session is gone. U1 keeps this purely
463/// in memory; persistence is U3.
464#[derive(Debug, Clone)]
465pub struct RunMeta {
466 pub run_id: String,
467 /// Owning agent. Set at `runs.start` from the resolved id; used by
468 /// the read RPCs' ownership check (U5/KTD10) and the disk key (U3).
469 pub agent_id: String,
470 /// Immutable WS `client_id` that durably called `runs.start`. This remains
471 /// the provenance identity written to `RunStarted`, proposal receipts, and
472 /// `RunEnded`, even if an authenticated replacement socket resumes the
473 /// in-memory run.
474 pub client_id: String,
475 /// The one live socket currently authorized to continue the run. Equal to
476 /// `client_id` for a fresh run; changed only by authenticated orphan
477 /// reclaim after the previous owner has disappeared from `sessions`.
478 pub active_client_id: String,
479 /// Stable predecessor returned by an idempotent `runs.resume` retry on the
480 /// winning replacement socket.
481 pub resume_predecessor_client_id: Option<String>,
482 /// Ephemeral, in-process authority window installed atomically when the
483 /// active socket leaves the session registry. It is intentionally absent
484 /// from RunStore/journals and therefore never survives daemon restart.
485 #[doc(hidden)]
486 pub resume_lease: Option<RunResumeLease>,
487 pub intent: String,
488 pub outcome_description: Option<String>,
489 pub started_at: chrono::DateTime<chrono::Utc>,
490 /// `None` while the run is in progress; `Some` once a terminal
491 /// record (a reported outcome or an `Incomplete` marker) is
492 /// written. The presence of this field is the "is this run still
493 /// open?" signal the grace window checks.
494 pub termination: Option<car_proto::RunTermination>,
495 /// When the terminal record was written, if any.
496 pub ended_at: Option<chrono::DateTime<chrono::Utc>>,
497 /// The ordered per-turn trace recorded for this run (agent run
498 /// tracing, U2). Each entry is a [`car_proto::RunRecord::Turn`]
499 /// produced by the recorder from a submitted proposal + its
500 /// `ActionResult`s. U1 leaves this empty; the U2 recorder appends to
501 /// it via [`ServerState::record_run_turns`]. U3 flushes this buffer
502 /// to disk and U4 broadcasts it — both read it through
503 /// [`ServerState::run_turns`]. The `turn.index` field is monotonic
504 /// across the run's proposals, so this Vec is the clean ordered
505 /// stream those units consume.
506 pub turns: Vec<car_proto::RunRecord>,
507 /// True only after the exact RunStarted row reached flush+fsync and the
508 /// authenticated run_started journal event received the same durability
509 /// acknowledgement. A reserved-but-uncommitted run blocks all proposal
510 /// writes and can only be resumed by its owning client.
511 pub start_committed: bool,
512 /// Stable terminal transaction preimage retained across a failed
513 /// RunStore or journal durability boundary. Retries must present the same
514 /// RunTermination and finish this exact RunEnded; a conflicting terminal
515 /// is rejected.
516 pub pending_terminal: Option<car_proto::RunEnded>,
517 /// Durable cancellation request. Presence quarantines new proposals and
518 /// `runs.complete` until CAR either confirms a Cancelled terminal or an
519 /// operator independently resolves the unconfirmed stop.
520 pub cancellation_pending: Option<car_proto::RunCancellationRequested>,
521 /// Stable response for same-key idempotent retries.
522 pub cancellation_receipt: Option<car_proto::RunCancelResponse>,
523 /// Stable fail-closed error once the durable JSONL is known corrupt.
524 /// A quarantined run accepts no more proposals/turns, cannot complete,
525 /// and cannot install or retain live subscribers.
526 pub trace_corruption: Option<String>,
527 /// Monotonic in-memory commit generation for durability work which drops
528 /// the global run-registry lock while filesystem I/O executes.
529 #[doc(hidden)]
530 pub durability_generation: u64,
531}
532
533#[derive(Debug, Clone)]
534pub enum RunReservation {
535 New,
536 Existing(RunMeta),
537}
538
539/// In-memory ownership result for the authenticated WebSocket resume path.
540/// `active_client_id` is the fenced socket allowed to use the original
541/// durable run provenance now.
542#[derive(Debug, Clone, PartialEq, Eq)]
543pub(crate) struct RunResumeBinding {
544 pub run_id: String,
545 pub agent_id: String,
546 pub active_client_id: String,
547 pub resumed_from_client_id: String,
548}
549
550/// Canonical terminal identity shared by the durable `RunEnded` row, terminal
551/// journal event, and `runs.complete` response. The input is exactly the RFC
552/// 8785/JCS serialization of `RunTermination`; it deliberately does not include
553/// timestamps, run ids, client ids, or Daily artifact bytes.
554pub(crate) fn run_completion_digest(
555 termination: &car_proto::RunTermination,
556) -> Result<String, String> {
557 let canonical = car_inference::catalog_identity::canonical_json(termination)?;
558 Ok(format!("{:x}", Sha256::digest(canonical.as_bytes())))
559}
560
561fn run_trace_corruption_message(run_id: &str, detail: impl std::fmt::Display) -> String {
562 format!(
563 "{} run `{run_id}`: {detail}",
564 car_proto::RUN_TRACE_CORRUPTION_MESSAGE_PREFIX
565 )
566}
567
568fn same_run_termination(
569 left: &car_proto::RunTermination,
570 right: &car_proto::RunTermination,
571) -> bool {
572 match (run_completion_digest(left), run_completion_digest(right)) {
573 (Ok(left), Ok(right)) => left == right,
574 _ => false,
575 }
576}
577
578fn same_run_ended(left: &car_proto::RunEnded, right: &car_proto::RunEnded) -> bool {
579 left.run_id == right.run_id
580 && left.client_id == right.client_id
581 && left.agent_id == right.agent_id
582 && left.completion_digest == right.completion_digest
583 && left.ended_at == right.ended_at
584 && same_run_termination(&left.termination, &right.termination)
585}
586
587struct StartupJournalCache {
588 journal_dir: PathBuf,
589 failures: Option<car_eventlog::JournalFailureInjector>,
590 logs: HashMap<String, EventLog>,
591}
592
593impl StartupJournalCache {
594 fn new(journal_dir: &Path, failures: Option<&car_eventlog::JournalFailureInjector>) -> Self {
595 Self {
596 journal_dir: journal_dir.to_path_buf(),
597 failures: failures.cloned(),
598 logs: HashMap::new(),
599 }
600 }
601
602 fn get(&mut self, client_id: &str) -> Result<&mut EventLog, String> {
603 if !self.logs.contains_key(client_id) {
604 let path = self.journal_dir.join(format!("{client_id}.jsonl"));
605 let log = match (path.exists(), self.failures.as_ref()) {
606 (true, Some(failures)) => {
607 EventLog::load_with_journal_failure_injector(&path, failures.clone())
608 .map_err(|error| error.to_string())
609 }
610 (true, None) => EventLog::load(&path).map_err(|error| error.to_string()),
611 (false, Some(failures)) => Ok(EventLog::with_journal_failure_injector(
612 path,
613 failures.clone(),
614 )),
615 (false, None) => Ok(EventLog::with_journal(path)),
616 }?;
617 self.logs.insert(client_id.to_string(), log);
618 }
619 self.logs
620 .get_mut(client_id)
621 .ok_or_else(|| "startup journal cache insertion failed".to_string())
622 }
623}
624
625fn append_recovered_cancellation_result_journal(
626 journal_dir: PathBuf,
627 failures: Option<car_eventlog::JournalFailureInjector>,
628 client_id: String,
629 result: car_proto::RunCancelResponse,
630) -> Result<bool, String> {
631 let mut journals = StartupJournalCache::new(&journal_dir, failures.as_ref());
632 let log = journals.get(&client_id)?;
633 log.bind_run(&result.run_id, &client_id)?;
634 let Value::Object(data) = serde_json::to_value(&result).map_err(|error| error.to_string())?
635 else {
636 return Err("recovered cancellation result did not serialize as an object".into());
637 };
638 let data: HashMap<String, Value> = data.into_iter().collect();
639 let already_durable = log.events().iter().any(|event| {
640 event.kind == car_eventlog::EventKind::RunCancellationResult
641 && event.run_id.as_deref() == Some(result.run_id.as_str())
642 && event.client_id.as_deref() == Some(client_id.as_str())
643 && event.action_id.as_deref() == result.action_id.as_deref()
644 && event.proposal_id.is_none()
645 && event.data == data
646 });
647 log.append_critical_bounded(
648 car_eventlog::EventKind::RunCancellationResult,
649 result.action_id.as_deref(),
650 None,
651 data,
652 CRITICAL_JOURNAL_ACKNOWLEDGEMENT_TIMEOUT,
653 )
654 .map_err(|error| format!("recovered cancellation journal append failed: {error}"))?;
655 Ok(!already_durable)
656}
657
658fn append_recovered_terminal_cancellation_journal(
659 journal_dir: PathBuf,
660 failures: Option<car_eventlog::JournalFailureInjector>,
661 client_id: String,
662 requested: car_proto::RunCancellationRequested,
663 ended: car_proto::RunEnded,
664 result: car_proto::RunCancelResponse,
665) -> Result<(), String> {
666 let mut journals = StartupJournalCache::new(&journal_dir, failures.as_ref());
667 let log = journals.get(&client_id)?;
668 log.bind_run(&result.run_id, &client_id)?;
669
670 let Value::Object(requested_data) =
671 serde_json::to_value(&requested).map_err(|error| error.to_string())?
672 else {
673 return Err("recovered cancellation request did not serialize as an object".into());
674 };
675 log.append_critical_bounded(
676 car_eventlog::EventKind::RunCancellationRequested,
677 requested.action_id.as_deref(),
678 None,
679 requested_data.into_iter().collect(),
680 CRITICAL_JOURNAL_ACKNOWLEDGEMENT_TIMEOUT,
681 )
682 .map_err(|error| format!("recovered cancellation request journal append failed: {error}"))?;
683
684 let completion_digest = ended
685 .completion_digest
686 .as_deref()
687 .ok_or_else(|| "recovered terminal is missing completion_digest".to_string())?;
688 let completed_data = HashMap::from([
689 ("termination_kind".to_string(), Value::from("incomplete")),
690 (
691 "completion_digest".to_string(),
692 Value::from(completion_digest),
693 ),
694 (
695 "termination".to_string(),
696 serde_json::to_value(&ended.termination).map_err(|error| error.to_string())?,
697 ),
698 ]);
699 log.append_critical_bounded(
700 car_eventlog::EventKind::RunCompleted,
701 None,
702 None,
703 completed_data,
704 CRITICAL_JOURNAL_ACKNOWLEDGEMENT_TIMEOUT,
705 )
706 .map_err(|error| format!("recovered terminal journal append failed: {error}"))?;
707
708 let Value::Object(result_data) =
709 serde_json::to_value(&result).map_err(|error| error.to_string())?
710 else {
711 return Err("recovered cancellation result did not serialize as an object".into());
712 };
713 log.append_critical_bounded(
714 car_eventlog::EventKind::RunCancellationResult,
715 result.action_id.as_deref(),
716 None,
717 result_data.into_iter().collect(),
718 CRITICAL_JOURNAL_ACKNOWLEDGEMENT_TIMEOUT,
719 )
720 .map_err(|error| format!("recovered cancellation result journal append failed: {error}"))?;
721 log.clear_run_binding(&result.run_id, &client_id)
722}
723
724fn reconcile_durable_run_journals(
725 journals: &mut StartupJournalCache,
726 store: &crate::run_store::RunStore,
727 acknowledgement_timeout: std::time::Duration,
728) -> Result<(), String> {
729 type DurableRunBoundaries = (
730 car_proto::RunStarted,
731 Option<car_proto::RunEnded>,
732 Option<car_proto::RunCancellationRequested>,
733 Option<car_proto::RunCancelResponse>,
734 );
735 let mut by_client: HashMap<String, Vec<DurableRunBoundaries>> = HashMap::new();
736 store.visit_run_boundaries(|started, ended, requested, result| {
737 let Some(client_id) = started.client_id.as_deref() else {
738 return;
739 };
740 if client_id.is_empty()
741 || client_id.contains('/')
742 || client_id.contains('\\')
743 || client_id == "."
744 || client_id == ".."
745 {
746 tracing::error!(run_id = %started.run_id, "run-journal outbox has unsafe client_id");
747 return;
748 }
749 by_client
750 .entry(client_id.to_string())
751 .or_default()
752 .push((started, ended, requested, result));
753 });
754
755 for (client_id, mut runs) in by_client {
756 runs.sort_by_key(|(started, _, _, _)| started.started_at);
757 let log = match journals.get(&client_id) {
758 Ok(log) => log,
759 Err(error) => {
760 tracing::error!(%client_id, %error, "cannot load run journal for outbox reconciliation");
761 continue;
762 }
763 };
764 for (started, ended, requested, result) in runs {
765 if let Err(error) = log.bind_run(&started.run_id, &client_id) {
766 tracing::error!(run_id = %started.run_id, %error, "cannot bind run journal during outbox reconciliation");
767 continue;
768 }
769 let mut start_data = HashMap::from([
770 (
771 "agent_id".to_string(),
772 Value::from(started.agent_id.clone()),
773 ),
774 ("intent".to_string(), Value::from(started.intent.clone())),
775 (
776 "started_at".to_string(),
777 serde_json::to_value(started.started_at).unwrap_or(Value::Null),
778 ),
779 ]);
780 if let Some(description) = &started.outcome_description {
781 start_data.insert(
782 "outcome_description".to_string(),
783 Value::from(description.clone()),
784 );
785 }
786 log.append_critical_bounded(
787 car_eventlog::EventKind::RunStarted,
788 None,
789 None,
790 start_data,
791 acknowledgement_timeout,
792 )
793 .map_err(|error| {
794 format!(
795 "startup reconciliation failed for run {} run_started: {error}",
796 started.run_id
797 )
798 })?;
799
800 if let Some(requested) = requested {
801 let Value::Object(data) = serde_json::to_value(&requested).unwrap_or(Value::Null)
802 else {
803 continue;
804 };
805 log.append_critical_bounded(
806 car_eventlog::EventKind::RunCancellationRequested,
807 requested.action_id.as_deref(),
808 None,
809 data.into_iter().collect(),
810 acknowledgement_timeout,
811 )
812 .map_err(|error| {
813 format!(
814 "startup reconciliation failed for run {} cancellation request: {error}",
815 started.run_id
816 )
817 })?;
818 }
819
820 if ended.is_none() {
821 if let Some(result) = result {
822 let Value::Object(data) = serde_json::to_value(&result).unwrap_or(Value::Null)
823 else {
824 continue;
825 };
826 log.append_critical_bounded(
827 car_eventlog::EventKind::RunCancellationResult,
828 result.action_id.as_deref(),
829 None,
830 data.into_iter().collect(),
831 acknowledgement_timeout,
832 )
833 .map_err(|error| {
834 format!(
835 "startup reconciliation failed for run {} cancellation result: {error}",
836 started.run_id
837 )
838 })?;
839 }
840 }
841
842 let Some(ended) = ended else {
843 continue;
844 };
845 let Some(completion_digest) = ended.completion_digest.as_deref() else {
846 log.clear_run_binding(&started.run_id, &client_id)?;
847 continue;
848 };
849 if ended.client_id.as_deref() != Some(client_id.as_str()) {
850 tracing::error!(run_id = %started.run_id, "RunEnded owner does not match RunStarted during outbox reconciliation");
851 log.clear_run_binding(&started.run_id, &client_id)?;
852 continue;
853 }
854 let termination_kind = match &ended.termination {
855 car_proto::RunTermination::Outcome { .. } => "outcome",
856 car_proto::RunTermination::Incomplete => "incomplete",
857 car_proto::RunTermination::Cancelled { .. } => "cancelled",
858 };
859 let data = HashMap::from([
860 (
861 "termination_kind".to_string(),
862 Value::from(termination_kind),
863 ),
864 (
865 "completion_digest".to_string(),
866 Value::from(completion_digest),
867 ),
868 (
869 "termination".to_string(),
870 serde_json::to_value(&ended.termination).unwrap_or(Value::Null),
871 ),
872 ]);
873 log.append_critical_bounded(
874 car_eventlog::EventKind::RunCompleted,
875 None,
876 None,
877 data,
878 acknowledgement_timeout,
879 )
880 .map_err(|error| {
881 format!(
882 "startup reconciliation failed for run {} run_completed: {error}",
883 started.run_id
884 )
885 })?;
886 log.clear_run_binding(&started.run_id, &client_id)?;
887 }
888 }
889 Ok(())
890}
891
892/// Finish proposal terminals whose actions already ran before the daemon lost
893/// the journal acknowledgement. The outbox owns the exact normal-serde result
894/// and JCS digest, so startup only replays that terminal row; it never invokes
895/// the runtime or a tool executor.
896fn reconcile_pending_proposal_journals(
897 journals: &mut StartupJournalCache,
898 store: &crate::run_store::RunStore,
899 acknowledgement_timeout: std::time::Duration,
900) -> Result<(), String> {
901 let pending_proposals = match store.all_pending_proposals() {
902 Ok(pending) => pending,
903 Err(error) => {
904 tracing::error!(%error, "cannot enumerate proposal-finalization outbox; all unresolved runs remain quarantined");
905 return Ok(());
906 }
907 };
908 for pending in pending_proposals {
909 match store.completed_proposal(
910 &pending.run_id,
911 &pending.client_id,
912 pending.requested_policy_session_id.as_deref(),
913 &pending.original_submission,
914 ) {
915 Ok(Some(receipt)) => {
916 if let Err(error) = store.cleanup_completed_proposal_guards(&receipt) {
917 tracing::error!(run_id = %pending.run_id, %error, "completed proposal response is durable; startup guard cleanup remains pending");
918 }
919 continue;
920 }
921 Ok(None) => {}
922 Err(error) => {
923 tracing::error!(run_id = %pending.run_id, %error, "cannot validate exact completed proposal response during startup cleanup");
924 continue;
925 }
926 }
927 let (started, marker) = match store.pending_provenance(&pending) {
928 Ok(provenance) => provenance,
929 Err(error) => {
930 tracing::error!(run_id = %pending.run_id, %error, "proposal-finalization provenance is invalid; preserving outcome-unknown quarantine");
931 continue;
932 }
933 };
934 let Some(client_id) = started.client_id.as_deref() else {
935 tracing::error!(run_id = %pending.run_id, "durable RunStarted is missing client_id");
936 continue;
937 };
938 if client_id.is_empty()
939 || client_id.contains('/')
940 || client_id.contains('\\')
941 || client_id == "."
942 || client_id == ".."
943 {
944 tracing::error!(run_id = %pending.run_id, "proposal-finalization outbox has unsafe client_id");
945 continue;
946 }
947 let canonical = match car_inference::catalog_identity::canonical_json(
948 &pending.proposal_result,
949 ) {
950 Ok(canonical) => canonical,
951 Err(error) => {
952 tracing::error!(run_id = %pending.run_id, %error, "proposal-finalization result is not RFC 8785 canonicalizable");
953 continue;
954 }
955 };
956 let digest = format!("{:x}", Sha256::digest(canonical.as_bytes()));
957 if digest != pending.result_digest {
958 tracing::error!(run_id = %pending.run_id, "proposal-finalization digest does not match its durable result preimage");
959 continue;
960 }
961
962 let log = match journals.get(client_id) {
963 Ok(log) => log,
964 Err(error) => {
965 tracing::error!(run_id = %pending.run_id, %error, "cannot load proposal journal for outbox reconciliation");
966 continue;
967 }
968 };
969 if let Err(error) = log.bind_run(&started.run_id, client_id) {
970 tracing::error!(run_id = %pending.run_id, %error, "cannot bind proposal journal during outbox reconciliation");
971 continue;
972 }
973 if let Some(policy_session_id) = marker.policy_session_id.as_deref() {
974 if let Err(error) = log.bind_policy_session(policy_session_id) {
975 tracing::error!(run_id = %pending.run_id, %error, "cannot restore proposal policy binding during outbox reconciliation");
976 continue;
977 }
978 }
979 store.ensure_proposal_turns(&pending).map_err(|error| {
980 format!(
981 "startup reconciliation failed for run {} proposal trace: {error}",
982 pending.run_id
983 )
984 })?;
985 log.append_critical_bounded(
986 car_eventlog::EventKind::ProposalCompleted,
987 None,
988 Some(&pending.final_proposal_id),
989 pending.event_data(),
990 acknowledgement_timeout,
991 )
992 .map_err(|error| {
993 format!(
994 "startup reconciliation failed for run {} proposal_completed: {error}",
995 pending.run_id
996 )
997 })?;
998 if let Some(policy_session_id) = marker.policy_session_id.as_deref() {
999 log.clear_policy_session(policy_session_id)?;
1000 }
1001 let receipt = match store.write_completed_proposal(&pending) {
1002 Ok(receipt) => receipt,
1003 Err(error) => {
1004 tracing::error!(run_id = %pending.run_id, %error, "cannot persist reconciled completed proposal response");
1005 continue;
1006 }
1007 };
1008 if let Err(error) = store.cleanup_completed_proposal_guards(&receipt) {
1009 tracing::error!(run_id = %pending.run_id, %error, "completed proposal response is durable; startup guard cleanup remains pending");
1010 continue;
1011 }
1012 }
1013 Ok(())
1014}
1015
1016/// A completed response can outlive either cleanup guard when the daemon dies
1017/// between independently-fsynced unlink boundaries. Reconcile only under the
1018/// receipt's exact typed authority; never dispatch or append another event.
1019fn reconcile_completed_proposal_guards(store: &crate::run_store::RunStore) {
1020 if let Err(error) = store.reconcile_completed_proposal_migration() {
1021 tracing::error!(%error, "cannot migrate completed proposal response index; guards remain quarantined");
1022 }
1023}
1024
1025/// Per-run turn ceiling — a runaway-loop backstop sized well above any
1026/// healthy main-agent-only cycle (tens of turns), never a trimmer. This is
1027/// a TRUE hard cap, and it is enforced HERE — inside
1028/// [`ServerState::record_run_turns`], under the `runs` lock — not only in
1029/// the WS handler's pre-check. The handler keeps a fast-path pre-check off a
1030/// lock-free snapshot, but the dispatcher spawns a task per frame, so
1031/// pipelined `runs.record_turns` batches can all read a sub-ceiling snapshot
1032/// and pass that pre-check before any of them appends. Only the under-lock
1033/// check below is atomic with the append, so it is the one that actually
1034/// bounds the run (ADV-1). A batch that would take the run PAST this many
1035/// recorded turns is refused WHOLE.
1036pub const RECORD_TURNS_RUN_CEILING: usize = 2000;
1037
1038/// Outcome of [`ServerState::record_run_turns`] — distinguishes the three
1039/// reasons a batch can fail to land so the caller maps each to the right
1040/// `runs.record_turns` drop reason (ADV-1). Before this enum the function
1041/// returned a bare `usize` (the new total, or `0` for "nothing appended"),
1042/// which collapsed an under-lock CEILING refusal and an unknown/terminal run
1043/// into the same `0` — the handler then mislabeled a ceiling refusal as
1044/// `run_terminal`.
1045#[derive(Debug, Clone, PartialEq, Eq)]
1046pub enum RecordRunTurnsOutcome {
1047 /// The batch was appended; the run's new total turn count.
1048 Appended { new_total: usize },
1049 /// The batch was refused WHOLE because it would take the run past
1050 /// [`RECORD_TURNS_RUN_CEILING`] (a runaway backstop). Maps to
1051 /// `dropped: "run_turn_limit"`.
1052 RefusedCeiling,
1053 /// The run id is unknown to this process OR already terminal — nothing
1054 /// was appended. Maps to `dropped: "run_terminal"` (the benign TOCTOU:
1055 /// the run went terminal between the handler's pre-check and the
1056 /// append), matching the prior silent-zero semantics.
1057 UnknownOrTerminal,
1058 /// The batch cannot advance canonical memory/subscribers. Ordinary I/O
1059 /// failures mean the durable append boundary was not acknowledged and an
1060 /// exact retry is safe. A `run trace corruption:` error instead means the
1061 /// bytes exposed committed middle corruption; the run is quarantined and
1062 /// must not be retried or completed.
1063 PersistenceFailed(String),
1064}
1065
1066pub(crate) enum RunSubscribePageResult {
1067 Ready(car_proto::RunSubscribeResponse),
1068 Durable {
1069 agent_id: String,
1070 status: car_proto::RunLiveStatus,
1071 },
1072}
1073
1074impl RunMeta {
1075 /// True once a terminal record (outcome or `Incomplete`) is set.
1076 pub fn is_terminal(&self) -> bool {
1077 self.termination.is_some()
1078 }
1079
1080 pub fn accepts_proposals(&self) -> bool {
1081 self.start_committed
1082 && self.pending_terminal.is_none()
1083 && self.cancellation_pending.is_none()
1084 && self.trace_corruption.is_none()
1085 && !self.is_terminal()
1086 }
1087
1088 /// The coarse live status of this run for the U4 subscribe snapshot /
1089 /// `runs.trace.event` (agent run tracing). Mirrors `RunStore`'s
1090 /// `RunStatus` but reads off the in-memory `RunMeta` so the live path
1091 /// never touches disk.
1092 pub fn live_status(&self) -> car_proto::RunLiveStatus {
1093 match &self.termination {
1094 None if self.cancellation_pending.is_some() => {
1095 car_proto::RunLiveStatus::CancellationPending
1096 }
1097 None => car_proto::RunLiveStatus::InProgress,
1098 Some(car_proto::RunTermination::Outcome { .. }) => car_proto::RunLiveStatus::Completed,
1099 Some(car_proto::RunTermination::Incomplete) => car_proto::RunLiveStatus::Incomplete,
1100 Some(car_proto::RunTermination::Cancelled { .. }) => {
1101 car_proto::RunLiveStatus::Cancelled
1102 }
1103 }
1104 }
1105
1106 /// Count of `RunRecord::Turn` entries in this run's buffer — the live
1107 /// turn cursor. Equals `turns.len()` because the buffer holds only
1108 /// `Turn` records (Started/Ended are not buffered here).
1109 pub fn turn_cursor(&self) -> usize {
1110 self.turns.len()
1111 }
1112}
1113
1114/// Default ceiling for the daemon→host `tools.execute` callback wait when
1115/// an action carries no explicit `timeout_ms`. Overridable via
1116/// `CAR_TOOL_TIMEOUT` (seconds). Raised from the old hardcoded 60s — real
1117/// tools (build steps, CLI drivers, slow APIs) routinely run longer, and a
1118/// 60s ceiling reaped them regardless of the agent's budget (Parslee-ai/car#259).
1119pub const DEFAULT_TOOL_TIMEOUT_MS: u64 = 300_000;
1120
1121/// Grace added to the callback wait above an action's own `timeout_ms`. The
1122/// executor (`car-engine`) already wraps each attempt in
1123/// `timeout(action.timeout_ms, dispatch)`; if this wait used the *same*
1124/// value it would race that outer deadline with zero slack. By waiting a
1125/// little longer, the executor's deadline fires first and reaps+rolls back
1126/// cleanly with its structured "action timed out" error — this wait is only
1127/// a transport backstop for the case the executor deadline doesn't apply.
1128const TOOL_TIMEOUT_GRACE_MS: u64 = 5_000;
1129
1130/// The callback-wait budget for one tool call.
1131///
1132/// - **`Some(X)`**: the executor is the authority (it bounds the attempt at
1133/// `X`); this wait is `X + grace`, a backstop that lets the executor reap
1134/// first. So the agent's per-action budget takes effect — the bug was that
1135/// this wait used a hardcoded 60s and ignored `X` entirely (car#259).
1136/// - **`None`**: the executor applies **no** deadline, so this wait is the
1137/// *sole* bound on the call — the `CAR_TOOL_TIMEOUT`-overridable
1138/// [`DEFAULT_TOOL_TIMEOUT_MS`].
1139fn tool_callback_timeout(action_timeout_ms: Option<u64>) -> std::time::Duration {
1140 let ms = match action_timeout_ms {
1141 Some(budget) => budget.saturating_add(TOOL_TIMEOUT_GRACE_MS),
1142 None => std::env::var("CAR_TOOL_TIMEOUT")
1143 .ok()
1144 .and_then(|s| s.trim().parse::<u64>().ok())
1145 .map(|secs| secs.saturating_mul(1000))
1146 .unwrap_or(DEFAULT_TOOL_TIMEOUT_MS),
1147 };
1148 std::time::Duration::from_millis(ms)
1149}
1150
1151/// Build and write the fire-and-forget `tools.cancel` notification for a
1152/// cancelled `tools.execute` callback (Parslee-ai/car#264).
1153///
1154/// This is the ONE place that constructs a [`ToolCancelRequest`] frame and
1155/// puts it on the wire; every cancellation path routes through
1156/// [`PendingToolCall`]'s drop, which calls this. Best-effort by design — the
1157/// daemon has already given up on the call, so a failed write changes nothing
1158/// it can act on.
1159pub(crate) async fn write_tool_cancel(
1160 channel: &WsChannel,
1161 request_id: String,
1162 action_id: String,
1163 reason: String,
1164) {
1165 use futures::SinkExt;
1166 let cancel = ToolCancelRequest {
1167 request_id,
1168 action_id,
1169 reason,
1170 };
1171 let notification = serde_json::json!({
1172 "jsonrpc": "2.0",
1173 "method": "tools.cancel",
1174 "params": cancel,
1175 });
1176 let Ok(text) = serde_json::to_string(¬ification) else {
1177 return;
1178 };
1179 let _ = channel
1180 .write
1181 .lock()
1182 .await
1183 .send(Message::Text(text.into()))
1184 .await;
1185}
1186
1187/// RAII guard covering one in-flight daemon→host `tools.execute` callback
1188/// (Parslee-ai/car#264).
1189///
1190/// Armed from the moment the oneshot sender lands in
1191/// [`WsChannel::pending`]; disarmed only when a `ToolExecuteResponse` is
1192/// actually received. If it is still armed when it drops, the call was
1193/// cancelled and the guard both releases the `pending` entry and tells the
1194/// host to abort the in-flight child.
1195///
1196/// Why RAII rather than an explicit cleanup branch: `tokio::time::timeout`
1197/// **drops** the inner future on expiry, and the daemon's callback wait is
1198/// deliberately the *outermost* of three deadlines
1199/// (`TOOL_TIMEOUT_GRACE_MS` above the executor's per-action budget). So on
1200/// every real reap the whole dispatch future is dropped while parked on
1201/// `rx`, and any code written after the `await` — including the explicit
1202/// timeout arm — never runs. A `Drop` impl is the only cleanup that survives
1203/// being dropped.
1204struct PendingToolCall {
1205 channel: Arc<WsChannel>,
1206 request_id: String,
1207 action_id: String,
1208 reason: String,
1209 armed: bool,
1210}
1211
1212impl PendingToolCall {
1213 fn new(channel: Arc<WsChannel>, request_id: String, action_id: String, reason: String) -> Self {
1214 Self {
1215 channel,
1216 request_id,
1217 action_id,
1218 reason,
1219 armed: true,
1220 }
1221 }
1222
1223 /// A response was delivered — this call completed, so no cancel is owed.
1224 fn disarm(&mut self) {
1225 self.armed = false;
1226 }
1227}
1228
1229impl Drop for PendingToolCall {
1230 fn drop(&mut self) {
1231 if !self.armed {
1232 return;
1233 }
1234 // `pending` is a `tokio::sync::Mutex` and the write is async, but
1235 // `Drop` is sync — so the cleanup has to be a detached task. Outside
1236 // a runtime (a plain `drop` in sync code) there is nothing to spawn
1237 // onto; skip quietly rather than panicking out of a destructor.
1238 let Ok(handle) = tokio::runtime::Handle::try_current() else {
1239 return;
1240 };
1241 let channel = self.channel.clone();
1242 let request_id = std::mem::take(&mut self.request_id);
1243 let action_id = std::mem::take(&mut self.action_id);
1244 let reason = std::mem::take(&mut self.reason);
1245 handle.spawn(async move {
1246 // Only cancel if WE claimed the pending entry. The response
1247 // demux in `handler.rs` removes it on the success path, and
1248 // connection teardown clears the whole map — in either case the
1249 // host is not waiting on anything we should abort.
1250 let claimed = channel.pending.lock().await.remove(&request_id).is_some();
1251 channel.active_actions.lock().await.remove(&request_id);
1252 if claimed {
1253 write_tool_cancel(&channel, request_id, action_id, reason).await;
1254 }
1255 });
1256 }
1257}
1258
1259/// Tool executor that sends callbacks to the client over WebSocket.
1260pub struct WsToolExecutor {
1261 pub channel: Arc<WsChannel>,
1262 negotiated_capabilities: Arc<std::sync::RwLock<std::collections::BTreeSet<String>>>,
1263 halted: Arc<AtomicBool>,
1264}
1265
1266#[async_trait::async_trait]
1267impl ToolExecutor for WsToolExecutor {
1268 async fn execute(&self, tool: &str, params: &Value) -> Result<Value, String> {
1269 // Legacy callers that don't have a proposal-level Action.id
1270 // (e.g. internal `executor.execute` chains in tests) — emit an
1271 // empty action_id so the client-side handler can still see the
1272 // payload shape and decide whether to fail loudly.
1273 self.execute_with_action(tool, params, "", None).await
1274 }
1275
1276 async fn execute_with_action(
1277 &self,
1278 tool: &str,
1279 params: &Value,
1280 action_id: &str,
1281 timeout_ms: Option<u64>,
1282 ) -> Result<Value, String> {
1283 self.execute_with_action_in_session(tool, params, action_id, timeout_ms, None, 1)
1284 .await
1285 }
1286
1287 async fn execute_with_action_in_session(
1288 &self,
1289 tool: &str,
1290 params: &Value,
1291 action_id: &str,
1292 timeout_ms: Option<u64>,
1293 session_id: Option<&str>,
1294 attempt: u32,
1295 ) -> Result<Value, String> {
1296 self.execute_callback(
1297 tool,
1298 params,
1299 action_id,
1300 timeout_ms,
1301 session_id,
1302 attempt,
1303 &HashMap::new(),
1304 None,
1305 false,
1306 )
1307 .await
1308 .map(|execution| execution.output)
1309 .map_err(|failure| failure.message)
1310 }
1311
1312 async fn execute_with_action_state_in_session(
1313 &self,
1314 tool: &str,
1315 params: &Value,
1316 action_id: &str,
1317 timeout_ms: Option<u64>,
1318 session_id: Option<&str>,
1319 attempt: u32,
1320 expected_effects: &HashMap<String, Value>,
1321 return_schema: Option<&Value>,
1322 ) -> Result<car_engine::ToolExecution, String> {
1323 self.execute_callback(
1324 tool,
1325 params,
1326 action_id,
1327 timeout_ms,
1328 session_id,
1329 attempt,
1330 expected_effects,
1331 return_schema,
1332 self.callback_state_negotiated(),
1333 )
1334 .await
1335 .map_err(|failure| failure.message)
1336 }
1337
1338 async fn execute_classified(
1339 &self,
1340 tool: &str,
1341 params: &Value,
1342 action_id: &str,
1343 timeout_ms: Option<u64>,
1344 session_id: Option<&str>,
1345 attempt: u32,
1346 expected_effects: &HashMap<String, Value>,
1347 return_schema: Option<&Value>,
1348 ) -> Result<car_engine::ToolExecution, ToolFailure> {
1349 self.execute_callback(
1350 tool,
1351 params,
1352 action_id,
1353 timeout_ms,
1354 session_id,
1355 attempt,
1356 expected_effects,
1357 return_schema,
1358 self.callback_state_negotiated(),
1359 )
1360 .await
1361 }
1362}
1363
1364impl WsToolExecutor {
1365 pub fn new(
1366 channel: Arc<WsChannel>,
1367 negotiated_capabilities: Arc<std::sync::RwLock<std::collections::BTreeSet<String>>>,
1368 halted: Arc<AtomicBool>,
1369 ) -> Self {
1370 Self {
1371 channel,
1372 negotiated_capabilities,
1373 halted,
1374 }
1375 }
1376
1377 #[cfg(test)]
1378 fn legacy(channel: Arc<WsChannel>) -> Self {
1379 Self::new(
1380 channel,
1381 Arc::new(std::sync::RwLock::new(std::collections::BTreeSet::new())),
1382 Arc::new(AtomicBool::new(false)),
1383 )
1384 }
1385
1386 fn callback_state_negotiated(&self) -> bool {
1387 self.negotiated_capabilities
1388 .read()
1389 .map(|capabilities| capabilities.contains(car_proto::TOOLS_CALLBACK_STATE_CAPABILITY))
1390 .unwrap_or(false)
1391 }
1392
1393 #[allow(clippy::too_many_arguments)]
1394 async fn execute_callback(
1395 &self,
1396 tool: &str,
1397 params: &Value,
1398 action_id: &str,
1399 timeout_ms: Option<u64>,
1400 session_id: Option<&str>,
1401 attempt: u32,
1402 expected_effects: &HashMap<String, Value>,
1403 return_schema: Option<&Value>,
1404 callback_state_negotiated: bool,
1405 ) -> Result<car_engine::ToolExecution, ToolFailure> {
1406 use futures::SinkExt;
1407
1408 // The JSON-RPC request id is the daemon's callback-routing key
1409 // (used by the pending-response map below). The `action_id`
1410 // FIELD on the payload is the originating proposal Action.id
1411 // surfaced to the host so process-wide handlers can route
1412 // concurrent callbacks back to per-call dispatchers
1413 // (Parslee-ai/car-releases#43 follow-up). They serve different
1414 // purposes and must stay distinct: routing id is daemon-side,
1415 // action id is host-side.
1416 let request_id = self.channel.next_request_id();
1417
1418 let callback = ToolExecuteRequest {
1419 action_id: action_id.to_string(),
1420 tool: tool.to_string(),
1421 parameters: params.clone(),
1422 // Surface the action's budget to the host so its own tool runner
1423 // can bound the work too (Parslee-ai/car#259) — was always None.
1424 timeout_ms,
1425 // The engine's real retry counter, 1-based. This was hardcoded to
1426 // `1`, which made the field a constant on the wire: a host building
1427 // the retry-disambiguating join the field exists for got a key that
1428 // never varied, and would only find out under the concurrent-retry
1429 // conditions the join is for (Parslee-ai/car#928).
1430 attempt,
1431 // Surface the routing id so the host can key a per-call abort
1432 // registry on it; a reap emits `tools.cancel` carrying the same id
1433 // (Parslee-ai/car#264).
1434 request_id: request_id.clone(),
1435 // Server-stamped correlation. The executor has always passed this
1436 // in; the parameter was `_session_id` and the value was discarded,
1437 // so every host that needed attribution had to rebuild it from
1438 // client-side convention (Parslee-ai/car#904).
1439 session_id: session_id.map(str::to_string),
1440 };
1441
1442 // Create a oneshot channel for the response
1443 let (tx, rx) = oneshot::channel();
1444 self.channel
1445 .pending
1446 .lock()
1447 .await
1448 .insert(request_id.clone(), tx);
1449 self.channel
1450 .active_actions
1451 .lock()
1452 .await
1453 .insert(request_id.clone(), action_id.to_string());
1454
1455 // From here on the call is in flight. The guard makes cleanup +
1456 // host-abort survive ANY cancellation of this future — the
1457 // executor's per-action `timeout_ms` deadline (the common case, it
1458 // fires `TOOL_TIMEOUT_GRACE_MS` before our own wait below), the
1459 // server's per-method deadline in `handler.rs`, or the callback wait
1460 // here — not just the one arm we can write code in
1461 // (Parslee-ai/car#264).
1462 let mut pending_call = PendingToolCall::new(
1463 self.channel.clone(),
1464 request_id.clone(),
1465 action_id.to_string(),
1466 format!("tool '{tool}' call cancelled before completion (request {request_id})"),
1467 );
1468
1469 // Send the callback to the client as a JSON-RPC request
1470 let rpc_request = serde_json::json!({
1471 "jsonrpc": "2.0",
1472 "method": "tools.execute",
1473 "params": callback,
1474 "id": request_id,
1475 });
1476
1477 let msg = Message::Text(
1478 serde_json::to_string(&rpc_request)
1479 .map_err(|e| e.to_string())?
1480 .into(),
1481 );
1482 self.channel
1483 .write
1484 .lock()
1485 .await
1486 .send(msg)
1487 .await
1488 .map_err(|e| format!("failed to send tool callback: {}", e))?;
1489
1490 // Wait for the client to respond, bounded by the action's budget
1491 // (or the configurable default) — NOT a hardcoded 60s. This is the
1492 // ceiling that previously ignored `Action.timeout_ms` and reaped
1493 // legitimately-long tool calls (Parslee-ai/car#259).
1494 let wait = tool_callback_timeout(timeout_ms);
1495 let response = match tokio::time::timeout(wait, rx).await {
1496 Ok(inner) => inner.map_err(|_| format!("tool '{}' callback channel closed", tool))?,
1497 Err(_) => {
1498 // Reaped by our own callback wait. Sharpen the guard's reason
1499 // to name the timeout, then let it do the cleanup + host abort
1500 // on the way out — the same path a dropped future takes
1501 // (Parslee-ai/car#264).
1502 let reason = format!("tool '{}' callback timed out ({}s)", tool, wait.as_secs());
1503 pending_call.reason = reason.clone();
1504 return Err(ToolFailure::ordinary(reason));
1505 }
1506 };
1507
1508 // A response was delivered (error field set or not): the call is
1509 // finished, so no cancel is owed to the host.
1510 self.channel.active_actions.lock().await.remove(&request_id);
1511 pending_call.disarm();
1512
1513 if let Some(err) = response.error {
1514 let failure = match response.terminal {
1515 false => ToolFailure::ordinary(err),
1516 true => {
1517 // Latch at the callback boundary, before returning to the
1518 // engine. Handler-level result inspection repeats this as
1519 // defense in depth, but this write closes the cancellation
1520 // gap between typed evidence arriving and execute() ending.
1521 self.halted.store(true, Ordering::Release);
1522 ToolFailure::terminal(err)
1523 }
1524 };
1525 return Err(failure);
1526 }
1527 decode_callback_execution(
1528 tool,
1529 action_id,
1530 response.output.unwrap_or(Value::Null),
1531 expected_effects,
1532 return_schema,
1533 callback_state_negotiated,
1534 )
1535 .map_err(ToolFailure::ordinary)
1536 }
1537}
1538
1539fn decode_callback_execution(
1540 tool: &str,
1541 action_id: &str,
1542 result: Value,
1543 expected_effects: &HashMap<String, Value>,
1544 return_schema: Option<&Value>,
1545 callback_state_negotiated: bool,
1546) -> Result<car_engine::ToolExecution, String> {
1547 if !callback_state_negotiated {
1548 return Ok(car_engine::ToolExecution::output_only(result));
1549 }
1550 let envelope = result.as_object().filter(|object| {
1551 object.len() == 2 && object.contains_key("output") && object.contains_key("state_changes")
1552 });
1553
1554 if expected_effects.is_empty() && envelope.is_none() {
1555 return Ok(car_engine::ToolExecution::output_only(result));
1556 }
1557 let envelope = envelope.ok_or_else(|| {
1558 format!(
1559 "tool '{tool}' callback for action '{action_id}' must return exact envelope {{output,state_changes}}"
1560 )
1561 })?;
1562 let output = envelope
1563 .get("output")
1564 .cloned()
1565 .expect("exact envelope contains output");
1566 let state_changes: HashMap<String, Value> = serde_json::from_value(
1567 envelope
1568 .get("state_changes")
1569 .cloned()
1570 .expect("exact envelope contains state_changes"),
1571 )
1572 .map_err(|_| {
1573 format!("tool '{tool}' callback for action '{action_id}' state_changes must be an object")
1574 })?;
1575 let expected_keys: std::collections::BTreeSet<&str> =
1576 expected_effects.keys().map(String::as_str).collect();
1577 let actual_keys: std::collections::BTreeSet<&str> =
1578 state_changes.keys().map(String::as_str).collect();
1579 if expected_keys != actual_keys {
1580 return Err(format!(
1581 "tool '{tool}' callback state_changes keys do not match action '{action_id}': expected {:?}, got {:?}",
1582 expected_keys, actual_keys
1583 ));
1584 }
1585 if let Some(schema) = return_schema {
1586 car_engine::validate_tool_output(tool, schema, &output)?;
1587 }
1588 let changes_value = serde_json::to_value(&state_changes).map_err(|error| {
1589 format!("tool '{tool}' callback state_changes serialization failed: {error}")
1590 })?;
1591 car_inference::catalog_identity::canonical_json(&changes_value).map_err(|error| {
1592 format!("tool '{tool}' callback state_changes failed JCS/I-JSON validation: {error}")
1593 })?;
1594 Ok(car_engine::ToolExecution {
1595 output,
1596 state_changes,
1597 })
1598}
1599
1600#[cfg(test)]
1601mod tool_cancel_tests {
1602 use super::*;
1603 use serde_json::json;
1604 use std::time::Duration;
1605
1606 /// Every `Message::Text` the executor wrote, parsed as JSON.
1607 fn written(frames: &std::sync::Arc<std::sync::Mutex<Vec<String>>>) -> Vec<Value> {
1608 frames
1609 .lock()
1610 .unwrap()
1611 .iter()
1612 .map(|t| serde_json::from_str::<Value>(t).expect("frame is JSON"))
1613 .collect()
1614 }
1615
1616 fn frame_with_method<'a>(frames: &'a [Value], method: &str) -> Option<&'a Value> {
1617 frames.iter().find(|f| f["method"] == json!(method))
1618 }
1619
1620 /// Let detached cleanup tasks (the `PendingToolCall` drop spawn) run.
1621 async fn settle() {
1622 for _ in 0..8 {
1623 tokio::task::yield_now().await;
1624 }
1625 tokio::time::sleep(Duration::from_millis(100)).await;
1626 for _ in 0..8 {
1627 tokio::task::yield_now().await;
1628 }
1629 }
1630
1631 /// The daemon's own callback wait expiring still cancels — the arm that
1632 /// used to hold the inline cleanup now delegates to the drop guard, and
1633 /// the observable behaviour (Err text, cancel frame, drained `pending`)
1634 /// must be unchanged.
1635 #[tokio::test(start_paused = true)]
1636 async fn callback_wait_expiry_cancels() {
1637 let (channel, frames) = WsChannel::test_capture();
1638 let channel = Arc::new(channel);
1639 let executor = WsToolExecutor::legacy(channel.clone());
1640
1641 // 1ms budget → a ~5s callback wait; paused time makes that instant.
1642 let err = executor
1643 .execute_with_action_in_session("drive_cli", &json!({}), "a0", Some(1), None, 1)
1644 .await
1645 .expect_err("the callback wait must expire");
1646 assert!(
1647 err.contains("tool 'drive_cli' callback timed out"),
1648 "unexpected error text: {err}"
1649 );
1650
1651 settle().await;
1652
1653 let frames = written(&frames);
1654 let execute = frame_with_method(&frames, "tools.execute").expect("tools.execute was sent");
1655 let cancel = frame_with_method(&frames, "tools.cancel")
1656 .expect("a tools.cancel notification must reach the wire");
1657 assert_eq!(
1658 cancel["params"]["request_id"], execute["params"]["request_id"],
1659 "cancel must correlate to the originating request_id"
1660 );
1661 assert_eq!(cancel["params"]["action_id"], json!("a0"));
1662 assert!(
1663 cancel["id"].is_null(),
1664 "cancel is a notification, not a request"
1665 );
1666 assert!(
1667 channel.pending.lock().await.is_empty(),
1668 "the pending entry must be released"
1669 );
1670 }
1671
1672 /// Parslee-ai/car#904 — the execution session must reach the wire.
1673 ///
1674 /// The executor has always passed a session id in; the parameter was named
1675 /// `_session_id` and the value was dropped on the floor, so every host that
1676 /// needed to know which mission a callback belonged to had to rebuild it
1677 /// from client-side convention. Asserted on the actual `tools.execute`
1678 /// frame rather than on the struct, because the struct field existing is
1679 /// not the property that was missing.
1680 #[tokio::test(start_paused = true)]
1681 async fn tools_execute_carries_the_execution_session() {
1682 let (channel, frames) = WsChannel::test_capture();
1683 let channel = Arc::new(channel);
1684 let executor = WsToolExecutor::legacy(channel.clone());
1685
1686 let _ = tokio::time::timeout(
1687 Duration::from_millis(50),
1688 executor.execute_with_action_in_session(
1689 "search",
1690 &json!({}),
1691 "a0",
1692 Some(10_000),
1693 Some("sess-42"),
1694 1,
1695 ),
1696 )
1697 .await;
1698 settle().await;
1699
1700 let frames = written(&frames);
1701 let execute = frame_with_method(&frames, "tools.execute").expect("tools.execute was sent");
1702 assert_eq!(
1703 execute["params"]["session_id"],
1704 json!("sess-42"),
1705 "the daemon must stamp the session it already knows, not leave \
1706 attribution to the host"
1707 );
1708 }
1709
1710 /// Parslee-ai/car#928 — the retry counter on the wire must be the engine's
1711 /// real one.
1712 ///
1713 /// It was hardcoded to `1` here, so the field a host would build its
1714 /// retry-disambiguating join on never varied. That is worse than the field
1715 /// being absent: the join looks correct and degenerates silently, and only
1716 /// under the concurrent-retry conditions it exists to handle.
1717 #[tokio::test(start_paused = true)]
1718 async fn tools_execute_carries_the_real_attempt_number() {
1719 for attempt in [1u32, 2, 7] {
1720 let (channel, frames) = WsChannel::test_capture();
1721 let channel = Arc::new(channel);
1722 let executor = WsToolExecutor::legacy(channel.clone());
1723
1724 let _ = tokio::time::timeout(
1725 Duration::from_millis(50),
1726 executor.execute_with_action_in_session(
1727 "search",
1728 &json!({}),
1729 "a0",
1730 Some(10_000),
1731 Some("sess-1"),
1732 attempt,
1733 ),
1734 )
1735 .await;
1736 settle().await;
1737
1738 let frames = written(&frames);
1739 let execute =
1740 frame_with_method(&frames, "tools.execute").expect("tools.execute was sent");
1741 assert_eq!(
1742 execute["params"]["attempt"],
1743 json!(attempt),
1744 "the payload must carry the engine's attempt, not a constant"
1745 );
1746 }
1747 }
1748
1749 /// The other half: a caller with no session must omit the key rather than
1750 /// send a null. `execute()` and in-process executors legitimately have
1751 /// none, and a null would make every such payload differ from the pre-#904
1752 /// shape for no gain.
1753 #[tokio::test(start_paused = true)]
1754 async fn a_sessionless_call_omits_the_session_key() {
1755 let (channel, frames) = WsChannel::test_capture();
1756 let channel = Arc::new(channel);
1757 let executor = WsToolExecutor::legacy(channel.clone());
1758
1759 let _ = tokio::time::timeout(
1760 Duration::from_millis(50),
1761 executor.execute_with_action_in_session(
1762 "search",
1763 &json!({}),
1764 "a0",
1765 Some(10_000),
1766 None,
1767 1,
1768 ),
1769 )
1770 .await;
1771 settle().await;
1772
1773 let frames = written(&frames);
1774 let execute = frame_with_method(&frames, "tools.execute").expect("tools.execute was sent");
1775 assert!(
1776 execute["params"].get("session_id").is_none(),
1777 "a sessionless call must omit the key entirely, got: {}",
1778 execute["params"]
1779 );
1780 }
1781
1782 /// REGRESSION (Parslee-ai/car#264): the executor's per-action deadline
1783 /// fires `TOOL_TIMEOUT_GRACE_MS` BEFORE the daemon's callback wait, so it
1784 /// drops this whole future while it is parked on `rx`. Before the drop
1785 /// guard, nothing ran on that path: no `tools.cancel` frame was ever
1786 /// written and the `pending` entry leaked until connection teardown,
1787 /// orphaning the host's in-flight child.
1788 #[tokio::test(start_paused = true)]
1789 async fn engine_deadline_drop_cancels() {
1790 let (channel, frames) = WsChannel::test_capture();
1791 let channel = Arc::new(channel);
1792 let executor = WsToolExecutor::legacy(channel.clone());
1793
1794 // Outer deadline (standing in for the engine's per-action timeout)
1795 // at 50ms; the callback wait would be 10s + 5s grace.
1796 let outcome = tokio::time::timeout(
1797 Duration::from_millis(50),
1798 executor.execute_with_action_in_session(
1799 "drive_cli",
1800 &json!({}),
1801 "a0",
1802 Some(10_000),
1803 None,
1804 1,
1805 ),
1806 )
1807 .await;
1808 assert!(
1809 outcome.is_err(),
1810 "the outer deadline must fire first and drop the dispatch future"
1811 );
1812
1813 settle().await;
1814
1815 let frames = written(&frames);
1816 let execute = frame_with_method(&frames, "tools.execute").expect("tools.execute was sent");
1817 let cancel = frame_with_method(&frames, "tools.cancel").expect(
1818 "dropping the dispatch future must still emit tools.cancel so the host child isn't orphaned",
1819 );
1820 assert_eq!(
1821 cancel["params"]["request_id"], execute["params"]["request_id"],
1822 "cancel must correlate to the originating request_id"
1823 );
1824 assert!(
1825 channel.pending.lock().await.is_empty(),
1826 "the pending entry must not leak"
1827 );
1828 }
1829
1830 /// The happy path must stay silent: a delivered response disarms the
1831 /// guard, so no host abort is emitted for a call that completed.
1832 #[tokio::test]
1833 async fn success_emits_no_cancel() {
1834 let (channel, frames) = WsChannel::test_capture();
1835 let channel = Arc::new(channel);
1836 let executor = WsToolExecutor::legacy(channel.clone());
1837
1838 let responder = channel.clone();
1839 let host = tokio::spawn(async move {
1840 loop {
1841 let claimed = {
1842 let mut pending = responder.pending.lock().await;
1843 let key = pending.keys().next().cloned();
1844 key.and_then(|k| pending.remove(&k).map(|tx| (k, tx)))
1845 };
1846 if let Some((request_id, tx)) = claimed {
1847 let _ = tx.send(ToolExecuteResponse {
1848 action_id: request_id,
1849 output: Some(json!({"ok": true})),
1850 error: None,
1851 terminal: false,
1852 });
1853 return;
1854 }
1855 tokio::task::yield_now().await;
1856 }
1857 });
1858
1859 let out = executor
1860 .execute_with_action_in_session("drive_cli", &json!({}), "a0", Some(10_000), None, 1)
1861 .await
1862 .expect("the host responded, so the call succeeds");
1863 assert_eq!(out, json!({"ok": true}));
1864 host.await.expect("responder task finishes");
1865
1866 settle().await;
1867
1868 let frames = written(&frames);
1869 assert!(
1870 frame_with_method(&frames, "tools.execute").is_some(),
1871 "tools.execute was sent"
1872 );
1873 assert!(
1874 frame_with_method(&frames, "tools.cancel").is_none(),
1875 "a completed call must not emit tools.cancel"
1876 );
1877 assert!(channel.pending.lock().await.is_empty());
1878 }
1879}
1880
1881/// The bare commodity-tool names owned by the engine's bound substrate.
1882/// When a session binds a non-Local substrate (`session.bindSubstrate`),
1883/// these names must reach the engine's substrate-after-fall_through path
1884/// (`executor.rs`), NOT the WS tool callback. Everything else — connector
1885/// `mcp_{slug}_*` routes and host-executed client tools — is untouched.
1886///
1887/// `calculate` is deliberately excluded: it is a pure, environment-free
1888/// tool that the engine already handles in-process via `agent_basics`, so
1889/// it never needs to be re-routed to a substrate.
1890pub(crate) const SUBSTRATE_OWNED_TOOLS: &[&str] = &[
1891 "read_file",
1892 "write_file",
1893 "edit_file",
1894 "list_dir",
1895 "find_files",
1896 "grep_files",
1897];
1898
1899/// Executor wrapper used **only** by substrate-bound sessions
1900/// (`session.bindSubstrate`). It is the low-blast-radius "option (a)" from
1901/// `docs/execution-substrate.md` §3: keep the existing
1902/// `share_with_fallback(ws_executor)` composition intact for connector
1903/// routes and host tool callbacks, but for the bare substrate-owned
1904/// built-in names ([`SUBSTRATE_OWNED_TOOLS`]) return an `"unknown tool:
1905/// <name>"` error so the engine's dispatch (`executor.rs`) takes the
1906/// `fall_through` branch and routes those names to the runtime's bound
1907/// substrate (the connector/VM [`car_engine::McpSubstrate`]) instead of the WS client.
1908///
1909/// This preserves every existing contract — GUI/connector/coder sessions
1910/// never construct one of these — and is what makes a connector-driven
1911/// session *coherent*: its file/exec built-ins land on the same machine as
1912/// its `mcp_{slug}_*` tools, not split between the WS client and a remote
1913/// MCP server.
1914pub struct SubstrateShadowExecutor {
1915 inner: Arc<dyn ToolExecutor>,
1916}
1917
1918impl SubstrateShadowExecutor {
1919 pub fn new(inner: Arc<dyn ToolExecutor>) -> Self {
1920 Self { inner }
1921 }
1922}
1923
1924#[async_trait::async_trait]
1925impl ToolExecutor for SubstrateShadowExecutor {
1926 async fn execute(&self, tool: &str, params: &Value) -> Result<Value, String> {
1927 self.execute_with_action(tool, params, "", None).await
1928 }
1929
1930 async fn execute_with_action(
1931 &self,
1932 tool: &str,
1933 params: &Value,
1934 action_id: &str,
1935 timeout_ms: Option<u64>,
1936 ) -> Result<Value, String> {
1937 self.execute_with_action_in_session(tool, params, action_id, timeout_ms, None, 1)
1938 .await
1939 }
1940
1941 async fn execute_with_action_in_session(
1942 &self,
1943 tool: &str,
1944 params: &Value,
1945 action_id: &str,
1946 timeout_ms: Option<u64>,
1947 session_id: Option<&str>,
1948 attempt: u32,
1949 ) -> Result<Value, String> {
1950 if SUBSTRATE_OWNED_TOOLS.contains(&tool) {
1951 // Signal the engine to fall through to the bound substrate.
1952 // The exact `"unknown tool"` prefix is the contract the engine
1953 // dispatch matches on (`executor.rs`).
1954 return Err(format!("unknown tool: {tool}"));
1955 }
1956 self.inner
1957 .execute_with_action_in_session(
1958 tool, params, action_id, timeout_ms, session_id, attempt,
1959 )
1960 .await
1961 }
1962
1963 async fn execute_with_action_state_in_session(
1964 &self,
1965 tool: &str,
1966 params: &Value,
1967 action_id: &str,
1968 timeout_ms: Option<u64>,
1969 session_id: Option<&str>,
1970 attempt: u32,
1971 expected_effects: &HashMap<String, Value>,
1972 return_schema: Option<&Value>,
1973 ) -> Result<car_engine::ToolExecution, String> {
1974 if SUBSTRATE_OWNED_TOOLS.contains(&tool) {
1975 return Err(format!("unknown tool: {tool}"));
1976 }
1977 self.inner
1978 .execute_with_action_state_in_session(
1979 tool,
1980 params,
1981 action_id,
1982 timeout_ms,
1983 session_id,
1984 attempt,
1985 expected_effects,
1986 return_schema,
1987 )
1988 .await
1989 }
1990
1991 async fn execute_classified(
1992 &self,
1993 tool: &str,
1994 params: &Value,
1995 action_id: &str,
1996 timeout_ms: Option<u64>,
1997 session_id: Option<&str>,
1998 attempt: u32,
1999 expected_effects: &HashMap<String, Value>,
2000 return_schema: Option<&Value>,
2001 ) -> Result<car_engine::ToolExecution, ToolFailure> {
2002 if SUBSTRATE_OWNED_TOOLS.contains(&tool) {
2003 return Err(ToolFailure::ordinary(format!("unknown tool: {tool}")));
2004 }
2005 self.inner
2006 .execute_classified(
2007 tool,
2008 params,
2009 action_id,
2010 timeout_ms,
2011 session_id,
2012 attempt,
2013 expected_effects,
2014 return_schema,
2015 )
2016 .await
2017 }
2018}
2019
2020/// Voice event sink that forwards events to a specific WebSocket client
2021/// as `voice.event` JSON-RPC notifications.
2022///
2023/// Each `voice.transcribe_stream.start` call constructs one of these
2024/// bound to the originating client's [`WsChannel`], so a client only
2025/// receives events for sessions it started.
2026pub struct WsVoiceEventSink {
2027 pub channel: Arc<WsChannel>,
2028}
2029
2030impl car_voice::VoiceEventSink for WsVoiceEventSink {
2031 fn send(&self, session_id: &str, event_json: String) {
2032 use futures::SinkExt;
2033 let channel = self.channel.clone();
2034 let session_id = session_id.to_string();
2035 tokio::spawn(async move {
2036 let payload: Value = serde_json::from_str(&event_json)
2037 .unwrap_or_else(|_| Value::String(event_json.clone()));
2038 let notification = serde_json::json!({
2039 "jsonrpc": "2.0",
2040 "method": "voice.event",
2041 "params": {
2042 "session_id": session_id,
2043 "event": payload,
2044 },
2045 });
2046 let Ok(text) = serde_json::to_string(¬ification) else {
2047 return;
2048 };
2049 let _ = channel
2050 .write
2051 .lock()
2052 .await
2053 .send(Message::Text(text.into()))
2054 .await;
2055 });
2056 }
2057
2058 fn send_binary(&self, frame: Vec<u8>) {
2059 use futures::SinkExt;
2060 let channel = self.channel.clone();
2061 tokio::spawn(async move {
2062 let _ = channel
2063 .write
2064 .lock()
2065 .await
2066 .send(Message::Binary(frame.into()))
2067 .await;
2068 });
2069 }
2070}
2071
2072/// Per-meeting fanout sink that ingests transcript text into a
2073/// session-scoped memgine using the `Arc<tokio::sync::Mutex<...>>`
2074/// wrapper, then forwards every event upstream untouched.
2075///
2076/// Lives here (not in `car-ffi-common`) because the engine handle uses
2077/// `tokio::sync::Mutex` per the "one-wrapper rule" — the FFI-common
2078/// `MeetingMemgineFanout` still uses `std::sync::Mutex` for the NAPI/
2079/// PyO3 bindings, which keep their sync wrappers. Each binding owns the
2080/// fanout that matches its lock primitive; the parsing/formatting logic
2081/// itself is shared via [`car_meeting::extract_transcript_for_ingest`].
2082///
2083/// `send` is called from the voice drain task and must be non-blocking,
2084/// so the lock acquisition is shipped to a `tokio::spawn`. Transcript
2085/// events are independent so reordering across spawned tasks is fine.
2086pub struct WsMemgineIngestSink {
2087 pub meeting_id: String,
2088 pub engine: Arc<Mutex<car_memgine::MemgineEngine>>,
2089 pub upstream: Arc<dyn car_voice::VoiceEventSink>,
2090}
2091
2092impl car_voice::VoiceEventSink for WsMemgineIngestSink {
2093 fn send(&self, voice_session_id: &str, event_json: String) {
2094 if let Ok(value) = serde_json::from_str::<Value>(&event_json) {
2095 if let Some((speaker, text)) = car_meeting::extract_transcript_for_ingest(
2096 &value,
2097 &self.meeting_id,
2098 voice_session_id,
2099 ) {
2100 let engine = self.engine.clone();
2101 tokio::spawn(async move {
2102 let mut guard = engine.lock().await;
2103 guard.ingest_conversation(&speaker, &text, chrono::Utc::now());
2104 });
2105 }
2106 }
2107 self.upstream.send(voice_session_id, event_json);
2108 }
2109}
2110
2111/// The last assistant turn a chat session produced, remembered so the *next*
2112/// user turn can be classified against it (the conversation-outcome signal).
2113/// The `trace_id`/`model_id` are carried straight from the `InferenceResult`
2114/// that produced the turn — never reconstructed from message order — so credit
2115/// always lands on the exact model/trace that generated the judged turn.
2116#[derive(Debug, Clone)]
2117pub struct LastChatTurn {
2118 /// The user request this assistant turn answered (for restatement similarity).
2119 pub user_text: String,
2120 pub assistant_text: String,
2121 pub trace_id: String,
2122 pub model_id: String,
2123}
2124
2125/// Per-client session.
2126pub struct ClientSession {
2127 pub client_id: String,
2128 pub runtime: Arc<Runtime>,
2129 pub channel: Arc<WsChannel>,
2130 pub host: Arc<crate::host::HostState>,
2131 /// Memgine handle. Wrapped in `tokio::sync::Mutex` so dispatcher
2132 /// handlers can hold the lock across `.await` points without
2133 /// risking poisoning. Migrated from `std::sync::Mutex` in the
2134 /// car-server-core extraction (U1) per the "one-wrapper rule".
2135 pub memgine: Arc<Mutex<car_memgine::MemgineEngine>>,
2136 /// Lazy browser session — first `browser.run` call launches Chromium,
2137 /// subsequent calls reuse it so element IDs resolve across invocations
2138 /// within the same WebSocket connection.
2139 pub browser: car_ffi_common::browser::BrowserSessionSlot,
2140 /// Per-connection auth state. Starts `false`; flips to `true`
2141 /// after a successful `session.auth` handshake. Always considered
2142 /// authenticated when `ServerState::auth_token` is unset (auth
2143 /// disabled). Closes Parslee-ai/car-releases#32.
2144 pub authenticated: Arc<std::sync::atomic::AtomicBool>,
2145 /// Negotiated daemon JSON-RPC protocol for this WebSocket connection.
2146 /// Starts at `0` (unnegotiated) and is set to
2147 /// `car_proto::PROTOCOL_VERSION` only after an exact-version
2148 /// `server.handshake`. The state is connection-scoped by construction, so
2149 /// a reconnect always has to negotiate again before using handshake-gated
2150 /// host/auth methods.
2151 pub negotiated_protocol_version: std::sync::atomic::AtomicU32,
2152 /// Capability names negotiated by this connection's authenticated
2153 /// `server.handshake`. Connection-scoped for the same reason as the wire
2154 /// version: a reconnect cannot inherit another socket's proof.
2155 pub negotiated_capabilities: Arc<std::sync::RwLock<std::collections::BTreeSet<String>>>,
2156 /// Bounded lifecycle state for inferences started on this exact socket.
2157 /// Keeping it per-session prevents an authenticated peer from probing IDs
2158 /// owned by another connection.
2159 pub inference_control: Arc<crate::inference_control::InferenceRegistry>,
2160 /// Host-management role (Parslee-ai/car#254). Starts `false`; flips
2161 /// to `true` only when the connection presents the per-launch host
2162 /// token via `session.auth { host_token }` (validated against
2163 /// `ServerState::host_token`). `authorize_run_access` requires this
2164 /// for cross-agent run-trace reads — being merely `host.subscribe`d
2165 /// is no longer sufficient, which is what closes the self-elevation
2166 /// hole. Cleared implicitly when the connection drops.
2167 pub is_host: std::sync::atomic::AtomicBool,
2168 /// Bound agent identity (#169). `Some(id)` once a lifecycle-agent
2169 /// child has called `session.auth { token, agent_id }` and the
2170 /// supervisor confirmed `agent_id` is supervised + token matches.
2171 /// Used by `agents.list` to surface which managed agents have
2172 /// actually attached vs. just being marked `Running` at the
2173 /// process level. Cleared at disconnect by `remove_session`.
2174 pub agent_id: Arc<tokio::sync::Mutex<Option<String>>>,
2175 /// Optional daemon-method allowlist bound by successful supervised-agent
2176 /// authentication. `None` is the backward-compatible unrestricted state;
2177 /// `Some(empty)` denies every application method. The dispatcher reads it
2178 /// once before any method-specific handler or notification interceptor.
2179 pub agent_method_allowlist: Arc<std::sync::RwLock<Option<std::collections::BTreeSet<String>>>>,
2180 /// Canonical schema digests for the narrow reverse-callback tools this
2181 /// client registered. Server-owned registry entries never land here.
2182 pub callback_tool_schema_digests:
2183 Arc<tokio::sync::RwLock<std::collections::HashMap<String, String>>>,
2184 /// Bound memory namespace (`session.auth { memory_namespace }`, #79/#80).
2185 ///
2186 /// The namespace identity has to survive the handshake, not just the
2187 /// lookup that binds `bound_memgine`. Without it the daemon knows *which
2188 /// graph* a session is using but not *what to call it*, so nothing can
2189 /// write that graph back to
2190 /// `~/.car/memory/memory-namespaces/<encoded-ns>.json` — which is
2191 /// why namespace memory was durable only until the next restart
2192 /// (car-releases#82). Paired with `bound_memgine` exactly as `agent_id` is.
2193 pub memory_namespace: tokio::sync::Mutex<Option<String>>,
2194 /// Bound persistent memgine (#170). `Some` after `session.auth`
2195 /// successfully attaches the connection to a daemon-owned
2196 /// per-agent memgine (paired with `agent_id`). Memory handlers
2197 /// route through [`ClientSession::effective_memgine`] which
2198 /// returns this when set, falling back to the ephemeral
2199 /// `memgine` field for browser/host/CLI connections.
2200 pub bound_memgine: tokio::sync::Mutex<Option<Arc<Mutex<car_memgine::MemgineEngine>>>>,
2201 /// The run currently bracketed on this connection (agent run
2202 /// tracing, U1). Set by `runs.start` **before** that handler
2203 /// responds, so the per-turn recorder (U2) always reads the
2204 /// `run_id` the bracket established — no race across the
2205 /// concurrently-spawned dispatch tasks (KTD3). Cleared by
2206 /// `runs.complete`. On disconnect with a still-set current run and
2207 /// no recorded terminal, the daemon marks it `Incomplete` (R5).
2208 pub current_run_id: tokio::sync::Mutex<Option<String>>,
2209 /// Serializes the authenticated run bracket on one WebSocket: start,
2210 /// proposal lifecycle, complete, and disconnect terminalization cannot
2211 /// interleave and produce ambiguous journal ordering.
2212 pub run_lifecycle_guard: Arc<tokio::sync::Mutex<()>>,
2213 /// Per-session permission-tier gate (survey §3.4.3/§5.2.5). Backs the
2214 /// `permission.*` JSON-RPC methods: the session holds the granted standing
2215 /// tier + risk classifier. Defaults to `SandboxEdit`, the tier
2216 /// `docs/websocket-protocol.md` publishes as the session default.
2217 ///
2218 /// `Arc`, not a bare lock, because this is also the gate
2219 /// [`crate::permission_gate::PermissionAdmissionGate`] enforces at
2220 /// proposal admission (Parslee-ai/car#890). ONE gate object behind both:
2221 /// a second instance would let the advisory `permission.evaluate` and the
2222 /// enforcement point disagree about the same action, which is precisely
2223 /// the bug class the admission gate was added to close.
2224 ///
2225 /// NOTE (kernel review C1): approval *records* do NOT live here. The
2226 /// gate's embedded ledger is deliberately unused in the daemon — every
2227 /// fingerprint-keyed HITL read/write goes through the SHARED
2228 /// [`ServerState::approval_ledger`] (journal-backed), so an approval
2229 /// recorded on one connection is visible to every other and survives
2230 /// restart. Tier state stays per-session; the approval store is
2231 /// daemon-wide.
2232 pub permission_gate: Arc<tokio::sync::RwLock<car_policy::PermissionGate>>,
2233 /// Connection-local fail-stop latch. A typed terminal tool callback sets
2234 /// it after its proposal has aborted and rolled back; the admission gate
2235 /// sharing this exact atomic rejects every later proposal on this socket.
2236 /// It is deliberately not durable and is discarded on reconnect.
2237 pub halted: Arc<AtomicBool>,
2238 /// Non-overlap guard for `evolution.run` on this session (kernel review
2239 /// S3): the dispatcher spawns requests concurrently even on one
2240 /// connection, and two interleaved cycles would double-dispatch the
2241 /// evolution mechanisms. A second `evolution.run` while one is in flight
2242 /// errs instead of overlapping.
2243 pub evolution_guard: crate::evolution::CycleGuard,
2244 /// Last assistant turn produced on this session's chat path, so the next
2245 /// user turn can be scored as a conversation outcome (Part B). `None` until
2246 /// the first chat reply; only set for declared-chat inference.
2247 pub last_chat_turn: tokio::sync::Mutex<Option<LastChatTurn>>,
2248 /// In-flight chat-infer count. Concurrent infers on one session have no
2249 /// well-defined turn adjacency (the dispatcher spawns a task per frame), so
2250 /// the outcome scorer only records a turn that began while the session was
2251 /// idle — otherwise it would score one infer's trace against an unrelated
2252 /// concurrent user turn (an active mislabel of routing stats). neo #1.
2253 pub chat_inflight: std::sync::atomic::AtomicUsize,
2254 /// Tenant identity bound at `session.auth { tenant_id }` (linus
2255 /// review C-4). Once bound, every tenant-scoped handler uses THIS
2256 /// identity: a per-request `tenant_id` param may restate it but a
2257 /// mismatch is rejected — an authenticated connection cannot hop
2258 /// into another tenant's namespace by editing request params.
2259 /// `None` (unbound) preserves the legacy per-request behavior.
2260 pub tenant: tokio::sync::Mutex<Option<String>>,
2261 /// Whether this connection has a live `tools.stream.event` forwarder
2262 /// task (C2). Set by `tools.stream.subscribe`; a concurrent second
2263 /// subscribe is a no-op instead of spawning a duplicate forwarder
2264 /// (which would double every notification). `Arc` so the forwarder
2265 /// task can RESET it on exit (write timeout on a stalled-but-
2266 /// recovering socket, broadcast closed) — a later re-subscribe then
2267 /// spawns a fresh forwarder rather than silently no-op'ing (Q2).
2268 pub tool_stream_subscribed: std::sync::Arc<std::sync::atomic::AtomicBool>,
2269}
2270
2271impl ClientSession {
2272 /// Returns the memgine handle the memory.* handlers should use:
2273 /// the bound per-agent memgine when this session attached via
2274 /// `session.auth { agent_id }` (#169 + #170), otherwise the
2275 /// ephemeral per-WS memgine. Cheap (one async lock + Arc clone).
2276 pub async fn effective_memgine(&self) -> Arc<Mutex<car_memgine::MemgineEngine>> {
2277 if let Some(eng) = self.bound_memgine.lock().await.as_ref() {
2278 return eng.clone();
2279 }
2280 self.memgine.clone()
2281 }
2282
2283 pub async fn bind_run_journal(&self, run_id: &str) -> Result<(), String> {
2284 self.runtime
2285 .event_log_handle()
2286 .lock()
2287 .await
2288 .bind_run(run_id, &self.client_id)
2289 }
2290
2291 pub async fn require_run_journal_binding(&self, run_id: &str) -> Result<(), String> {
2292 let log = self.runtime.event_log_handle();
2293 let log = log.lock().await;
2294 match log.active_run_binding() {
2295 Some((bound_run, bound_client, _))
2296 if bound_run == run_id && bound_client == self.client_id =>
2297 {
2298 Ok(())
2299 }
2300 Some((bound_run, bound_client, _)) => Err(format!(
2301 "active journal binding mismatch: expected run_id `{run_id}` / client_id `{}`, got `{bound_run}` / `{bound_client}`",
2302 self.client_id
2303 )),
2304 None => Err(format!(
2305 "active run `{run_id}` has no authenticated journal binding"
2306 )),
2307 }
2308 }
2309
2310 pub async fn clear_run_journal_binding(&self, run_id: &str) -> Result<(), String> {
2311 self.runtime
2312 .event_log_handle()
2313 .lock()
2314 .await
2315 .clear_run_binding(run_id, &self.client_id)
2316 }
2317
2318 pub async fn append_run_terminal_event(
2319 &self,
2320 ended: &car_proto::RunEnded,
2321 ) -> Result<(), String> {
2322 self.append_run_terminal_event_once(ended, &self.client_id)
2323 .await
2324 .map_err(|error| error.to_string())
2325 }
2326
2327 /// Append a terminal from an authenticated replacement socket while
2328 /// preserving the immutable client identity in the durable run trace.
2329 pub(crate) async fn append_resumed_run_terminal_event(
2330 &self,
2331 ended: &car_proto::RunEnded,
2332 durable_client_id: &str,
2333 ) -> Result<(), String> {
2334 self.append_run_terminal_event_once(ended, durable_client_id)
2335 .await
2336 .map_err(|error| error.to_string())
2337 }
2338
2339 pub async fn append_run_cancellation_requested_event(
2340 &self,
2341 requested: &car_proto::RunCancellationRequested,
2342 ) -> Result<(), String> {
2343 self.require_run_journal_binding(&requested.run_id).await?;
2344 let value = serde_json::to_value(requested).map_err(|error| error.to_string())?;
2345 let Value::Object(data) = value else {
2346 return Err("cancellation request did not serialize as an object".into());
2347 };
2348 self.runtime
2349 .event_log_handle()
2350 .lock()
2351 .await
2352 .append_critical_async(
2353 car_eventlog::EventKind::RunCancellationRequested,
2354 requested.action_id.as_deref(),
2355 None,
2356 data.into_iter().collect(),
2357 CRITICAL_JOURNAL_ACKNOWLEDGEMENT_TIMEOUT,
2358 )
2359 .await
2360 .map_err(|error| error.to_string())?;
2361 Ok(())
2362 }
2363
2364 pub async fn append_run_cancellation_result_event(
2365 &self,
2366 result: &car_proto::RunCancelResponse,
2367 ) -> Result<(), String> {
2368 self.require_run_journal_binding(&result.run_id).await?;
2369 let value = serde_json::to_value(result).map_err(|error| error.to_string())?;
2370 let Value::Object(data) = value else {
2371 return Err("cancellation result did not serialize as an object".into());
2372 };
2373 self.runtime
2374 .event_log_handle()
2375 .lock()
2376 .await
2377 .append_critical_async(
2378 car_eventlog::EventKind::RunCancellationResult,
2379 result.action_id.as_deref(),
2380 None,
2381 data.into_iter().collect(),
2382 CRITICAL_JOURNAL_ACKNOWLEDGEMENT_TIMEOUT,
2383 )
2384 .await
2385 .map_err(|error| error.to_string())?;
2386 Ok(())
2387 }
2388
2389 async fn append_run_terminal_event_once(
2390 &self,
2391 ended: &car_proto::RunEnded,
2392 durable_client_id: &str,
2393 ) -> Result<(), car_eventlog::CriticalAppendError> {
2394 let rejected = |reason| car_eventlog::CriticalAppendError::Rejected { reason };
2395 self.require_run_journal_binding(&ended.run_id)
2396 .await
2397 .map_err(rejected)?;
2398 if ended.client_id.as_deref() != Some(durable_client_id) {
2399 return Err(rejected(
2400 "RunEnded client_id does not match the durable run owner".into(),
2401 ));
2402 }
2403 let completion_digest = ended
2404 .completion_digest
2405 .as_deref()
2406 .ok_or_else(|| rejected("RunEnded is missing completion_digest".into()))?;
2407 let termination_kind = match &ended.termination {
2408 car_proto::RunTermination::Outcome { .. } => "outcome",
2409 car_proto::RunTermination::Incomplete => "incomplete",
2410 car_proto::RunTermination::Cancelled { .. } => "cancelled",
2411 };
2412 let log = self.runtime.event_log_handle();
2413 let mut log = log.lock().await;
2414 log.append_critical_async(
2415 car_eventlog::EventKind::RunCompleted,
2416 None,
2417 None,
2418 [
2419 (
2420 "termination_kind".to_string(),
2421 Value::from(termination_kind),
2422 ),
2423 (
2424 "completion_digest".to_string(),
2425 Value::from(completion_digest),
2426 ),
2427 (
2428 "termination".to_string(),
2429 serde_json::to_value(&ended.termination)
2430 .map_err(|error| rejected(error.to_string()))?,
2431 ),
2432 ]
2433 .into(),
2434 CRITICAL_JOURNAL_ACKNOWLEDGEMENT_TIMEOUT,
2435 )
2436 .await?;
2437 Ok(())
2438 }
2439
2440 pub async fn append_run_started_event(
2441 &self,
2442 started: &car_proto::RunStarted,
2443 ) -> Result<(), String> {
2444 self.require_run_journal_binding(&started.run_id).await?;
2445 if started.client_id.as_deref() != Some(self.client_id.as_str()) {
2446 return Err("RunStarted client_id does not match the live ClientSession".into());
2447 }
2448 let mut data = HashMap::from([
2449 (
2450 "agent_id".to_string(),
2451 Value::from(started.agent_id.clone()),
2452 ),
2453 ("intent".to_string(), Value::from(started.intent.clone())),
2454 (
2455 "started_at".to_string(),
2456 serde_json::to_value(started.started_at).unwrap_or(Value::Null),
2457 ),
2458 ]);
2459 if let Some(description) = &started.outcome_description {
2460 data.insert(
2461 "outcome_description".to_string(),
2462 Value::from(description.clone()),
2463 );
2464 }
2465 self.runtime
2466 .event_log_handle()
2467 .lock()
2468 .await
2469 .append_critical_async(
2470 car_eventlog::EventKind::RunStarted,
2471 None,
2472 None,
2473 data,
2474 CRITICAL_JOURNAL_ACKNOWLEDGEMENT_TIMEOUT,
2475 )
2476 .await
2477 .map_err(|error| error.to_string())?;
2478 Ok(())
2479 }
2480}
2481
2482/// Builder for constructing a [`ServerState`] with embedder-supplied
2483/// dependencies. Embedders (e.g. `tokhn-daemon`) use this to inject
2484/// their own memgine handle and other shared infrastructure; the
2485/// Approval-gate policy for high-risk WS methods.
2486///
2487/// Every method in `methods` must be acknowledged via
2488/// `host.resolve_approval` before the dispatcher will route the
2489/// request to its handler. The dispatcher waits up to `timeout` for
2490/// a resolution; on timeout (or any non-`approve` resolution) the
2491/// request fails with JSON-RPC error `-32003`.
2492///
2493/// Default: gate enabled, the host-automation surface
2494/// (`automation.run_applescript`, `automation.run_powershell`,
2495/// `automation.shortcuts.run`, `messages.send`, `mail.send`,
2496/// `calendar.create_event`, `calendar.update_event`, `calendar.delete_event`,
2497/// `vision.ocr`), 60-second timeout.
2498/// `car-server --no-approvals` (or embedders calling
2499/// [`ServerStateConfig::with_approval_gate`] with `enabled=false`)
2500/// turns it off — only appropriate when no untrusted caller can
2501/// reach the WS port.
2502#[derive(Debug, Clone)]
2503pub struct ApprovalGate {
2504 /// Master switch. When `false`, every method dispatches without
2505 /// raising an approval — the pre-2026-05 behaviour.
2506 pub enabled: bool,
2507 /// Methods that require approval. Match is by exact method-name
2508 /// string against the JSON-RPC `method` field.
2509 pub methods: std::collections::HashSet<String>,
2510 /// How long to wait for the user to resolve the approval before
2511 /// timing out and surfacing an error to the caller.
2512 pub timeout: std::time::Duration,
2513}
2514
2515impl Default for ApprovalGate {
2516 fn default() -> Self {
2517 let methods = [
2518 "automation.run_applescript",
2519 "automation.run_powershell",
2520 "automation.shortcuts.run",
2521 "messages.send",
2522 "mail.send",
2523 "calendar.create_event",
2524 "calendar.update_event",
2525 "calendar.delete_event",
2526 "vision.ocr",
2527 ]
2528 .iter()
2529 .map(|s| s.to_string())
2530 .collect();
2531 Self {
2532 enabled: true,
2533 methods,
2534 timeout: std::time::Duration::from_secs(60),
2535 }
2536 }
2537}
2538
2539impl ApprovalGate {
2540 /// Disable the gate entirely. Equivalent to passing
2541 /// `car-server --no-approvals`. Only appropriate when no
2542 /// untrusted caller can reach the WS port.
2543 pub fn disabled() -> Self {
2544 Self {
2545 enabled: false,
2546 methods: std::collections::HashSet::new(),
2547 timeout: std::time::Duration::from_secs(60),
2548 }
2549 }
2550
2551 /// `true` if this method must be acknowledged before dispatch.
2552 pub fn requires_approval(&self, method: &str) -> bool {
2553 self.enabled && self.methods.contains(method)
2554 }
2555}
2556
2557#[cfg(test)]
2558mod approval_gate_tests {
2559 use super::ApprovalGate;
2560
2561 #[test]
2562 fn local_calendar_mutations_are_high_risk_but_reads_are_not() {
2563 let gate = ApprovalGate::default();
2564 for method in [
2565 "calendar.create_event",
2566 "calendar.update_event",
2567 "calendar.delete_event",
2568 ] {
2569 assert!(
2570 gate.requires_approval(method),
2571 "{method} must require approval"
2572 );
2573 }
2574 assert!(!gate.requires_approval("calendar.list"));
2575 assert!(!gate.requires_approval("calendar.events"));
2576 }
2577}
2578
2579/// Mirror the authoritative user profile into memgine's always-present
2580/// Identity layer. Snapshots intentionally omit this node; callers invoke this
2581/// after loading `identity.json` instead, so only one durable copy exists.
2582pub(crate) fn mirror_identity_into_memgine(
2583 engine: &mut car_memgine::MemgineEngine,
2584 identity: &car_identity::AssistantIdentity,
2585) {
2586 let focus_areas: Vec<&str> = identity
2587 .focus_areas
2588 .iter()
2589 .map(|area| area.as_str())
2590 .collect();
2591 let apps: Vec<&str> = identity.apps.iter().map(String::as_str).collect();
2592 engine.ingest_identity_profile(
2593 identity.user_name.as_deref(),
2594 identity.role.as_deref(),
2595 &focus_areas,
2596 &apps,
2597 );
2598}
2599
2600/// standalone `car-server` binary uses [`ServerState::standalone`]
2601/// which calls `with_config` under the hood.
2602pub struct ServerStateConfig {
2603 pub journal_dir: PathBuf,
2604 /// Peer-message audit journal. `None` derives a path once from
2605 /// [`Self::journal_dir`] and the construction-time CAR state root. Tests
2606 /// and embedders can override it without mutating process-global `CAR_HOME`.
2607 pub peer_audit_journal: Option<PathBuf>,
2608 /// Optional pre-constructed memgine engine. When `None`, each
2609 /// `create_session` call builds a fresh engine; embedders that want
2610 /// to share a single engine across sessions can supply a clone of
2611 /// their `Arc<Mutex<MemgineEngine>>` here.
2612 pub shared_memgine: Option<Arc<Mutex<car_memgine::MemgineEngine>>>,
2613 /// Assistant identity/profile store. Production uses the CAR state root;
2614 /// tests and embedders inject a scratch root so identity writes and startup
2615 /// rehydration never touch the operator's real profile.
2616 pub identity_store: car_identity::IdentityStore,
2617 /// Optional pre-constructed inference engine.
2618 pub inference: Option<Arc<car_inference::InferenceEngine>>,
2619 /// Optional embedder-supplied A2A runtime. Used by the in-core
2620 /// `A2aDispatcher` to execute peer-driven proposals. When `None`,
2621 /// the dispatcher uses a fresh `Runtime` with `register_agent_basics`
2622 /// — peer agents see CAR's built-in tools and nothing else,
2623 /// matching the behaviour of the standalone `start_a2a_listener`.
2624 pub a2a_runtime: Option<Arc<car_engine::Runtime>>,
2625 /// Optional embedder-supplied A2A task store. When `None`,
2626 /// defaults to `InMemoryTaskStore`. tokhn-style embedders that
2627 /// want a polling-friendly persistent store plug it in here.
2628 pub a2a_store: Option<Arc<dyn car_a2a::TaskStore>>,
2629 /// Optional embedder-supplied agent card factory. When `None`,
2630 /// the dispatcher serves a card built from the A2A runtime's
2631 /// tool schemas at construction time, advertising its public URL
2632 /// as `ws://127.0.0.1:9100/` (the WS surface the dispatcher itself
2633 /// is reachable on).
2634 pub a2a_card_source: Option<Arc<car_a2a::AgentCardSource>>,
2635 /// Approval-gate policy. When `None`, the dispatcher uses
2636 /// [`ApprovalGate::default`] (gate ON, the macOS-automation
2637 /// surface gated, 60s timeout). Pass
2638 /// [`ApprovalGate::disabled`] to opt out — only appropriate
2639 /// when no untrusted caller can reach the WS port.
2640 pub approval_gate: Option<ApprovalGate>,
2641 /// Journal path for the daemon-wide shared HITL approval ledger
2642 /// ([`ServerState::approval_ledger`]). `None` = the default
2643 /// `~/.car/approvals.jsonl`. Tests and embedders that must not touch the
2644 /// user's real home point this at their own path.
2645 pub approval_journal: Option<PathBuf>,
2646 /// Directory for the daemon-wide trajectory store
2647 /// ([`ServerState::trajectory_store`]). `None` = the default
2648 /// `~/.car/trajectories/`. Tests and embedders that must not touch the
2649 /// user's real home point this at their own path, same as
2650 /// `approval_journal`.
2651 pub trajectory_dir: Option<PathBuf>,
2652 /// Deterministic durability fault seam used by direct lifecycle tests.
2653 pub run_store_failures: Option<crate::run_store::RunStoreFailureInjector>,
2654 /// Deterministic blocking-read seam used by run-liveness tests.
2655 #[doc(hidden)]
2656 pub run_store_summary_read_gate: Option<crate::run_store::RunStoreSummaryReadGate>,
2657 /// Test-only scheduling seam around durable run-summary publication.
2658 #[doc(hidden)]
2659 pub run_store_summary_write_gate: Option<crate::run_store::RunStoreSummaryWriteGate>,
2660 /// Deterministic blocking append seam used by run-liveness tests.
2661 #[doc(hidden)]
2662 pub run_store_append_gate: Option<crate::run_store::RunStoreAppendGate>,
2663 #[doc(hidden)]
2664 pub run_completion_fence_gate: Option<RunCompletionFenceGate>,
2665 /// Deterministic blocking durable-lookup seam used by run-liveness tests.
2666 #[doc(hidden)]
2667 pub run_store_lookup_gate: Option<crate::run_store::RunStoreLookupGate>,
2668 /// Deterministic first-use directory-entry durability seam for RunStore.
2669 pub run_store_private_path_failures: Option<car_secrets::PrivatePathDurabilityFailureInjector>,
2670 /// Deterministic critical-journal fault seam used by direct lifecycle tests.
2671 pub journal_failures: Option<car_eventlog::JournalFailureInjector>,
2672 /// Private self-heal detection and bounded-attempt ledger. The standalone daemon sets this to
2673 /// `<CAR_HOME>/selfheal/detections.jsonl`; embedders/tests can isolate it.
2674 pub selfheal_ledger: Option<PathBuf>,
2675 /// Detection cadence. Eligible default-on auto-fix runs from the same
2676 /// non-overlapping tick and is independently controlled by config.toml.
2677 pub selfheal_interval_secs: u64,
2678 /// Deterministic evidence seam for integration tests and embedders.
2679 #[doc(hidden)]
2680 pub selfheal_evidence: Option<crate::selfheal::SelfhealEvidence>,
2681 /// Bounded source-checkout probe. Embedders fail closed unless they supply
2682 /// candidates; the standalone daemon derives this from process-local paths.
2683 pub selfheal_source_probe: crate::selfheal::SelfhealSourceProbe,
2684 /// Deterministic seam for the checkout-built replay-verb eligibility probe.
2685 #[doc(hidden)]
2686 pub selfheal_replay_verb_probe: crate::selfheal::SelfhealReplayVerbProbe,
2687 /// Test-only override for the startup reconciliation acknowledgement bound.
2688 #[doc(hidden)]
2689 pub startup_reconciliation_acknowledgement_timeout: std::time::Duration,
2690 /// Test-only override for the bounded in-process resume lease.
2691 #[doc(hidden)]
2692 pub run_resume_lease: std::time::Duration,
2693}
2694
2695impl ServerStateConfig {
2696 /// Minimal config suitable for the standalone car-server binary:
2697 /// only the journal dir is required; everything else is lazily
2698 /// constructed at first use.
2699 pub fn new(journal_dir: PathBuf) -> Self {
2700 Self {
2701 journal_dir,
2702 peer_audit_journal: None,
2703 shared_memgine: None,
2704 identity_store: car_identity::IdentityStore::from_home(),
2705 inference: None,
2706 a2a_runtime: None,
2707 a2a_store: None,
2708 a2a_card_source: None,
2709 approval_gate: None,
2710 approval_journal: None,
2711 trajectory_dir: None,
2712 run_store_failures: None,
2713 run_store_summary_read_gate: None,
2714 run_store_summary_write_gate: None,
2715 run_store_append_gate: None,
2716 run_completion_fence_gate: None,
2717 run_store_lookup_gate: None,
2718 run_store_private_path_failures: None,
2719 journal_failures: None,
2720 selfheal_ledger: None,
2721 selfheal_interval_secs: crate::selfheal::DEFAULT_SELFHEAL_INTERVAL_SECS,
2722 selfheal_evidence: None,
2723 selfheal_source_probe: crate::selfheal::SelfhealSourceProbe::empty(),
2724 selfheal_replay_verb_probe: crate::selfheal::SelfhealReplayVerbProbe::default(),
2725 startup_reconciliation_acknowledgement_timeout:
2726 STARTUP_RECONCILIATION_ACKNOWLEDGEMENT_TIMEOUT,
2727 run_resume_lease: RUN_RESUME_LEASE,
2728 }
2729 }
2730
2731 pub fn with_shared_memgine(mut self, engine: Arc<Mutex<car_memgine::MemgineEngine>>) -> Self {
2732 self.shared_memgine = Some(engine);
2733 self
2734 }
2735
2736 /// Override the assistant identity/profile store.
2737 pub fn with_identity_store(mut self, store: car_identity::IdentityStore) -> Self {
2738 self.identity_store = store;
2739 self
2740 }
2741
2742 /// Override the peer-message audit journal path.
2743 pub fn with_peer_audit_journal(mut self, path: PathBuf) -> Self {
2744 self.peer_audit_journal = Some(path);
2745 self
2746 }
2747
2748 /// Override the shared approval-ledger journal path (default
2749 /// `~/.car/approvals.jsonl`).
2750 pub fn with_approval_journal(mut self, path: PathBuf) -> Self {
2751 self.approval_journal = Some(path);
2752 self
2753 }
2754
2755 pub fn with_trajectory_dir(mut self, dir: PathBuf) -> Self {
2756 self.trajectory_dir = Some(dir);
2757 self
2758 }
2759
2760 pub fn with_run_store_failures(
2761 mut self,
2762 failures: crate::run_store::RunStoreFailureInjector,
2763 ) -> Self {
2764 self.run_store_failures = Some(failures);
2765 self
2766 }
2767
2768 #[doc(hidden)]
2769 pub fn with_run_store_summary_read_gate(
2770 mut self,
2771 gate: crate::run_store::RunStoreSummaryReadGate,
2772 ) -> Self {
2773 self.run_store_summary_read_gate = Some(gate);
2774 self
2775 }
2776
2777 #[doc(hidden)]
2778 pub fn with_run_store_summary_write_gate(
2779 mut self,
2780 gate: crate::run_store::RunStoreSummaryWriteGate,
2781 ) -> Self {
2782 self.run_store_summary_write_gate = Some(gate);
2783 self
2784 }
2785
2786 #[doc(hidden)]
2787 pub fn with_run_store_append_gate(
2788 mut self,
2789 gate: crate::run_store::RunStoreAppendGate,
2790 ) -> Self {
2791 self.run_store_append_gate = Some(gate);
2792 self
2793 }
2794
2795 #[doc(hidden)]
2796 pub fn with_run_completion_fence_gate(mut self, gate: RunCompletionFenceGate) -> Self {
2797 self.run_completion_fence_gate = Some(gate);
2798 self
2799 }
2800
2801 #[doc(hidden)]
2802 pub fn with_run_resume_lease(mut self, lease: std::time::Duration) -> Self {
2803 self.run_resume_lease = lease;
2804 self
2805 }
2806
2807 #[doc(hidden)]
2808 pub fn with_run_store_lookup_gate(
2809 mut self,
2810 gate: crate::run_store::RunStoreLookupGate,
2811 ) -> Self {
2812 self.run_store_lookup_gate = Some(gate);
2813 self
2814 }
2815
2816 pub fn with_run_store_private_path_failures(
2817 mut self,
2818 failures: car_secrets::PrivatePathDurabilityFailureInjector,
2819 ) -> Self {
2820 self.run_store_private_path_failures = Some(failures);
2821 self
2822 }
2823
2824 pub fn with_journal_failures(mut self, failures: car_eventlog::JournalFailureInjector) -> Self {
2825 self.journal_failures = Some(failures);
2826 self
2827 }
2828
2829 pub fn with_selfheal_ledger(mut self, path: PathBuf) -> Self {
2830 self.selfheal_ledger = Some(path);
2831 self
2832 }
2833
2834 pub fn with_selfheal_interval_secs(mut self, interval_secs: u64) -> Self {
2835 self.selfheal_interval_secs = interval_secs.max(1);
2836 self
2837 }
2838
2839 #[doc(hidden)]
2840 pub fn with_selfheal_evidence(mut self, evidence: crate::selfheal::SelfhealEvidence) -> Self {
2841 self.selfheal_evidence = Some(evidence);
2842 self
2843 }
2844
2845 pub fn with_selfheal_source_probe(
2846 mut self,
2847 probe: crate::selfheal::SelfhealSourceProbe,
2848 ) -> Self {
2849 self.selfheal_source_probe = probe;
2850 self
2851 }
2852
2853 #[doc(hidden)]
2854 pub fn with_selfheal_replay_verb_probe(
2855 mut self,
2856 probe: crate::selfheal::SelfhealReplayVerbProbe,
2857 ) -> Self {
2858 self.selfheal_replay_verb_probe = probe;
2859 self
2860 }
2861
2862 #[doc(hidden)]
2863 pub fn with_startup_reconciliation_acknowledgement_timeout(
2864 mut self,
2865 timeout: std::time::Duration,
2866 ) -> Self {
2867 self.startup_reconciliation_acknowledgement_timeout = timeout;
2868 self
2869 }
2870
2871 pub fn with_inference(mut self, engine: Arc<car_inference::InferenceEngine>) -> Self {
2872 self.inference = Some(engine);
2873 self
2874 }
2875
2876 /// Plug in an embedder-supplied runtime for the A2A dispatcher.
2877 /// Use case: tokhn-daemon wants peers to see its OPA preflight
2878 /// tooling, not just CAR's `register_agent_basics` defaults.
2879 pub fn with_a2a_runtime(mut self, runtime: Arc<car_engine::Runtime>) -> Self {
2880 self.a2a_runtime = Some(runtime);
2881 self
2882 }
2883
2884 /// Plug in an embedder-supplied task store for the A2A
2885 /// dispatcher. Use case: tokhn's polling-friendly persistent
2886 /// store keyed by their session id.
2887 pub fn with_a2a_store(mut self, store: Arc<dyn car_a2a::TaskStore>) -> Self {
2888 self.a2a_store = Some(store);
2889 self
2890 }
2891
2892 /// Plug in an embedder-supplied agent card factory. The factory
2893 /// is invoked on every `agent/getAuthenticatedExtendedCard`
2894 /// dispatch, so embedders can reflect runtime tool changes.
2895 pub fn with_a2a_card_source(mut self, source: Arc<car_a2a::AgentCardSource>) -> Self {
2896 self.a2a_card_source = Some(source);
2897 self
2898 }
2899
2900 /// Override the approval-gate policy. Pass
2901 /// [`ApprovalGate::disabled`] to skip the gate entirely (only
2902 /// appropriate when no untrusted caller can reach the WS port);
2903 /// pass a customised [`ApprovalGate`] to add or remove methods
2904 /// or to change the timeout.
2905 pub fn with_approval_gate(mut self, gate: ApprovalGate) -> Self {
2906 self.approval_gate = Some(gate);
2907 self
2908 }
2909}
2910
2911/// The opted-in shared org-scope delivery subsystem + which org it serves. See
2912/// [`ServerState::org_sync`]. Held behind the same `std::sync::Mutex` as the
2913/// personal subsystem holder (init is serialized by the oplog's advisory lock);
2914/// the subsystem itself lives behind a `tokio::sync::Mutex`.
2915pub struct OrgSyncHolder {
2916 pub org: String,
2917 pub subsystem: Arc<tokio::sync::Mutex<crate::sync::SyncSubsystem>>,
2918}
2919
2920/// Which subsystem a write routes to. Pure decision, extracted so it is unit-
2921/// testable without a full daemon.
2922#[derive(Debug, PartialEq, Eq)]
2923enum SyncRoute {
2924 User,
2925 Org,
2926}
2927
2928/// Route by scope: a `Scope::Shared{org}` whose org matches the opted-in
2929/// `org_holder` goes to the org delivery subsystem; EVERYTHING else — `Personal`,
2930/// a different org, or no opted-in holder — goes to the personal subsystem. So
2931/// with `org_holder = None` (org-scope off) every scope routes `User`, byte-
2932/// identical to before.
2933fn route_for_scope(scope: &car_sync::Scope, org_holder: Option<&str>) -> SyncRoute {
2934 if let car_sync::Scope::Shared { org } = scope {
2935 if org_holder == Some(org.as_str()) {
2936 return SyncRoute::Org;
2937 }
2938 }
2939 SyncRoute::User
2940}
2941
2942/// Global server state shared across all connections.
2943pub struct ServerState {
2944 pub journal_dir: PathBuf,
2945 /// Resolved once when the state is constructed. Peer-message writes must
2946 /// use this path and never consult process-global `CAR_HOME` themselves.
2947 pub peer_audit_journal: PathBuf,
2948 pub sessions: Mutex<HashMap<String, Arc<ClientSession>>>,
2949 /// Serializes only session registry insertion/removal with the final
2950 /// liveness check + owner swap in `runs.resume`. This avoids holding the
2951 /// global session map while waiting on per-run durability and turns the
2952 /// replacement-present / old-owner-absent decision into one atomic fence.
2953 run_resume_liveness: Mutex<()>,
2954 run_resume_lease: std::time::Duration,
2955 run_completion_fence_gate: Option<RunCompletionFenceGate>,
2956 pub inference: std::sync::OnceLock<Arc<car_inference::InferenceEngine>>,
2957 pub host: Arc<crate::host::HostState>,
2958 /// When `Some`, `create_session` clones this handle into every new
2959 /// `ClientSession.memgine` — embedders that want a single shared
2960 /// memgine across all WS sessions set this. Standalone car-server
2961 /// leaves it `None`, which gives each session its own engine
2962 /// (preserving today's behavior).
2963 pub shared_memgine: Option<Arc<Mutex<car_memgine::MemgineEngine>>>,
2964 /// Source of truth for the assistant identity and user profile. Kept on the
2965 /// state so RPC tests/embedders can isolate it from the process environment.
2966 pub identity_store: car_identity::IdentityStore,
2967 /// Feedback doctor inputs resolved through [`ServerStateConfig`].
2968 pub(crate) feedback_diagnostics: crate::feedback::FeedbackDiagnostics,
2969 /// Daemon-wide trajectory store, cloned into **every** session's
2970 /// `Runtime` so execution traces persist to one place.
2971 ///
2972 /// Shared rather than per-session on purpose. A trajectory is evidence
2973 /// about how a tool behaves, and that evidence does not belong to the
2974 /// connection that happened to produce it — per-session stores would
2975 /// scatter the history across connections and reset it on every
2976 /// reconnect, leaving the derived success rates built from a handful of
2977 /// samples. This is the same reasoning that makes the approval ledger
2978 /// daemon-wide.
2979 ///
2980 /// Until now nothing called `Runtime::with_trajectory_store`, so
2981 /// `persist_trajectory` returned early on every execution and the store
2982 /// was dead code: `ToolFeedback::from_trajectories` existed with no data
2983 /// to read. Wiring it here is what makes the feedback loop real.
2984 pub trajectory_store: Arc<car_memgine::TrajectoryStore>,
2985 /// Daemon-wide self-healing detector/fixer + CAR_HOME ledger. Shared
2986 /// across sessions so `selfheal.run` and the cadence cannot overlap and
2987 /// every caller reads the same dismissals/detections.
2988 pub selfheal: Arc<crate::selfheal::SelfhealService>,
2989 /// The self-healing *repair* loop: reads a configured tracker, runs a coder
2990 /// session, gates the result on a multi-model panel, opens a pull request.
2991 ///
2992 /// Distinct from [`Self::selfheal`], which detects and never writes. Shared
2993 /// across sessions for the same reason: `heal.run` and the cadence must not
2994 /// overlap, and both must read the same claim ledger.
2995 pub heal: Arc<crate::coder::heal_service::HealService>,
2996 /// Process-wide voice session registry. Each
2997 /// `voice.transcribe_stream.start` call registers its own per-client
2998 /// [`WsVoiceEventSink`] so events route back to the originating WS
2999 /// connection only.
3000 pub voice_sessions: Arc<car_voice::VoiceSessionRegistry>,
3001 /// Process-wide meeting registry. Meeting ids are global; each
3002 /// meeting binds to the originating client's WS for upstream
3003 /// events but persists transcripts to the resolved
3004 /// `.car/meetings/<id>/` regardless of which client started it.
3005 pub meetings: Arc<car_meeting::MeetingRegistry>,
3006 /// Process-wide A2UI surface store. Agent-produced surfaces are
3007 /// visible to every host UI subscriber, independent of the
3008 /// WebSocket session that applied the update.
3009 pub a2ui: car_a2ui::A2uiSurfaceStore,
3010 /// In-process UI-improvement agent. Invoked from
3011 /// `handle_a2ui_render_report` with each inbound report; returned
3012 /// `Decision::Patch` envelopes are applied via the standard
3013 /// `apply_a2ui_envelope` path so all subscribers see the patch.
3014 /// `Arc` so the agent's interior `DashMap` state survives across
3015 /// handler calls even when `ServerState` is cheap-cloned.
3016 pub ui_agent: Arc<car_ui_agent::UIImprovementAgent>,
3017 /// Per-surface oscillation detector for the UI-improvement
3018 /// loop. Sits between the agent's `Decision::Patch` and the
3019 /// apply path so A→B→A patch cycles get cooled down without
3020 /// the agent itself having to track history. neo's review:
3021 /// "controllers use workqueue backoff; reconcilers stay
3022 /// stateless."
3023 pub ui_agent_oscillation: Arc<crate::ui_agent_loop::OscillationDetector>,
3024 /// Per-surface iteration budget. Backstop against runaway
3025 /// loops the oscillation detector misses — caps total agent-
3026 /// driven patches per surface at `DEFAULT_MAX_ITERATIONS`.
3027 pub ui_agent_budget: Arc<crate::ui_agent_loop::IterationBudget>,
3028 /// Process-wide concurrency gate for inference RPC handlers. Sized
3029 /// from host RAM at startup, overridable via
3030 /// [`crate::admission::ENV_MAX_CONCURRENT`]. Without this, N
3031 /// concurrent users multiply KV-cache and activation memory and
3032 /// take the host out (#114-adjacent: filed alongside the daemon
3033 /// always-on rework). The semaphore lives on `ServerState` so it
3034 /// is shared across every WebSocket session in the same process.
3035 pub admission: Arc<crate::admission::InferenceAdmission>,
3036 /// Server-side A2A continuation auth keyed by A2UI surface id.
3037 /// Kept out of `A2uiSurface.owner` so host renderers never see
3038 /// bearer/API-key material.
3039 pub a2ui_route_auth: Mutex<HashMap<String, A2aRouteAuth>>,
3040 /// Lifecycle-managed agents — declarative manifest at
3041 /// `~/.car/agents.json` driving spawn/restart/stop. Closes
3042 /// Parslee-ai/car-releases#27. Lazy-initialized so embedders that
3043 /// don't want process supervision don't pay the disk-touch cost
3044 /// at server start.
3045 pub supervisor: std::sync::OnceLock<Arc<car_registry::supervisor::Supervisor>>,
3046 /// Declarative (in-daemon) agent registry — a parallel store to the
3047 /// supervisor for agents the coder→agent loop builds. Lazy-initialized on
3048 /// first use (`~/.car/declagents.json`).
3049 pub declagents: std::sync::OnceLock<Arc<car_registry::declarative::DeclRegistry>>,
3050 /// Routing learning state — per-agent success stats + agent→agent edge
3051 /// weights for capability-similarity routing (`~/.car/routing.json`).
3052 /// Lazy-initialized on first use, sibling to [`Self::declagents`].
3053 pub routing: std::sync::OnceLock<Arc<car_registry::routing::RoutingStore>>,
3054 /// Multi-device sync + execution-lease subsystem (B6). Lazily opened on
3055 /// first `sync.*`/`lease.*` contact, rooted at `<journal_dir>/sync/` — a
3056 /// daemon-held [`car_sync::SyncSession`] over an `FsRelay` (the single-user
3057 /// two-device loopback works out of the box) plus an in-process
3058 /// linearizable [`car_sync::InMemoryLeaseCoordinator`]. **One subsystem per
3059 /// daemon = one device in the sync fleet.** The `std::sync::Mutex<Option>`
3060 /// serializes the fallible first-open (the oplog journal holds an exclusive
3061 /// advisory lock, so a double-open would fail) without holding the lock
3062 /// across the subsystem's async work — the subsystem itself lives behind a
3063 /// `tokio::sync::Mutex` inside the `Arc`. A distributed relay + a
3064 /// cross-daemon linearizable `LeaseCoordinator` backend is the documented
3065 /// B6 follow-up; the traits are its contract.
3066 pub sync: std::sync::Mutex<Option<Arc<tokio::sync::Mutex<crate::sync::SyncSubsystem>>>>,
3067 /// The SHARED org-scope delivery subsystem — a SECOND device oplog whose relay
3068 /// scope is `org:{orgId}`, so `Scope::Shared{org}` ops converge across an org's
3069 /// members. `None` unless org-scope is opted in (the same gated
3070 /// `PARSLEE_SYNC_ORG_SCOPE` branch that resolves the org key builds it);
3071 /// populated by `open_sync_subsystem` alongside the personal subsystem. Only
3072 /// the `Scope::Shared` write sites + `pump` route to it — the personal path
3073 /// (`sync_subsystem()` + all readers) is untouched. Slice 8a.
3074 pub org_sync: std::sync::Mutex<Option<OrgSyncHolder>>,
3075 /// Manifest path this daemon is *observing* but does NOT own.
3076 /// Set by `car-server` when boot-time supervisor construction
3077 /// fails with [`car_registry::supervisor::SupervisorError::AlreadyRunning`]
3078 /// — another car-server process on the host holds the exclusive
3079 /// lock on this manifest. In that state, `supervisor()` returns a
3080 /// clear "observe-only" error so mutation handlers refuse
3081 /// (preventing the duplicate-spawn bug from
3082 /// Parslee-ai/car-releases#44), while read-only handlers
3083 /// (`agents.list`, `agents.health`) fall back to
3084 /// [`car_registry::supervisor::Supervisor::list_from_manifest`] /
3085 /// [`car_registry::supervisor::Supervisor::health_from_manifest`]
3086 /// so operators can still inspect what the primary daemon is
3087 /// supervising.
3088 pub observer_manifest_path: std::sync::OnceLock<PathBuf>,
3089 /// In-core A2A dispatcher — embedders that consume `car-server-core`
3090 /// get A2A reachability "for free" without standing up a separate
3091 /// HTTP listener. Closes Parslee-ai/car-releases#28. Lazy-init so
3092 /// the embedder can override the runtime / task store / agent card
3093 /// via [`ServerStateConfig::with_a2a_runtime`] etc. before the
3094 /// first dispatch.
3095 pub a2a_dispatcher: std::sync::OnceLock<Arc<car_a2a::A2aDispatcher>>,
3096 /// WS clients subscribed to A2UI envelope events. After every
3097 /// successful `a2ui.apply` / `a2ui.ingest`, the resulting
3098 /// `A2uiApplyResult` is broadcast to every subscriber as an
3099 /// `a2ui.event` JSON-RPC notification. Closes
3100 /// Parslee-ai/car-releases#29. Subscribers register via the
3101 /// `a2ui/subscribe` method and are auto-cleaned on WS disconnect.
3102 pub a2ui_subscribers: Mutex<HashMap<String, Arc<WsChannel>>>,
3103 /// Per-launch auth token. When `Some`, the WS dispatcher rejects
3104 /// non-auth methods on unauthenticated sessions until the client
3105 /// calls `session.auth` with the matching value. When `None`,
3106 /// auth is disabled and every connection works as before. Set
3107 /// at startup by `car-server` unless `--no-auth` is passed
3108 /// (default flipped 2026-05); embedders that want to enable
3109 /// auth call [`ServerState::install_auth_token`]. Closes
3110 /// Parslee-ai/car-releases#32.
3111 /// Shared MCP tool executor backing remote connectors. One per
3112 /// process: every WS session's runtime executor is a
3113 /// [`car_engine::McpToolExecutor::share_with_fallback`] view over
3114 /// this, so connector tools route here while non-connector tools
3115 /// fall back to the per-session [`WsToolExecutor`]. Connectors
3116 /// (their live sessions + routes) live on this shared instance, so
3117 /// a connector enabled in one WS session is reachable from all.
3118 pub mcp_executor: Arc<car_engine::McpToolExecutor>,
3119 /// Remote MCP connector manager (`~/.car/connectors.json`).
3120 /// Lazy-initialized over [`mcp_executor`](Self::mcp_executor) so
3121 /// embedders that never touch connectors pay no disk cost. See
3122 /// [`ServerState::connectors`].
3123 pub connectors: std::sync::OnceLock<Arc<car_connectors::ConnectorManager>>,
3124 /// One-shot guard so persisted connectors are loaded + dialed at
3125 /// most once per process (see [`ServerState::ensure_connectors_loaded`]).
3126 connectors_loaded: std::sync::atomic::AtomicBool,
3127 /// Runtime approval-transport channel supervisor (Units 1/2/3). Lazy-
3128 /// initialized at daemon boot by `spawn_channel_pollers`, which records the
3129 /// channels it spawned at startup. The host-gated `messaging.config.set`
3130 /// handler reaches it via `ServerState` to spawn a channel's watcher the
3131 /// instant the user enables it — no restart (U1). Also holds the per-channel
3132 /// liveness `messaging.status` reads (U2) and the send path writes (U3).
3133 /// `None` for embedders that never boot the channel pollers.
3134 pub channel_supervisor: std::sync::OnceLock<Arc<crate::channel_supervisor::ChannelSupervisor>>,
3135 /// Daemon-owned operations which must outlive the WebSocket request that
3136 /// started them. The tasks hold no [`ClientSession`] or [`WsChannel`]; only
3137 /// their oneshot response waiters are connection-scoped.
3138 ///
3139 /// Two kinds of work live here. Credential-backed auth operations (attempt
3140 /// completion and ordinary auth mutations), so a socket close can release
3141 /// its sink immediately without cancelling a coordinator/keychain operation
3142 /// halfway through its durable result. And `coder.start`, which registers a
3143 /// session and provisions a worktree *before* a multi-minute contract
3144 /// derivation: once that side effect exists on disk, the run's lifetime
3145 /// must not be owned by the board that asked for it.
3146 pub durable_tasks: Mutex<tokio::task::JoinSet<String>>,
3147 /// Per-daemon incarnation token persisted into redeeming auth leases.
3148 /// Awaiting-callback reservations intentionally have no daemon owner and
3149 /// survive a restart; a redeeming lease owned by a different incarnation
3150 /// is an orphan and is terminally failed by completion-status reconciliation.
3151 pub auth_completion_owner_id: String,
3152 pub auth_token: std::sync::OnceLock<String>,
3153 /// Per-launch **host** token — a credential distinct from
3154 /// `auth_token`, granting the host-management role (host-class
3155 /// reads like cross-agent run traces). Critically it is **never**
3156 /// served over `GET /auth-token`: a session becomes host-role only
3157 /// by presenting this via `session.auth { host_token }`, and the
3158 /// only way to obtain it is reading the `0600` `host-token` file,
3159 /// which a different local user cannot. This is what stops any
3160 /// authenticated local client from self-elevating to host and
3161 /// reading every agent's run traces (Parslee-ai/car#254). When
3162 /// `None`, host-role can't be granted (no host reads).
3163 pub host_token: std::sync::OnceLock<String>,
3164 /// Mobile Parslee Core runtime URL this daemon can hand to consumer host apps.
3165 /// Set by `car-server` at startup from the daemon bind address or an
3166 /// explicit public URL override. The token remains the per-launch auth
3167 /// token; this field only names the WebSocket endpoint.
3168 pub mobile_runtime_url: std::sync::OnceLock<String>,
3169 /// Explicit public mobile runtime URL to register after an authorized
3170 /// credential action succeeds. Absent for the default loopback URL, which
3171 /// is discoverable locally but must not be advertised to Parslee cloud.
3172 pub mobile_registration_url: std::sync::OnceLock<String>,
3173 /// Parslee cloud identity activated by an explicit credential-backed
3174 /// request after `car auth login` has been completed.
3175 pub parslee_session: std::sync::OnceLock<crate::parslee_auth::ParsleeSession>,
3176 /// `agent_id -> client_id` map of currently-attached lifecycle
3177 /// agents (#169). Populated by the `session.auth` handler when a
3178 /// supervised child presents its `agent_id` + per-agent token;
3179 /// drained on disconnect by `remove_session`. Single-claim:
3180 /// a second connection presenting the same `agent_id` is
3181 /// rejected so the daemon-side per-agent state stays unambiguous.
3182 pub attached_agents: Mutex<HashMap<String, String>>,
3183 /// `agent_id -> persistent memgine` map (#170). Lazy-loaded on
3184 /// first connection per id from `~/.car/memory/agents/<id>.jsonl`,
3185 /// retained across daemon restart, surviving any single
3186 /// disconnect/reconnect of the supervised child. Connections
3187 /// that auth without an `agent_id` (browser, host, ad-hoc CLI)
3188 /// keep the per-WS ephemeral memgine on `ClientSession.memgine`
3189 /// — no behaviour change.
3190 pub agent_memgines: Mutex<HashMap<String, Arc<Mutex<car_memgine::MemgineEngine>>>>,
3191 /// Daemon-owned memgines keyed by a host-declared **memory namespace**
3192 /// (`session.auth { memory_namespace }`), lazily loaded from
3193 /// `~/.car/memory/memory-namespaces/<encoded-ns>.json` — the filename is a
3194 /// percent-encoding of the namespace, injective so two namespaces can never
3195 /// share a snapshot file (#891).
3196 ///
3197 /// A separate axis from `agent_memgines`, deliberately: one agent may work
3198 /// across several namespaces, and two hosts may share a namespace without
3199 /// sharing an identity (Parslee-ai/car-releases#79). A host that binds none
3200 /// keeps today's behaviour — the daemon's shared graph — so the MCP-to-WS
3201 /// shared knowledge base is untouched.
3202 pub namespace_memgines: Mutex<HashMap<String, Arc<Mutex<car_memgine::MemgineEngine>>>>,
3203 /// Live coder sessions keyed by `session_id` (built-in coding agent).
3204 /// Process-wide so a session outlives the WS connection that started
3205 /// it; terminal sessions stay listed for `coder.get`/`coder.list`
3206 /// history until daemon restart (snapshots persist under
3207 /// `~/.car/coder/`).
3208 pub coder_sessions: Mutex<crate::coder::rpc::CoderSessionMap>,
3209 /// Monotonic base for [`Self::coder_disk_gc_at`], taken at construction.
3210 pub coder_disk_gc_base: std::time::Instant,
3211 /// Seconds since [`Self::coder_disk_gc_base`] at the last coder state-dir
3212 /// sweep. Rate-limits the disk GC amortized onto `coder.start` (car#1339)
3213 /// so a burst of starts re-reads the directory once, not once each.
3214 ///
3215 /// Monotonic rather than Unix seconds because this is an INTERVAL: a wall
3216 /// clock that steps backwards would stamp a future value and suppress every
3217 /// later sweep for the daemon's lifetime, which is the bug this closes,
3218 /// reintroduced through the clock.
3219 ///
3220 /// On the state rather than a process-global static so each `ServerState` —
3221 /// every test builds its own — carries its own stamp, and one test's sweep
3222 /// cannot suppress another's.
3223 pub coder_disk_gc_at: AtomicU64,
3224 /// Live `coder.event` subscribers keyed by `(session_id, client_id)`
3225 /// — explicit fanout, same shape as `run_subscribers`. Lock order:
3226 /// the session's event buffer → this map; never the reverse.
3227 pub coder_subscribers: Mutex<HashMap<(String, String), Arc<WsChannel>>>,
3228 /// `coder.watch` board subscribers keyed by `client_id` — one
3229 /// subscription per connection covering EVERY session, so a board learns
3230 /// about runs started by any other client (`car code`, CarHost, milo)
3231 /// without polling. Fanned to as `coder.session_changed`. Lock order:
3232 /// a session's event buffer → `coder_subscribers` → this map; never the
3233 /// reverse.
3234 ///
3235 /// The value carries a **registration generation** alongside the channel.
3236 /// A board re-registers periodically (its `coder.watch` is idempotent and
3237 /// returns a fresh snapshot), and the fanout sheds a wedged watcher *after*
3238 /// releasing this lock — so a shed that removed by `client_id` alone would
3239 /// delete a registration made in that window and silently unwatch a healthy
3240 /// board. The shed compares the generation and removes only the entry it
3241 /// actually timed out on.
3242 pub coder_watchers: Mutex<HashMap<String, (u64, Arc<WsChannel>)>>,
3243 /// The one long-lived task that fans `coder.session_changed` out to
3244 /// [`coder_watchers`](Self::coder_watchers), lazily started on the first
3245 /// notification. Deliberately a single drain rather than a task per event:
3246 /// a running session emits on every tool call, and spawn-per-event against
3247 /// a half-open board socket accumulated blocked tasks without bound, each
3248 /// pinning the socket's write half past teardown (car#209's shape on a new
3249 /// path). Daemon-owned, so it is correctly absent from any connection's
3250 /// `conn_tasks`; it exits when this `ServerState` drops.
3251 pub(crate) coder_watch_notify: std::sync::OnceLock<mpsc::UnboundedSender<String>>,
3252 /// Live `coder.discuss.*` runtimes keyed by `discussion_id`. Model history
3253 /// is checkpointed separately; recovery rebuilds this connection-owned
3254 /// runtime after validating the saved repository and principal binding.
3255 pub coder_discussions: Mutex<crate::coder::discuss::DiscussionMap>,
3256 /// Serializes recovery admission, without holding the live registry lock
3257 /// while a repository runtime is rebuilt.
3258 pub(crate) coder_discussion_recovery: Mutex<()>,
3259 /// Open-discussion slots, one permit per allowed discussion. A permit is
3260 /// taken BEFORE `coder.discuss.start` does any of its expensive async work
3261 /// and lives inside the [`DiscussionEntry`](crate::coder::discuss::DiscussionEntry)
3262 /// it admitted, so it comes back when the discussion is dropped. Counting
3263 /// the map instead was a TOCTOU check — the count was read, the lock
3264 /// released, and two awaits (substrate bind + runtime build) ran before the
3265 /// insert, so N pipelined starts all passed a cap of N=8.
3266 pub(crate) coder_discussion_slots: Arc<tokio::sync::Semaphore>,
3267 /// In-flight `agents.chat` sessions keyed by `session_id`. See
3268 /// [`ChatSession`] for shape. Populated by `agents.chat`,
3269 /// cleared on terminal `agent.chat.event` or
3270 /// `agents.chat.cancel`. Disconnect cleanup happens in
3271 /// `remove_session` — any in-flight session bound to either the
3272 /// disconnecting host or agent client is dropped so subsequent
3273 /// stray notifications from a respawned agent fall on the floor
3274 /// rather than racing into a stale stream.
3275 pub chat_sessions: Mutex<HashMap<String, ChatSession>>,
3276 /// Per-recipient channel guards for peer messaging (`agents.message`).
3277 ///
3278 /// Keyed by recipient agent id. Bounds the *channel* rather than the
3279 /// sender's authority: size, identical repeats, per-sender rate, and queue
3280 /// depth. Two agents answering each other form a loop that no policy rule
3281 /// catches, because neither is misbehaving — this is what terminates it.
3282 ///
3283 /// Entries are dropped in `remove_session` when the agent detaches, so a
3284 /// restarted agent starts with a clean budget rather than inheriting the
3285 /// rate history of the process that used to own its name.
3286 pub peer_guards: Mutex<HashMap<String, car_peers::DeliveryGuard>>,
3287 /// MCP protocol sessions that currently own a peer identity.
3288 ///
3289 /// The HTTP transport mints these on `initialize` and removes them on
3290 /// protocol DELETE or idle expiry. Receive-capable sessions hold a bounded
3291 /// polled inbox; CAR-spawned batch CLI sessions are recorded only long
3292 /// enough to attribute their sends and never appear as recipients.
3293 pub(crate) mcp_peer_sessions: Mutex<HashMap<String, crate::peers::McpPeerSession>>,
3294 /// Earned standing per sending principal — a peer key inbound, a local
3295 /// agent id outbound.
3296 ///
3297 /// **Deliberately not reaped on detach**, unlike [`Self::peer_guards`] one
3298 /// line above. That hook exists so a respawned agent does not inherit a
3299 /// dead process's rate and dedupe budget, which is *channel* state.
3300 /// Standing is the opposite kind: clearing it on disconnect would let a
3301 /// degraded agent wipe its record by reattaching, which is a cheaper
3302 /// erasure than the renaming this design already refuses to allow.
3303 /// Bounded by TTL and map cap instead, pruned on write.
3304 pub peer_standing: Mutex<HashMap<String, crate::peers::PeerStanding>>,
3305 /// Peer messages held for operator approval, oldest first.
3306 ///
3307 /// A message is held when the sender's `read_only` posture is
3308 /// `require_approval`. Bounded at [`car_peers::HOLD_CAP`]; past that the
3309 /// oldest is dropped, so a stream of held messages cannot grow without
3310 /// limit. Kept flat rather than per-recipient because approval is an
3311 /// operator action over the whole queue, not a per-agent one.
3312 ///
3313 /// Deliberately in memory only: a held message is a live decision awaiting a
3314 /// human, and one that outlived a daemon restart would be delivered into a
3315 /// world that had moved on.
3316 pub held_peer_messages: Mutex<std::collections::VecDeque<crate::peers::HeldPeerMessage>>,
3317 /// Live view of CAR daemons discovered on the local network.
3318 ///
3319 /// `None` when mDNS could not start (no multicast, a locked-down sandbox) —
3320 /// LAN discovery is then simply absent rather than the daemon failing to
3321 /// boot. A network with no peers and a network CAR cannot browse look the
3322 /// same to a caller, which is why the status surface reports which it is.
3323 pub lan_discovery: std::sync::Mutex<Option<car_a2a::lan::LanDirectory>>,
3324 /// This daemon's ed25519 peer identity, and the keys it accepts.
3325 ///
3326 /// The identity signs outbound cross-host requests; the trust set decides
3327 /// which inbound signers are CAR. Both are `None` until the A2A surface
3328 /// starts, because neither is meaningful without a network surface to
3329 /// authenticate.
3330 pub peer_identity: std::sync::Mutex<Option<Arc<car_a2a::peer_auth::PeerIdentity>>>,
3331 pub peer_trust: car_a2a::peer_auth::PeerTrust,
3332 /// In-process collectors for `agent.chat` streams that have **no** host UI
3333 /// to forward to — currently the A2A conversational bridge, which reverse-
3334 /// calls `agent.chat` on a host session and aggregates the streamed deltas
3335 /// into a single reply. Keyed by `session_id`. When a collector exists for a
3336 /// session, `try_forward_agent_chat_event` feeds chunks here instead of
3337 /// forwarding to a host channel. The collecting task owns the entry's
3338 /// lifetime (inserts before the reverse-call, removes when done/timed out).
3339 pub chat_collectors: Mutex<HashMap<String, ChatCollector>>,
3340 /// Standing deterministic chat goals keyed by `session_id`. This is a
3341 /// host-facing status/control registry, separate from ephemeral
3342 /// `chat_sessions` routing so a goal can be inspected after a turn finishes.
3343 pub chat_goals: Mutex<HashMap<String, ChatGoalState>>,
3344 /// Agent runs keyed by `run_id` (agent run tracing, U1). Process-
3345 /// wide (not per-session) so a run's record outlives the WS
3346 /// connection that produced it — the durable, connection-
3347 /// independent boundary `client_id` cannot be (R1). Populated by
3348 /// `runs.start`, made terminal by `runs.complete`, and swept to
3349 /// `Incomplete` on a mid-run disconnect past the grace window
3350 /// (R5). U2/U3 build the per-turn recorder and disk store on top
3351 /// of this registry.
3352 pub runs: Mutex<HashMap<String, RunMeta>>,
3353 /// Per-run serialization for trace append/fsync and terminal persistence.
3354 /// Filesystem work never holds the global [`runs`](Self::runs) lock.
3355 run_durability_locks: Mutex<HashMap<String, std::sync::Weak<Mutex<()>>>>,
3356 /// Live `runs.trace.event` subscribers keyed by `(run_id,
3357 /// host_client_id)` (agent run tracing, U4). Each value is a
3358 /// [`crate::host::RunTraceSubscriber`] — the producer side of a
3359 /// bounded channel whose dedicated drain task writes frames to that
3360 /// connection's WS. Two CarHost windows on one run register two
3361 /// distinct entries (explicit fanout — the built-in notification
3362 /// registry is single-subscriber-per-method).
3363 ///
3364 /// **Lock contract (invariant #1):** `runs.subscribe` snapshots the
3365 /// run's turns AND inserts the subscriber while holding the
3366 /// [`runs`](Self::runs) lock; the recorder (`record_run_turns`),
3367 /// `start_run`, `complete_run`, and `mark_run_incomplete` append to
3368 /// `runs` and push to `run_subscribers` while holding the SAME
3369 /// `runs` lock — so snapshot/register and append/notify are
3370 /// serialized. No turn appended in the snapshot/register window is
3371 /// dropped (gap) or double-delivered (dup). The lock order is always
3372 /// `runs` → `run_subscribers`; never the reverse.
3373 pub run_subscribers: Mutex<HashMap<(String, String), crate::host::RunTraceSubscriber>>,
3374 /// Every browser the drawer can reach (`browser.view.*`), keyed by
3375 /// conversation/agent-session — plus the one standing user session every
3376 /// conversation without an agent-attached browser shares. Owns the
3377 /// per-view snapshot/cursor/subscriber fanout; see
3378 /// [`crate::browser_view`].
3379 pub browser_views: Arc<crate::browser_view::BrowserViewRegistry>,
3380 /// Disk-backed run-trace store (agent run tracing, U3). Source of
3381 /// truth for REPLAY (U5) — persists each run's `RunStarted`, turns,
3382 /// and terminal record as JSONL under `~/.car/runs/{agent_id}/` so a
3383 /// run survives daemon restarts (R4), bounded by retention/GC (R6)
3384 /// and protected at rest with `0600`/`0700` perms + backup exclusion
3385 /// (R14). The in-memory [`runs`](Self::runs) buffer stays the source
3386 /// for the LIVE stream (U4); this store mirrors what was recorded.
3387 /// Derived from `journal_dir` at construction (sibling `runs/`).
3388 pub run_store: crate::run_store::RunStore,
3389 journal_failures: Option<car_eventlog::JournalFailureInjector>,
3390 /// Bound MCP HTTP-streamable URL (e.g.
3391 /// `"http://127.0.0.1:9102/mcp"`) — `car-server` installs this
3392 /// after binding the listener. Used by the
3393 /// `agents.invoke_external` handler to default
3394 /// `InvokeOptions.mcp_endpoint` so external agents
3395 /// (Claude Code today) load the daemon's CAR namespace via
3396 /// `--mcp-config` automatically. `None` when MCP isn't bound
3397 /// (e.g. `--mcp-bind disabled`).
3398 pub mcp_url: std::sync::OnceLock<String>,
3399 /// Approval gate for high-risk WS methods (audit 2026-05). The
3400 /// gate intercepts `automation.run_applescript`,
3401 /// `automation.shortcuts.run`, `messages.send`, `mail.send`, the three
3402 /// mutating `calendar.*` methods, and `vision.ocr` before they dispatch, raises a
3403 /// `host.create_approval` for the user to act on, and waits
3404 /// (with a timeout) for `host.resolve_approval`. Approve →
3405 /// dispatch continues; deny / timeout → JSON-RPC error code
3406 /// `-32003`. The set of gated methods and the wait timeout are
3407 /// embedder-overridable via
3408 /// [`ServerStateConfig::with_approval_gate`].
3409 pub approval_gate: ApprovalGate,
3410 /// Out-of-process supervision (proposal item 5). Holds the subscribed
3411 /// supervisors and the intents parked on their verdicts; the matching
3412 /// [`crate::supervision::SupervisionGate`] is registered as an
3413 /// `AdmissionGate` on every session runtime alongside
3414 /// `StaticVerificationGate`.
3415 ///
3416 /// Daemon-wide rather than per-session, deliberately: a supervisor connects
3417 /// on its OWN WebSocket and supervises proposals submitted on OTHER
3418 /// sessions. A per-session registry would only ever see the supervisor's
3419 /// own (empty) traffic.
3420 ///
3421 /// Costs nothing until someone subscribes — the gate returns `Allow`
3422 /// without building an intent while the subscriber set is empty.
3423 pub supervision: Arc<crate::supervision::SupervisionRegistry>,
3424 /// The daemon-wide shared HITL approval ledger (kernel review C1) —
3425 /// journal-backed at `~/.car/approvals.jsonl` (override via
3426 /// [`ServerStateConfig::with_approval_journal`]). ONE ledger for all
3427 /// sessions: `permission.approve`/`reject` on any connection records
3428 /// here, and every ledger consumer (`permission.evaluate`/`pending`,
3429 /// `evolution.run`, `cascade.run`, `skill.enforce_deployment`/
3430 /// `ingest_governed`) reads here — so an approval granted by a host
3431 /// connection is visible to the agent connection that surfaced it and
3432 /// survives daemon restart. Per-session gates keep only tier/classifier
3433 /// state (see [`ClientSession::permission_gate`]). Falls back to an
3434 /// in-memory ledger (loudly warned, approvals NOT restart-durable) only
3435 /// when the journal is unopenable.
3436 pub approval_ledger: Arc<tokio::sync::RwLock<car_policy::ApprovalLedger>>,
3437 /// The in-process harness evaluator, when the binary installed one — the
3438 /// thing that lets `evolution.run` grade a harness candidate ITSELF
3439 /// instead of waiting for an operator to run `car-bench-harness` twice and
3440 /// hand both files back in.
3441 ///
3442 /// `None` on a build that did not install one (every embedder, and any
3443 /// binary other than `car-server`): `evolution.run` with `harness_measure`
3444 /// then ERRS rather than quietly falling back to HITL, because an opt-in
3445 /// that silently does nothing would report an unattended cycle that never
3446 /// measured anything.
3447 ///
3448 /// A `std::sync::RwLock` on purpose — no caller holds it across an await;
3449 /// they clone the `Arc` out and drop the guard.
3450 pub harness_measurer: std::sync::RwLock<Option<Arc<dyn crate::evolution::HarnessMeasurer>>>,
3451 /// A2A-runtime / store / card factory carried over from the
3452 /// embedder's [`ServerStateConfig`]. Consumed lazily on first
3453 /// `a2a_dispatcher()` call so embedders can construct
3454 /// `ServerState` without paying the runtime spin-up cost when
3455 /// they don't actually use the A2A surface.
3456 pub(crate) a2a_runtime: std::sync::Mutex<Option<Arc<car_engine::Runtime>>>,
3457 pub(crate) a2a_store: std::sync::Mutex<Option<Arc<dyn car_a2a::TaskStore>>>,
3458 pub(crate) a2a_card_source: std::sync::Mutex<Option<Arc<car_a2a::AgentCardSource>>>,
3459}
3460
3461/// Default location of the shared approval-ledger journal: `approvals.jsonl`
3462/// directly under the CAR state root, so it follows `CAR_HOME` rather than
3463/// pinning itself to `~/.car` (which is still where it lands when `CAR_HOME` is
3464/// unset — HOME, or USERPROFILE on Windows). Creates the root best-effort;
3465/// `None` when neither `CAR_HOME` nor a home directory is resolvable.
3466fn default_approval_journal_path() -> Option<PathBuf> {
3467 let dir = car_home::root()?;
3468 let _ = std::fs::create_dir_all(&dir);
3469 Some(dir.join("approvals.jsonl"))
3470}
3471
3472/// Resolve the peer audit journal once, while constructing [`ServerState`].
3473///
3474/// The standalone daemon's default journal directory is `<CAR_HOME>/journals`,
3475/// so preserve the established `<CAR_HOME>/peer-messages.jsonl` location in
3476/// that configuration. A custom journal directory — including every hermetic
3477/// test tempdir — owns its peer audit journal directly. No writer re-reads
3478/// `CAR_HOME`, so process-global state cannot redirect a configured test state.
3479fn default_peer_audit_journal_path(journal_dir: &Path) -> PathBuf {
3480 if let Some(car_dir) = car_home::root() {
3481 if journal_dir == car_dir.join("journals") {
3482 return car_dir.join("peer-messages.jsonl");
3483 }
3484 }
3485 journal_dir.join("peer-messages.jsonl")
3486}
3487
3488/// The daemon's state root: `CAR_HOME` when set, otherwise `~/.car` (HOME, or
3489/// USERPROFILE on Windows).
3490///
3491/// One convention, used by [`default_approval_journal_path`], the `messaging.*`
3492/// handlers and now the project policy loader — a second one would mean an
3493/// operator's rules living somewhere their pairing config does not. Going
3494/// through `car_home` is what keeps that true once an operator relocates the
3495/// root: everything moves together or nothing does.
3496fn car_home_dir() -> Option<PathBuf> {
3497 car_home::root()
3498}
3499
3500/// Register the declarative rules in `<car_dir>/policies/*.toml` on `runtime`.
3501///
3502/// **A malformed file is fatal, on purpose. Do not downgrade this to a
3503/// warning.** The loader's own words (`car_policy::load_policy_dir`): "a
3504/// security rule that fails to parse must surface, never be silently skipped."
3505/// A rule file exists to *stop* something; a typo that turns "never message
3506/// anyone outside the allowlist" into a no-op fails open, and it fails open
3507/// silently, at exactly the moment the operator believes they are protected. A
3508/// daemon that refuses to open the session is loud, immediate, and trivially
3509/// fixed; a daemon that logs a warning nobody reads is neither. The error text
3510/// names the offending file (`PolicyLoadError`'s `Display`) so the fix is
3511/// mechanical.
3512///
3513/// A **missing** directory is not a failure — most projects have no rules, and
3514/// `load_policy_dir` already returns an empty set for it.
3515///
3516/// Split out of [`ServerState::create_session`] so the fatal-vs-silent
3517/// behaviour is testable against a temp directory instead of the real `$HOME`.
3518pub(crate) async fn apply_project_policies(
3519 runtime: &Runtime,
3520 car_dir: &std::path::Path,
3521) -> Result<(), String> {
3522 match runtime.load_project_policies(car_dir).await {
3523 Ok(_) => Ok(()),
3524 Err(e) => Err(format!(
3525 "refusing to start with unreadable project policy rules: {e}. \
3526 Fix or remove the file — a policy rule that fails to parse is a \
3527 security control that would silently not exist."
3528 )),
3529 }
3530}
3531
3532impl ServerState {
3533 async fn run_durability_lock(&self, run_id: &str) -> Arc<Mutex<()>> {
3534 let mut locks = self.run_durability_locks.lock().await;
3535 if let Some(lock) = locks.get(run_id).and_then(std::sync::Weak::upgrade) {
3536 return lock;
3537 }
3538 locks.retain(|_, lock| lock.strong_count() > 0);
3539 let lock = Arc::new(Mutex::new(()));
3540 locks.insert(run_id.to_string(), Arc::downgrade(&lock));
3541 lock
3542 }
3543
3544 async fn quarantine_run_trace_locked(
3545 &self,
3546 runs: &mut HashMap<String, RunMeta>,
3547 run_id: &str,
3548 detail: String,
3549 ) -> String {
3550 let message = run_trace_corruption_message(run_id, detail);
3551 if let Some(meta) = runs.get_mut(run_id) {
3552 if meta.trace_corruption.as_deref() != Some(message.as_str()) {
3553 meta.durability_generation = meta.durability_generation.wrapping_add(1);
3554 }
3555 meta.trace_corruption = Some(message.clone());
3556 }
3557 let mut subs = self.run_subscribers.lock().await;
3558 subs.retain(|(subscribed_run, _), _| subscribed_run != run_id);
3559 message
3560 }
3561
3562 /// Publish corruption found by a strict replay read into live lifecycle
3563 /// state. Disk scanning and marker persistence happen before this call;
3564 /// only the per-run durability gate and short registry/subscriber mutation
3565 /// execute here, so a corrupt trace cannot remain writable or subscribed.
3566 pub(crate) async fn quarantine_run_trace_from_read(
3567 &self,
3568 run_id: &str,
3569 detail: String,
3570 ) -> String {
3571 let durability_lock = self.run_durability_lock(run_id).await;
3572 let _durability_guard = durability_lock.lock().await;
3573 let mut runs = self.runs.lock().await;
3574 self.quarantine_run_trace_locked(&mut runs, run_id, detail)
3575 .await
3576 }
3577
3578 /// Constructor for the standalone `car-server` binary. Each WS
3579 /// connection gets its own per-session memgine — matches the
3580 /// pre-extraction default and is correct for a single-process
3581 /// daemon serving one user at a time.
3582 ///
3583 /// **Embedders must not call this.** It silently leaves
3584 /// `shared_memgine = None`, which re-introduces the dual-memgine
3585 /// bug U7 was created to prevent (one engine in the embedder, a
3586 /// fresh one inside every WS session). Embedders use
3587 /// [`ServerState::embedded`] instead, which makes the shared
3588 /// engine handle a required argument so it cannot be forgotten.
3589 pub fn standalone(journal_dir: PathBuf) -> Self {
3590 Self::with_config(ServerStateConfig::new(journal_dir))
3591 }
3592
3593 /// The daemon-held multi-device-sync + execution-lease subsystem (B6),
3594 /// lazily opened on first `sync.*`/`lease.*` contact and rooted at
3595 /// `<journal_dir>/sync/`. Serialized first-open (see the [`Self::sync`]
3596 /// field docs): the oplog journal's exclusive advisory lock means only one
3597 /// open may succeed, so the fallible init runs under the field's
3598 /// `std::sync::Mutex` while the subsystem itself is shared behind a
3599 /// `tokio::sync::Mutex`.
3600 pub fn sync_subsystem(
3601 &self,
3602 ) -> Result<Arc<tokio::sync::Mutex<crate::sync::SyncSubsystem>>, String> {
3603 let mut guard = self
3604 .sync
3605 .lock()
3606 .map_err(|_| "sync subsystem init lock poisoned".to_string())?;
3607 if let Some(existing) = guard.as_ref() {
3608 return Ok(existing.clone());
3609 }
3610 let root = self.journal_dir.join("sync");
3611 let subsystem = self.open_sync_subsystem(&root)?;
3612 let arc = Arc::new(tokio::sync::Mutex::new(subsystem));
3613 *guard = Some(arc.clone());
3614 Ok(arc)
3615 }
3616
3617 /// Route a write to the right subsystem BY SCOPE: an opted-in
3618 /// `Scope::Shared{org}` goes to the shared org delivery subsystem (relay scope
3619 /// `org:{orgId}`, so it converges cross-member); everything else — `Personal`,
3620 /// or an org with no opted-in delivery — goes to the personal subsystem, which
3621 /// is byte-identical to before when org-scope is off. Used ONLY at the write
3622 /// sites; every reader keeps calling [`Self::sync_subsystem`]. (8a)
3623 pub fn subsystem_for_scope(
3624 &self,
3625 scope: &car_sync::Scope,
3626 ) -> Result<Arc<tokio::sync::Mutex<crate::sync::SyncSubsystem>>, String> {
3627 // Init the personal subsystem first — this also builds + stores the org
3628 // subsystem when opted in (via `open_sync_subsystem`).
3629 let user = self.sync_subsystem()?;
3630 let guard = self
3631 .org_sync
3632 .lock()
3633 .map_err(|_| "org sync lock poisoned".to_string())?;
3634 match route_for_scope(scope, guard.as_ref().map(|h| h.org.as_str())) {
3635 SyncRoute::Org => Ok(guard
3636 .as_ref()
3637 .expect("route_for_scope returns Org only when a holder exists")
3638 .subsystem
3639 .clone()),
3640 SyncRoute::User => Ok(user),
3641 }
3642 }
3643
3644 /// The opted-in shared org delivery subsystem, if any — for pumping both scopes.
3645 pub fn org_subsystem(&self) -> Option<Arc<tokio::sync::Mutex<crate::sync::SyncSubsystem>>> {
3646 match self.org_sync.lock() {
3647 Ok(g) => g.as_ref().map(|h| h.subsystem.clone()),
3648 // Surface the poison loudly — do NOT silently degrade to personal-only
3649 // sync (linus B). Poison only occurs on a panic while locked.
3650 Err(_) => {
3651 tracing::error!(
3652 "org_sync lock poisoned — skipping the org pump (org-scope delivery degraded)"
3653 );
3654 None
3655 }
3656 }
3657 }
3658
3659 /// Build the sync subsystem for `root`, selecting the backend from config.
3660 ///
3661 /// Default is the local shared-directory `FsRelay`. When
3662 /// `PARSLEE_SYNC_BACKEND=parslee` (env or keychain), it builds the
3663 /// Parslee-backed device: a `ParsleeSyncTransport` at
3664 /// `PARSLEE_CAR_SYNC_BASE_URL` (default `{api_base}/sync`), authed with the
3665 /// live refresh-aware bearer, scoped `user:{account_id}`, E2E-encrypted under
3666 /// a passphrase-derived key (`PARSLEE_SYNC_PASSPHRASE`, the zero-knowledge
3667 /// cross-device key — same passphrase → same key on every device, never sent
3668 /// to the server). Missing a Parslee login falls back to local (logged);
3669 /// missing the passphrase is a hard error (refusing to sync unencrypted).
3670 fn open_sync_subsystem(
3671 &self,
3672 root: &std::path::Path,
3673 ) -> Result<crate::sync::SyncSubsystem, String> {
3674 let backend = car_secrets::resolve_env_or_keychain("PARSLEE_SYNC_BACKEND")
3675 .unwrap_or_default()
3676 .to_lowercase();
3677 if backend != "parslee" {
3678 return crate::sync::SyncSubsystem::open(root);
3679 }
3680
3681 let Some(account_id) = self
3682 .parslee_session
3683 .get()
3684 .map(|s| s.identity.account_id.clone())
3685 .filter(|a| !a.is_empty())
3686 else {
3687 tracing::warn!(
3688 "sync backend=parslee but no Parslee login is present; using local sync until login"
3689 );
3690 return crate::sync::SyncSubsystem::open(root);
3691 };
3692
3693 let Some(passphrase) = car_secrets::resolve_env_or_keychain("PARSLEE_SYNC_PASSPHRASE")
3694 .filter(|p| !p.is_empty())
3695 else {
3696 return Err(
3697 "sync backend=parslee requires PARSLEE_SYNC_PASSPHRASE (the zero-knowledge \
3698 cross-device sync key) — set it in the keychain; refusing to sync unencrypted"
3699 .to_string(),
3700 );
3701 };
3702
3703 let base_url = car_secrets::resolve_env_or_keychain("PARSLEE_CAR_SYNC_BASE_URL")
3704 .filter(|u| !u.is_empty())
3705 .unwrap_or_else(|| {
3706 let api_base = car_auth::api_base(None);
3707 format!("{}/sync", api_base.trim_end_matches('/'))
3708 });
3709
3710 let bearer: car_parslee::sync_transport::BearerFn = Arc::new(car_auth::access_token);
3711 let transport = Arc::new(
3712 car_parslee::sync_transport::ParsleeSyncTransport::new(base_url, bearer)
3713 .map_err(|e| format!("sync: build Parslee transport: {e}"))?,
3714 );
3715 // Argon2id-stretch the passphrase into the master ONCE at open (memory-hard
3716 // work factor for a password-equivalent secret), then feed BOTH the
3717 // per-audience provider and the org identity from it — so Argon2id runs
3718 // exactly once, not once per derivation.
3719 let master = car_sync::StretchedMaster::from_passphrase(passphrase.as_bytes(), &account_id);
3720 let base: Arc<dyn car_sync::SyncKeyProvider> =
3721 Arc::new(car_sync::DerivedKeyProvider::from_master(&master));
3722
3723 // Org-scope shared-brain activation — REAL provisioning, but DEFAULT-OFF.
3724 // `PARSLEE_SYNC_ORG_SCOPE=<orgId>:<granter_hex>[,<granter_hex>...]` names the
3725 // org AND the admin-designated trusted granter key(s), held LOCALLY (never
3726 // backend-fetched — see `parse_org_scope_config`). Unset → `base`
3727 // (byte-identical to the personal-only path). Malformed → hard error above.
3728 // The configured org MUST match the signed-in `active_organization` or we
3729 // refuse loudly. Turning it ON for a real tenant is slice 12, still gated on
3730 // the cryptographer audit of the login_secret→identity entropy dependency.
3731 let mut pending_org: Option<OrgSyncHolder> = None;
3732 let provider: Arc<dyn car_sync::SyncKeyProvider> = match Self::parse_org_scope_config(
3733 car_secrets::resolve_env_or_keychain("PARSLEE_SYNC_ORG_SCOPE"),
3734 )? {
3735 None => base,
3736 Some((org_id, granters)) => {
3737 // Bind the configured org to the signed-in active organization — never
3738 // encrypt org ops for an org this account is not actually a member of.
3739 let active_org = self
3740 .parslee_session
3741 .get()
3742 .and_then(|s| s.identity.active_organization.clone())
3743 .filter(|o| !o.is_empty());
3744 if active_org.as_deref() != Some(org_id.as_str()) {
3745 return Err(format!(
3746 "PARSLEE_SYNC_ORG_SCOPE org {org_id:?} does not match the signed-in active \
3747 organization {active_org:?} — refusing to activate org scope"
3748 ));
3749 }
3750
3751 // Bring the directory trait into scope for `publish_pubkey`.
3752 use car_sync::OrgKeyDirectory as _;
3753 let my_secret = car_sync::derive_x25519_identity(&master, &account_id);
3754 let mut directory = car_sync::NetworkOrgKeyDirectory::new(
3755 transport.clone(),
3756 format!("org:{org_id}"),
3757 );
3758
3759 // Announce our X25519 identity pubkey so a granter can wrap K_org for
3760 // us (idempotent LWW). Non-fatal: a failure just means future grants
3761 // may not reach us; any already-resolved roots still work. This routes
3762 // to the bearer-authenticated backend, but whether the backend
3763 // ENFORCES "only this account may publish its own pubkey" (and "only a
3764 // K_org holder may publish a wrap") is the confidentiality-critical,
3765 // AUDIT-OWNED check — this is not a guarantee against a hostile backend.
3766 let my_pub = car_sync::x25519_public(&my_secret);
3767 let my_pub_hex: String = my_pub
3768 .as_bytes()
3769 .iter()
3770 .map(|b| format!("{b:02x}"))
3771 .collect();
3772 if let Err(e) = directory.publish_pubkey(&account_id, &my_pub_hex) {
3773 tracing::warn!(org = %org_id, error = %e, "org-scope: failed to publish member pubkey — future grants may not reach this member");
3774 }
3775
3776 // Resolve EVERY epoch of K_org we can unwrap (slice 10) → the
3777 // multi-epoch map the cipher selects by kid. `org → { epoch → K_org }`.
3778 // Fail-closed: Err (directory unreachable) and empty (ungranted) both
3779 // leave org scope on DenyCipher, NEVER demoted to the personal key.
3780 let mut roots = std::collections::HashMap::new();
3781 match car_sync::resolve_all_org_roots(
3782 &directory,
3783 &org_id,
3784 &my_secret,
3785 &account_id,
3786 &granters,
3787 ) {
3788 Ok(map) if !map.is_empty() => {
3789 tracing::info!(org = %org_id, epochs = ?map.keys().collect::<Vec<_>>(), "org-scope: resolved K_org generations");
3790 roots.insert(org_id.clone(), map);
3791 }
3792 Ok(_) => tracing::warn!(
3793 org = %org_id,
3794 "org-scope: no trusted grant for this member — org scope fails closed (DenyCipher)"
3795 ),
3796 Err(e) => tracing::warn!(
3797 org = %org_id, error = %e,
3798 "org-scope: org-key directory unreachable — org scope fails closed (DenyCipher), NOT demoted to the personal key"
3799 ),
3800 }
3801 // ALWAYS OrgAwareKeyProvider once opted in — NEVER the bare
3802 // DerivedKeyProvider, which has no Scope::Shared case and would
3803 // silently encrypt org ops under the PERSONAL key (a fail-open).
3804 let provider: Arc<dyn car_sync::SyncKeyProvider> =
3805 Arc::new(car_sync::OrgAwareKeyProvider::new(base, roots));
3806
3807 // Open the SHARED org delivery subsystem: a SECOND device oplog
3808 // under `root/org-{orgId}/` whose relay scope is `org:{orgId}`, with
3809 // the SAME OrgAwareKeyProvider (it keys Scope::Shared{org} on K_org).
3810 // Scope::Shared{org} writes route here (subsystem_for_scope) and
3811 // converge across the org's members. `org_id` is a canonical slug
3812 // (validated on parse), so it is filesystem-safe. (8a)
3813 let org_root = root.join(format!("org-{org_id}"));
3814 let org_subsystem = crate::sync::SyncSubsystem::open_remote(
3815 &org_root,
3816 transport.clone(),
3817 format!("org:{org_id}"),
3818 provider.clone(),
3819 )?;
3820 // Do NOT commit to `org_sync` yet — hold the org subsystem in a
3821 // local and only install it AFTER the personal open below succeeds.
3822 // Else a failed personal open would orphan the org journal's
3823 // exclusive lock and permanently wedge sync retry (linus A).
3824 pending_org = Some(OrgSyncHolder {
3825 org: org_id.clone(),
3826 subsystem: Arc::new(tokio::sync::Mutex::new(org_subsystem)),
3827 });
3828 tracing::info!(org = %org_id, "org-scope: opened shared org delivery subsystem (relay org:{org_id})");
3829 provider
3830 }
3831 };
3832 let user = crate::sync::SyncSubsystem::open_remote(
3833 root,
3834 transport,
3835 format!("user:{account_id}"),
3836 provider,
3837 )?;
3838 // Personal open succeeded — NOW commit the org holder. On the failure path
3839 // above, `pending_org` drops here, releasing the org journal lock so a
3840 // retry can re-open cleanly.
3841 if let Some(holder) = pending_org {
3842 *self
3843 .org_sync
3844 .lock()
3845 .map_err(|_| "org sync init lock poisoned".to_string())? = Some(holder);
3846 }
3847 Ok(user)
3848 }
3849
3850 /// Parse `PARSLEE_SYNC_ORG_SCOPE` (`<orgId>:<granter_hex>[,<granter_hex>...]`).
3851 /// `Ok(None)` when unset/empty → the personal-only path. `Err` on ANY malformed
3852 /// value — a typo must be a loud failure, never a silent fall-through to a
3853 /// wrong or unauthenticated state. Returns the org id + the (≥1) admin-designated
3854 /// trusted granter verifying keys.
3855 ///
3856 /// The granter set is held LOCALLY (operator-provisioned), NEVER fetched from
3857 /// the backend: whoever fills the `trusted` slice holds the authority, and if
3858 /// the same backend that serves the directory could name the granters it could
3859 /// name ITSELF a granter and substitute `K_org`. A backend-sourced / signed
3860 /// granter manifest is an audit-owned trust decision (slice 12), not this.
3861 /// Pure over the raw value so it is unit-testable without the environment.
3862 fn parse_org_scope_config(
3863 raw: Option<String>,
3864 ) -> Result<Option<(String, Vec<car_sync::OrgVerifyingKey>)>, String> {
3865 let Some(raw) = raw.filter(|s| !s.is_empty()) else {
3866 return Ok(None);
3867 };
3868 let (org_id, granters_csv) = raw.split_once(':').ok_or_else(|| {
3869 "PARSLEE_SYNC_ORG_SCOPE must be <orgId>:<granter_ed25519_pubkey_hex>[,...]".to_string()
3870 })?;
3871 // Honor the "loud failure" rule: a non-canonical org id (space, Unicode,
3872 // the platform's raw tenant form) must error here, not silently resolve
3873 // nothing — it would never match a real wrap's canonical `org` anyway.
3874 car_sync::require_canonical_org(org_id)
3875 .map_err(|e| format!("PARSLEE_SYNC_ORG_SCOPE: bad orgId: {e}"))?;
3876 let granters = granters_csv
3877 .split(',')
3878 .map(str::trim)
3879 .filter(|h| !h.is_empty())
3880 .map(|hex| {
3881 car_sync::parse_ed25519_verifying(hex)
3882 .map_err(|e| format!("PARSLEE_SYNC_ORG_SCOPE: bad granter key {hex:?}: {e}"))
3883 })
3884 .collect::<Result<Vec<_>, _>>()?;
3885 if granters.is_empty() {
3886 return Err(
3887 "PARSLEE_SYNC_ORG_SCOPE: at least one granter ed25519 pubkey is required".into(),
3888 );
3889 }
3890 Ok(Some((org_id.to_string(), granters)))
3891 }
3892
3893 /// Persist the daemon-wide standing chat-goal registry. The snapshot is
3894 /// taken under the async mutex, then serialized and atomically rewritten on
3895 /// the blocking pool so chat/event dispatch does not block a Tokio worker.
3896 pub async fn persist_chat_goals(&self) -> Result<(), String> {
3897 let path = chat_goals_path_from_journal_dir(&self.journal_dir);
3898 let goals = self.chat_goals.lock().await.clone();
3899 tokio::task::spawn_blocking(move || {
3900 write_chat_goals_atomic(&path, &goals)
3901 .map_err(|e| format!("write {}: {e}", path.display()))
3902 })
3903 .await
3904 .map_err(|e| format!("persist chat goals join: {e}"))?
3905 }
3906
3907 /// Constructor for embedders (e.g. `tokhn-daemon`). The shared
3908 /// memgine handle is **required**: every WS session created by
3909 /// this state will reuse the same engine, preventing the
3910 /// dual-memgine bug.
3911 ///
3912 /// For embedders that also want to inject a pre-warmed inference
3913 /// engine or other advanced wiring, build a [`ServerStateConfig`]
3914 /// directly and call [`ServerState::with_config`].
3915 pub fn embedded(
3916 journal_dir: PathBuf,
3917 shared_memgine: Arc<Mutex<car_memgine::MemgineEngine>>,
3918 ) -> Self {
3919 Self::with_config(ServerStateConfig::new(journal_dir).with_shared_memgine(shared_memgine))
3920 }
3921
3922 /// Build a `ServerState` from a [`ServerStateConfig`] — the path
3923 /// embedders use when they need to inject a shared memgine *and*
3924 /// a pre-warmed inference engine, or any other advanced wiring
3925 /// the convenience constructors don't cover.
3926 pub fn with_config(cfg: ServerStateConfig) -> Self {
3927 Self::try_with_config(cfg)
3928 .unwrap_or_else(|error| panic!("server state startup failed: {error}"))
3929 }
3930
3931 /// Build a state with test-owned feedback diagnostic roots and no ambient
3932 /// registry probes. This bypasses the production diagnostic default rather
3933 /// than constructing it and then overriding it, so state construction never
3934 /// resolves `HOME`/`USERPROFILE` for feedback. Public only because Cargo
3935 /// integration tests link `car-server-core` without `cfg(test)`.
3936 #[doc(hidden)]
3937 pub fn with_config_and_isolated_feedback_diagnostics(
3938 cfg: ServerStateConfig,
3939 models_dir: PathBuf,
3940 huggingface_hub_root: PathBuf,
3941 ) -> Self {
3942 Self::try_with_config_and_feedback_diagnostics(
3943 cfg,
3944 crate::feedback::FeedbackDiagnostics::isolated(models_dir, huggingface_hub_root),
3945 )
3946 .unwrap_or_else(|error| panic!("server state startup failed: {error}"))
3947 }
3948
3949 /// Fallible startup surface used by the daemon before it begins listening.
3950 /// Durable lifecycle outbox replay is acknowledgement-bounded; any
3951 /// durability-unknown result stops construction and leaves its RunStore
3952 /// preimage/quarantine intact for an exact later retry.
3953 pub fn try_with_config(cfg: ServerStateConfig) -> Result<Self, String> {
3954 Self::try_with_config_and_feedback_diagnostics(
3955 cfg,
3956 crate::feedback::FeedbackDiagnostics::production(),
3957 )
3958 }
3959
3960 fn try_with_config_and_feedback_diagnostics(
3961 cfg: ServerStateConfig,
3962 feedback_diagnostics: crate::feedback::FeedbackDiagnostics,
3963 ) -> Result<Self, String> {
3964 let startup_acknowledgement_timeout = cfg.startup_reconciliation_acknowledgement_timeout;
3965 if startup_acknowledgement_timeout.is_zero()
3966 || startup_acknowledgement_timeout > car_eventlog::MAX_CRITICAL_ACKNOWLEDGEMENT_TIMEOUT
3967 {
3968 return Err(format!(
3969 "startup reconciliation acknowledgement timeout must be between 1ns and {}ms, got {}ms",
3970 car_eventlog::MAX_CRITICAL_ACKNOWLEDGEMENT_TIMEOUT.as_millis(),
3971 startup_acknowledgement_timeout.as_millis()
3972 ));
3973 }
3974 let inference = std::sync::OnceLock::new();
3975 if let Some(eng) = cfg.inference {
3976 // OnceLock::set returns Err if already set — fresh OnceLock
3977 // means it's empty, so this is infallible here.
3978 let _ = inference.set(eng);
3979 }
3980 let voice_sessions = Arc::new(car_voice::VoiceSessionRegistry::new());
3981 // Reap sessions whose clients dropped without calling
3982 // voice.transcribe_stream.stop (WS disconnect, process exit,
3983 // etc.). Listener handles otherwise leak for the daemon's
3984 // lifetime. `with_config` is sync but always called from the
3985 // `#[tokio::main]` entry point, so `Handle::try_current()`
3986 // inside `start_sweeper` finds the runtime.
3987 voice_sessions.start_sweeper();
3988 // UI-improvement agent is pure decision logic — no I/O, no
3989 // persistence handle. Memgine ingest of strategy outcomes is
3990 // the caller's responsibility (handler.rs after a successful
3991 // Decision::Patch). Keeps the agent crate Mutex-flavor
3992 // agnostic so it can compose with std/tokio mutex callers.
3993 let ui_agent = Arc::new(car_ui_agent::UIImprovementAgent::with_default_strategies());
3994 let ui_agent_oscillation = Arc::new(crate::ui_agent_loop::OscillationDetector::new());
3995 let ui_agent_budget = Arc::new(crate::ui_agent_loop::IterationBudget::new());
3996 // Disk-backed run-trace store (U3). Rooted at the `runs/` sibling
3997 // of the journal dir (`~/.car/runs`), with retention read from
3998 // `~/.car/config.toml`. Boot-time GC enforces the retention cap
3999 // (R6) — best-effort; a failed GC must never block startup, so the
4000 // count is dropped. Never evicts an in-progress run.
4001 let mut run_store = crate::run_store::RunStore::from_journal_dir(&cfg.journal_dir);
4002 if let Some(failures) = cfg.run_store_failures.clone() {
4003 run_store = run_store.with_failure_injector(failures);
4004 }
4005 if let Some(gate) = cfg.run_store_summary_read_gate.clone() {
4006 run_store = run_store.with_summary_read_gate(gate);
4007 }
4008 if let Some(gate) = cfg.run_store_summary_write_gate.clone() {
4009 run_store = run_store.with_summary_write_gate(gate);
4010 }
4011 if let Some(gate) = cfg.run_store_append_gate.clone() {
4012 run_store = run_store.with_append_gate(gate);
4013 }
4014 if let Some(gate) = cfg.run_store_lookup_gate.clone() {
4015 run_store = run_store.with_lookup_gate(gate);
4016 }
4017 if let Some(failures) = cfg.run_store_private_path_failures.clone() {
4018 run_store = run_store.with_private_path_failure_injector(failures);
4019 }
4020 run_store
4021 .reconcile_proposal_retry_rollbacks()
4022 .map_err(|error| format!("proposal retry rollback reconciliation failed: {error}"))?;
4023 let chat_goals = load_chat_goals_from_disk(&cfg.journal_dir);
4024 let selfheal_ledger = cfg.selfheal_ledger.clone().unwrap_or_else(|| {
4025 cfg.journal_dir
4026 .parent()
4027 .unwrap_or(&cfg.journal_dir)
4028 .join("selfheal")
4029 .join("detections.jsonl")
4030 });
4031 // Config lives beside the daemon's other state. A missing or empty
4032 // `heal.toml` is a disabled loop, which is the default: there are no
4033 // compiled-in targets, because a loop that opened pull requests in a
4034 // repository nobody named would be a surprise in somebody else's
4035 // tracker.
4036 let heal_state_dir =
4037 crate::coder::rpc::coder_state_dir().unwrap_or_else(|_| cfg.journal_dir.join("coder"));
4038 let heal_config_dir = car_home::root_or_relative();
4039 let heal = Arc::new(crate::coder::heal_service::HealService::with_config_dir(
4040 crate::coder::heal_config::HealConfig::load(&heal_config_dir),
4041 heal_state_dir,
4042 heal_config_dir,
4043 ));
4044 let selfheal = Arc::new(crate::selfheal::SelfhealService::open(
4045 selfheal_ledger,
4046 cfg.selfheal_interval_secs,
4047 cfg.selfheal_evidence.clone(),
4048 cfg.selfheal_source_probe.clone(),
4049 cfg.selfheal_replay_verb_probe.clone(),
4050 )?);
4051 let mut startup_journals =
4052 StartupJournalCache::new(&cfg.journal_dir, cfg.journal_failures.as_ref());
4053 // Establish the durable RunStarted bracket before replaying a pending
4054 // proposal terminal. Then adopt the crash orphan and run lifecycle
4055 // reconciliation again to add RunCompleted. Both passes share the
4056 // per-client journal cache, are exact-row idempotent, and preserve:
4057 // RunStarted -> ProposalCompleted -> RunCompleted.
4058 reconcile_durable_run_journals(
4059 &mut startup_journals,
4060 &run_store,
4061 startup_acknowledgement_timeout,
4062 )?;
4063 reconcile_pending_proposal_journals(
4064 &mut startup_journals,
4065 &run_store,
4066 startup_acknowledgement_timeout,
4067 )?;
4068 reconcile_completed_proposal_guards(&run_store);
4069 // The in-memory map is empty at boot, so any remaining on-disk
4070 // InProgress run is a crashed prior process; mark it Incomplete so it
4071 // is terminal and age-GC-eligible (FIX 4). Adoption must precede the
4072 // only boot GC so those runs do not remain immortal.
4073 let adopted = run_store.adopt_orphans();
4074 tracing::info!(
4075 adopted,
4076 "adopted crash-orphaned run traces at daemon startup"
4077 );
4078 reconcile_durable_run_journals(
4079 &mut startup_journals,
4080 &run_store,
4081 startup_acknowledgement_timeout,
4082 )?;
4083 let evicted = run_store.gc();
4084 if evicted > 0 {
4085 // Said out loud, for the same reason the coder GC below says it: an
4086 // operator whose own config just ate part of their store has to be
4087 // able to see it happened. This was silent, which is what made a
4088 // zero cap a data-loss incident rather than a log line (car#1338) —
4089 // and `max_age_days = 1` where someone meant `10` still deletes nine
4090 // days of traces, which the age of the policy has nothing to do with.
4091 tracing::info!(
4092 evicted,
4093 max_per_agent = ?run_store.retention().count_cap(),
4094 max_age_days = ?run_store.retention().age_cap_days(),
4095 "pruned run traces (~/.car/config.toml [runs] retention)"
4096 );
4097 }
4098 // Adopt crash/restart-orphaned coder sessions the same way: the
4099 // in-memory `coder_sessions` map is empty at boot, so any non-terminal
4100 // snapshot under the coder state dir is a prior process's orphan. Mark
4101 // genuinely-stranded ones Failed ("daemon restarted mid-session") and
4102 // preserve a `needs_approval` orphan whose worktree still exists so the
4103 // user can inspect/approve it by hand. Best-effort — never blocks boot.
4104 if let Ok(coder_dir) = crate::coder::rpc::coder_state_dir() {
4105 let _coder_adopted = crate::coder::session::adopt_orphaned_sessions(&coder_dir);
4106 // Then prune, the same way `run_store.gc()` above prunes run
4107 // traces. Adoption runs first on purpose: it is what turns a
4108 // stranded orphan terminal, and only terminal sessions are
4109 // collectable. An orphan adopted just now carries a fresh
4110 // `updated_at`, so this boot never collects what this boot
4111 // rewrote — it ages out on a later one.
4112 let retention = crate::coder::config::CoderConfig::load().session_retention();
4113 let collected = crate::coder::session::gc_sessions(
4114 &coder_dir,
4115 &retention,
4116 crate::coder::session::SweepScope::Boot,
4117 );
4118 if collected > 0 {
4119 // Said out loud, unlike `run_store.gc()` above. That policy has
4120 // existed since the run store did; this one is NEW and applies
4121 // retroactively, so the first boot after an upgrade can delete
4122 // years of snapshots under caps the operator never chose. An
4123 // operator who did not want that has to be able to see it
4124 // happened.
4125 tracing::info!(
4126 collected,
4127 max_sessions = retention.max_sessions,
4128 max_age_days = retention.max_age_days,
4129 "pruned coder session snapshots (~/.car/coder.toml retention)"
4130 );
4131 }
4132 }
4133 // The daemon-wide shared approval ledger (kernel review C1): one
4134 // journal-backed store for every session's HITL records, so approvals
4135 // cross connections and survive restart. In-memory fallback is loud —
4136 // a silent downgrade here would fake durability.
4137 let peer_audit_journal = cfg
4138 .peer_audit_journal
4139 .clone()
4140 .unwrap_or_else(|| default_peer_audit_journal_path(&cfg.journal_dir));
4141 let approval_ledger = {
4142 let path = cfg.approval_journal.or_else(default_approval_journal_path);
4143 match path {
4144 Some(p) => {
4145 // The journal's parent must exist before the first
4146 // append-mode open (ApprovalLedger::record does not
4147 // create directories).
4148 if let Some(parent) = p.parent() {
4149 let _ = std::fs::create_dir_all(parent);
4150 }
4151 match car_policy::ApprovalLedger::with_journal(&p) {
4152 Ok(l) => {
4153 if l.skipped_on_load() > 0 {
4154 tracing::warn!(
4155 path = %p.display(),
4156 skipped = l.skipped_on_load(),
4157 "approval-ledger journal had unparseable lines; partial ledger loaded"
4158 );
4159 }
4160 l
4161 }
4162 Err(e) => {
4163 tracing::warn!(
4164 path = %p.display(), error = %e,
4165 "approval-ledger journal unopenable; falling back to IN-MEMORY \
4166 ledger — approvals will NOT survive restart"
4167 );
4168 car_policy::ApprovalLedger::new()
4169 }
4170 }
4171 }
4172 None => {
4173 tracing::warn!(
4174 "no CAR_HOME/HOME/USERPROFILE to place approvals.jsonl under the CAR \
4175 state root; approval ledger is IN-MEMORY — approvals will NOT survive \
4176 restart"
4177 );
4178 car_policy::ApprovalLedger::new()
4179 }
4180 }
4181 };
4182 // Identity nodes are deliberately absent from memgine snapshots:
4183 // identity.json is authoritative and is re-mirrored on every daemon
4184 // startup. This avoids two durable copies drifting after a profile edit.
4185 if let Some(engine) = cfg.shared_memgine.as_ref() {
4186 match cfg.identity_store.load() {
4187 Ok(identity) => {
4188 let mut engine = engine.try_lock().map_err(|_| {
4189 "server state startup could not initialize the shared memgine identity: \
4190 the supplied engine is already locked"
4191 .to_string()
4192 })?;
4193 mirror_identity_into_memgine(&mut engine, &identity);
4194 }
4195 Err(error) => tracing::warn!(
4196 error = %error,
4197 "identity profile could not be mirrored at daemon startup"
4198 ),
4199 }
4200 }
4201
4202 // Built before the struct literal so the browser-view registry can be
4203 // pointed at the SAME `HostState` it will broadcast sign-in attention
4204 // on (`host.event`) — see `crate::browser_attention`.
4205 let host = Arc::new(crate::host::HostState::new());
4206 let browser_views = Arc::new(crate::browser_view::BrowserViewRegistry::default());
4207 browser_views.set_signin_attention(Arc::new(
4208 crate::browser_attention::HostSignInAttention::new(Arc::clone(&host)),
4209 ));
4210 Ok(Self {
4211 journal_dir: cfg.journal_dir,
4212 peer_audit_journal,
4213 sessions: Mutex::new(HashMap::new()),
4214 inference,
4215 host,
4216 shared_memgine: cfg.shared_memgine,
4217 identity_store: cfg.identity_store,
4218 feedback_diagnostics,
4219 trajectory_store: Arc::new(car_memgine::TrajectoryStore::new(
4220 &cfg.trajectory_dir
4221 .unwrap_or_else(car_memgine::TrajectoryStore::default_path),
4222 )),
4223 selfheal,
4224 heal,
4225 voice_sessions,
4226 meetings: Arc::new(car_meeting::MeetingRegistry::new()),
4227 a2ui: car_a2ui::A2uiSurfaceStore::new(),
4228 ui_agent,
4229 ui_agent_oscillation,
4230 ui_agent_budget,
4231 admission: Arc::new(crate::admission::InferenceAdmission::new()),
4232 a2ui_route_auth: Mutex::new(HashMap::new()),
4233 supervisor: std::sync::OnceLock::new(),
4234 declagents: std::sync::OnceLock::new(),
4235 routing: std::sync::OnceLock::new(),
4236 sync: std::sync::Mutex::new(None),
4237 org_sync: std::sync::Mutex::new(None),
4238 observer_manifest_path: std::sync::OnceLock::new(),
4239 a2a_dispatcher: std::sync::OnceLock::new(),
4240 a2a_runtime: std::sync::Mutex::new(cfg.a2a_runtime),
4241 a2a_store: std::sync::Mutex::new(cfg.a2a_store),
4242 a2a_card_source: std::sync::Mutex::new(cfg.a2a_card_source),
4243 a2ui_subscribers: Mutex::new(HashMap::new()),
4244 mcp_executor: Arc::new(car_engine::McpToolExecutor::new()),
4245 connectors: std::sync::OnceLock::new(),
4246 connectors_loaded: std::sync::atomic::AtomicBool::new(false),
4247 channel_supervisor: std::sync::OnceLock::new(),
4248 durable_tasks: Mutex::new(tokio::task::JoinSet::new()),
4249 auth_completion_owner_id: uuid::Uuid::new_v4().simple().to_string(),
4250 auth_token: std::sync::OnceLock::new(),
4251 host_token: std::sync::OnceLock::new(),
4252 mobile_runtime_url: std::sync::OnceLock::new(),
4253 mobile_registration_url: std::sync::OnceLock::new(),
4254 parslee_session: std::sync::OnceLock::new(),
4255 attached_agents: Mutex::new(HashMap::new()),
4256 agent_memgines: Mutex::new(HashMap::new()),
4257 namespace_memgines: Mutex::new(HashMap::new()),
4258 coder_sessions: Mutex::new(HashMap::new()),
4259 coder_disk_gc_base: std::time::Instant::now(),
4260 // `0` IS "just swept": boot ran the same sweep a few lines above,
4261 // and zero seconds have elapsed since the base taken on the line
4262 // above this one. The first `coder.start` is therefore a no-op
4263 // rather than an immediate re-scan.
4264 coder_disk_gc_at: AtomicU64::new(0),
4265 coder_subscribers: Mutex::new(HashMap::new()),
4266 coder_watchers: Mutex::new(HashMap::new()),
4267 coder_watch_notify: std::sync::OnceLock::new(),
4268 coder_discussions: Mutex::new(HashMap::new()),
4269 coder_discussion_recovery: Mutex::new(()),
4270 coder_discussion_slots: Arc::new(tokio::sync::Semaphore::new(
4271 crate::coder::discuss::MAX_OPEN_DISCUSSIONS,
4272 )),
4273 chat_sessions: Mutex::new(HashMap::new()),
4274 peer_guards: Mutex::new(HashMap::new()),
4275 mcp_peer_sessions: Mutex::new(HashMap::new()),
4276 peer_standing: Mutex::new(HashMap::new()),
4277 held_peer_messages: Mutex::new(std::collections::VecDeque::new()),
4278 lan_discovery: std::sync::Mutex::new(None),
4279 peer_identity: std::sync::Mutex::new(None),
4280 peer_trust: car_a2a::peer_auth::PeerTrust::new(),
4281 chat_collectors: Mutex::new(HashMap::new()),
4282 chat_goals: Mutex::new(chat_goals),
4283 runs: Mutex::new(HashMap::new()),
4284 run_resume_liveness: Mutex::new(()),
4285 run_resume_lease: cfg.run_resume_lease,
4286 run_completion_fence_gate: cfg.run_completion_fence_gate,
4287 run_durability_locks: Mutex::new(HashMap::new()),
4288 run_subscribers: Mutex::new(HashMap::new()),
4289 browser_views,
4290 run_store,
4291 journal_failures: cfg.journal_failures,
4292 mcp_url: std::sync::OnceLock::new(),
4293 approval_gate: cfg.approval_gate.unwrap_or_default(),
4294 supervision: Arc::new(crate::supervision::SupervisionRegistry::default()),
4295 approval_ledger: Arc::new(tokio::sync::RwLock::new(approval_ledger)),
4296 // Installed by the daemon binary at startup (see
4297 // `car-server/src/main.rs`), never here: the implementation lives
4298 // in `car-bench`, which depends on this crate.
4299 harness_measurer: std::sync::RwLock::new(None),
4300 })
4301 }
4302
4303 /// Install the in-process harness evaluator `evolution.run`'s
4304 /// `harness_measure` path grades candidates with. Called once at daemon
4305 /// startup; a later call replaces it.
4306 pub fn set_harness_measurer(&self, m: Arc<dyn crate::evolution::HarnessMeasurer>) {
4307 match self.harness_measurer.write() {
4308 Ok(mut guard) => *guard = Some(m),
4309 // Poisoned only if a panic happened while the lock was held.
4310 // Refusing loudly beats leaving `evolution.run` to believe an
4311 // evaluator is installed when none is.
4312 Err(e) => tracing::error!(
4313 error = %e,
4314 "harness_measurer lock poisoned; the in-process harness evaluator was NOT installed"
4315 ),
4316 }
4317 }
4318
4319 /// The installed harness evaluator, if any. Clones the `Arc` out and drops
4320 /// the guard so callers never hold a `std::sync` lock across an await.
4321 pub fn harness_measurer(&self) -> Option<Arc<dyn crate::evolution::HarnessMeasurer>> {
4322 match self.harness_measurer.read() {
4323 Ok(guard) => guard.clone(),
4324 Err(e) => {
4325 tracing::error!(error = %e, "harness_measurer lock poisoned; treating as absent");
4326 None
4327 }
4328 }
4329 }
4330
4331 /// Start one daemon-owned operation and return a connection-scoped result
4332 /// receiver. Dropping the receiver never cancels the operation.
4333 ///
4334 /// The caller awaits the receiver exactly as it would have awaited the
4335 /// operation, so response shape and timing are unchanged for a connection
4336 /// that stays alive. What changes is the failure mode: when the connection
4337 /// goes away, the per-connection `JoinSet` aborts only this waiter, and the
4338 /// operation itself runs to completion on the daemon.
4339 pub async fn spawn_durable_operation<F, E>(
4340 &self,
4341 operation_name: impl Into<String>,
4342 operation: F,
4343 ) -> tokio::sync::oneshot::Receiver<Result<serde_json::Value, E>>
4344 where
4345 F: std::future::Future<Output = Result<serde_json::Value, E>> + Send + 'static,
4346 E: Send + 'static,
4347 {
4348 let (result_tx, result_rx) = tokio::sync::oneshot::channel();
4349 let operation_name = operation_name.into();
4350 let mut tasks = self.durable_tasks.lock().await;
4351 while let Some(result) = tasks.try_join_next() {
4352 if let Err(error) = result {
4353 tracing::warn!(error = %error, "daemon-owned task failed to join");
4354 }
4355 }
4356 tasks.spawn(async move {
4357 let result = operation.await;
4358 if result_tx.send(result).is_err() {
4359 tracing::debug!(
4360 operation = %operation_name,
4361 "daemon-owned operation finished after its response waiter closed"
4362 );
4363 }
4364 operation_name
4365 });
4366 result_rx
4367 }
4368
4369 /// Start one fire-and-reconcile daemon-owned operation.
4370 pub async fn spawn_durable_task<F>(&self, operation_name: impl Into<String>, operation: F)
4371 where
4372 F: std::future::Future<Output = ()> + Send + 'static,
4373 {
4374 let operation_name = operation_name.into();
4375 let mut tasks = self.durable_tasks.lock().await;
4376 while let Some(result) = tasks.try_join_next() {
4377 if let Err(error) = result {
4378 tracing::warn!(error = %error, "daemon-owned task failed to join");
4379 }
4380 }
4381 tasks.spawn(async move {
4382 operation.await;
4383 operation_name
4384 });
4385 }
4386
4387 /// Enable the per-launch auth handshake. After this call, every
4388 /// new WS connection must call `session.auth` with `token` as
4389 /// the first frame; otherwise the connection is closed. Called
4390 /// by `car-server` at startup unless `--no-auth` is set
4391 /// (default flipped 2026-05); embedders supply their own token
4392 /// if they want the same posture. Returns `Err(token)` when
4393 /// auth was already installed.
4394 pub fn install_auth_token(&self, token: String) -> Result<(), String> {
4395 self.auth_token.set(token)
4396 }
4397
4398 /// Install the per-launch host token (Parslee-ai/car#254). A
4399 /// session that later presents this via `session.auth { host_token }`
4400 /// is granted the host-management role (`ClientSession::is_host`),
4401 /// which `authorize_run_access` requires for cross-agent run-trace
4402 /// reads. Set by `car-server` at startup (mints + writes the `0600`
4403 /// `host-token` file) unless `--no-auth` is set. Returns `Err(token)`
4404 /// when a host token was already installed.
4405 pub fn install_host_token(&self, token: String) -> Result<(), String> {
4406 self.host_token.set(token)
4407 }
4408
4409 pub fn install_mobile_runtime_url(&self, url: String) -> Result<(), String> {
4410 self.mobile_runtime_url.set(url)
4411 }
4412
4413 pub fn install_mobile_registration_url(&self, url: String) -> Result<(), String> {
4414 self.mobile_registration_url.set(url)
4415 }
4416
4417 pub fn install_parslee_session(
4418 &self,
4419 session: crate::parslee_auth::ParsleeSession,
4420 ) -> Result<(), crate::parslee_auth::ParsleeSession> {
4421 self.parslee_session.set(session)
4422 }
4423
4424 /// Install the runtime channel supervisor (Units 1/2/3). Called by
4425 /// `car-server` at boot right after `spawn_channel_pollers` builds it, so
4426 /// the host-gated `messaging.config.set` handler can reach it via
4427 /// `ServerState` to spawn a channel's watcher on enable. Idempotent on the
4428 /// first call; a second install is rejected (returns the supervisor back).
4429 pub fn install_channel_supervisor(
4430 &self,
4431 supervisor: Arc<crate::channel_supervisor::ChannelSupervisor>,
4432 ) -> Result<(), Arc<crate::channel_supervisor::ChannelSupervisor>> {
4433 self.channel_supervisor.set(supervisor)
4434 }
4435
4436 /// Install the bound MCP URL after car-server's listener is up.
4437 /// Idempotent on the first call; subsequent calls are accepted
4438 /// silently (matches the supervisor / a2a_dispatcher install
4439 /// idiom). Returns `Err(())` when an MCP URL was already
4440 /// installed — embedders should treat this as "another
4441 /// component beat us to it" and use whichever value is now set.
4442 pub fn install_mcp_url(&self, url: String) -> Result<(), String> {
4443 self.mcp_url.set(url)
4444 }
4445
4446 /// Lazy-initialize and return the remote MCP connector manager,
4447 /// backed by `~/.car/connectors.json` over the shared
4448 /// [`mcp_executor`](Self::mcp_executor). Construction is sync and
4449 /// touches no network — call [`ensure_connectors_loaded`](crate::session::ServerState::ensure_connectors_loaded) to dial
4450 /// persisted connectors.
4451 pub fn connectors(&self) -> Arc<car_connectors::ConnectorManager> {
4452 self.connectors
4453 .get_or_init(|| {
4454 let mgr = car_connectors::ConnectorManager::new(self.mcp_executor.clone())
4455 .unwrap_or_else(|_| {
4456 // HOME unresolved (rare): fall back to a CWD-relative
4457 // manifest so connectors still work this session.
4458 car_connectors::ConnectorManager::with_path(
4459 self.mcp_executor.clone(),
4460 std::path::PathBuf::from("connectors.json"),
4461 )
4462 });
4463 Arc::new(mgr)
4464 })
4465 .clone()
4466 }
4467
4468 /// Load persisted connectors and dial them, **once** per process.
4469 /// Idempotent: the first caller does the work; later callers return
4470 /// immediately. `car-server` calls this at boot so connector tools
4471 /// are live before clients connect; the `connectors.*` handlers call
4472 /// it too, so an embedder that skips the boot call still gets a
4473 /// lazy load on first use. Enabled tools discovered here are
4474 /// registered into any already-open sessions.
4475 pub async fn ensure_connectors_loaded(&self) {
4476 use std::sync::atomic::Ordering;
4477 if self.connectors_loaded.swap(true, Ordering::SeqCst) {
4478 return;
4479 }
4480 let mgr = self.connectors();
4481 match mgr.load_and_connect().await {
4482 Ok(entries) => self.register_connector_entries(&entries).await,
4483 Err(e) => tracing::warn!("connector load failed: {e}"),
4484 }
4485 }
4486
4487 /// Register connector [`car_engine::ToolEntry`]s into every currently-open
4488 /// session's runtime registry, so a tool enabled mid-session
4489 /// becomes visible without a reconnect. New sessions pick up the
4490 /// enabled set in [`create_session`](crate::session::ServerState::create_session).
4491 pub async fn register_connector_entries(&self, entries: &[car_engine::ToolEntry]) {
4492 if entries.is_empty() {
4493 return;
4494 }
4495 let sessions: Vec<Arc<ClientSession>> =
4496 self.sessions.lock().await.values().cloned().collect();
4497 for session in sessions {
4498 for entry in entries {
4499 session.runtime.register_tool_entry(entry.clone()).await;
4500 }
4501 }
4502 }
4503
4504 /// Unregister connector tools (by canonical name) from every open
4505 /// session's runtime, so a disabled or removed connector's tools
4506 /// stop being visible to the model and accepted by the validator.
4507 pub async fn unregister_connector_tools(&self, canonical_names: &[String]) {
4508 if canonical_names.is_empty() {
4509 return;
4510 }
4511 let sessions: Vec<Arc<ClientSession>> =
4512 self.sessions.lock().await.values().cloned().collect();
4513 for session in sessions {
4514 for name in canonical_names {
4515 session.runtime.unregister_tool(name).await;
4516 }
4517 }
4518 }
4519
4520 /// Whether any currently connected WS client holds the host-management
4521 /// role — authenticated via `session.auth { host_token }`
4522 /// (`ClientSession::is_host`), NOT the same thing as `host.subscribe`
4523 /// membership (see the doc comment on the auth check this mirrors,
4524 /// around `handle_session_auth`). In practice: is a CarHost / Command
4525 /// Deck app connected right now, so its browser drawer is a real
4526 /// visible surface?
4527 ///
4528 /// Scans the (small) live session set fresh on every call — no caching,
4529 /// so this is always current. The deciding signal for Task 7's
4530 /// headless/headed browser-launch default and its sign-in host-gone
4531 /// fallback (`assistant::browser_tools::HostConnectivity`).
4532 pub async fn any_host_connected(&self) -> bool {
4533 self.sessions
4534 .lock()
4535 .await
4536 .values()
4537 .any(|s| s.is_host.load(std::sync::atomic::Ordering::Acquire))
4538 }
4539
4540 /// Lazy-initialize and return the agent supervisor. The first
4541 /// call constructs a [`car_registry::supervisor::Supervisor`] backed by
4542 /// `~/.car/agents.json` + `~/.car/logs/`. Embedders that need a
4543 /// non-default location should call
4544 /// [`ServerState::install_supervisor`] before any handler runs.
4545 ///
4546 /// In observer mode (set via [`install_observer_manifest`](crate::session::ServerState::install_observer_manifest)),
4547 /// returns a clear error mentioning the manifest path the
4548 /// primary daemon owns. This prevents the second daemon from
4549 /// re-attempting `user_default()` (which would also fail with
4550 /// `AlreadyRunning`) on every WS call, and gives mutation
4551 /// handlers a stable refusal path. Read-only handlers
4552 /// (`agents.list`, `agents.health`) should call
4553 /// [`Self::observer_manifest_path`] first and fall back to
4554 /// [`car_registry::supervisor::Supervisor::list_from_manifest`] /
4555 /// `health_from_manifest` when set. Closes
4556 /// Parslee-ai/car-releases#44.
4557 /// The declarative-agent registry, lazy-initialized on first use.
4558 pub fn declagents(&self) -> Result<Arc<car_registry::declarative::DeclRegistry>, String> {
4559 if let Some(r) = self.declagents.get() {
4560 return Ok(r.clone());
4561 }
4562 let r = Arc::new(car_registry::declarative::DeclRegistry::user_default()?);
4563 let _ = self.declagents.set(r);
4564 Ok(self.declagents.get().expect("set or pre-existing").clone())
4565 }
4566
4567 pub fn routing(&self) -> Result<Arc<car_registry::routing::RoutingStore>, String> {
4568 if let Some(r) = self.routing.get() {
4569 return Ok(r.clone());
4570 }
4571 let r = Arc::new(car_registry::routing::RoutingStore::user_default()?);
4572 let _ = self.routing.set(r);
4573 Ok(self.routing.get().expect("set or pre-existing").clone())
4574 }
4575
4576 pub fn supervisor(&self) -> Result<Arc<car_registry::supervisor::Supervisor>, String> {
4577 if let Some(s) = self.supervisor.get() {
4578 return Ok(s.clone());
4579 }
4580 if let Some(p) = self.observer_manifest_path.get() {
4581 return Err(format!(
4582 "this car-server is observe-only — another car-server process \
4583 holds the supervisor lock for {}. Mutations refuse here; route \
4584 them to the primary daemon, or stop the other car-server first.",
4585 p.display()
4586 ));
4587 }
4588 let s = car_registry::supervisor::Supervisor::user_default()
4589 .map(Arc::new)
4590 .map_err(|e| e.to_string())?;
4591 // OnceLock::set returns the original arg back on collision —
4592 // a concurrent caller racing through user_default. Take
4593 // whichever wins.
4594 let _ = self.supervisor.set(s);
4595 Ok(self.supervisor.get().expect("set or pre-existing").clone())
4596 }
4597
4598 /// Replace the lazy default with a caller-supplied supervisor.
4599 /// Returns `Err(())` when a supervisor was already installed.
4600 /// Used by the standalone `car-server` binary to call
4601 /// `start_all()` on a known-good handle without paying the
4602 /// lazy-init lookup cost.
4603 pub fn install_supervisor(
4604 &self,
4605 supervisor: Arc<car_registry::supervisor::Supervisor>,
4606 ) -> Result<(), Arc<car_registry::supervisor::Supervisor>> {
4607 self.supervisor.set(supervisor)
4608 }
4609
4610 /// Non-acquiring read of the currently-installed supervisor.
4611 /// Unlike [`supervisor`](Self::supervisor), this does NOT lazy-
4612 /// init via `user_default()` — it returns `None` instead of
4613 /// constructing a fresh `Supervisor` and acquiring the
4614 /// `<manifest>.lock` as a side effect. Use this from read-only
4615 /// metadata paths (`host.subscribe` identity, status surfaces)
4616 /// where causing lock acquisition on observation would be a
4617 /// Heisenberg subscribe — the act of asking "do you own the
4618 /// lock?" must not be the act of taking it.
4619 pub fn supervisor_if_installed(&self) -> Option<Arc<car_registry::supervisor::Supervisor>> {
4620 self.supervisor.get().cloned()
4621 }
4622
4623 /// Mark this daemon as *observing* a manifest owned by another
4624 /// car-server process. After this call, `supervisor()` returns
4625 /// an "observe-only" error and read-only handlers
4626 /// (`agents.list`, `agents.health`) fall back to the static
4627 /// `Supervisor::list_from_manifest` / `health_from_manifest`
4628 /// paths. Idempotent — subsequent calls with the same path are
4629 /// no-ops; a different path returns `Err(())`. Closes
4630 /// Parslee-ai/car-releases#44.
4631 pub fn install_observer_manifest(&self, path: PathBuf) -> Result<(), PathBuf> {
4632 self.observer_manifest_path.set(path)
4633 }
4634
4635 /// Path of the manifest this daemon is observing but not
4636 /// supervising. `None` when this daemon owns the supervisor
4637 /// (the normal case) or when no manifest is configured at all
4638 /// (no `HOME`, embedder didn't install one).
4639 pub fn observer_manifest_path(&self) -> Option<&PathBuf> {
4640 self.observer_manifest_path.get()
4641 }
4642
4643 /// Lazy-initialize and return the in-core A2A dispatcher. The
4644 /// first call constructs an [`car_a2a::A2aDispatcher`] from
4645 /// either the embedder's overrides (set via
4646 /// [`ServerStateConfig::with_a2a_runtime`] / `with_a2a_store` /
4647 /// `with_a2a_card_source`) or sensible defaults: a fresh
4648 /// `Runtime` with `register_agent_basics` registered, an
4649 /// `InMemoryTaskStore`, and a card built from the runtime's
4650 /// tool schemas advertising `ws://127.0.0.1:9100/` as the
4651 /// public URL. Closes Parslee-ai/car-releases#28.
4652 pub async fn a2a_dispatcher(&self) -> Arc<car_a2a::A2aDispatcher> {
4653 if let Some(d) = self.a2a_dispatcher.get() {
4654 return d.clone();
4655 }
4656
4657 // Embedder overrides take precedence; fall back to defaults
4658 // for each slot independently (so an embedder that only
4659 // wants a custom card can leave the runtime + store at
4660 // defaults). `Mutex::take()` consumes the slot so the
4661 // defaults aren't reconstructed on a racing init that loses
4662 // the OnceLock::set call below.
4663 let runtime = self
4664 .a2a_runtime
4665 .lock()
4666 .expect("a2a_runtime mutex poisoned")
4667 .take();
4668 let runtime = match runtime {
4669 Some(r) => r,
4670 None => {
4671 let r = Arc::new(car_engine::Runtime::new());
4672 r.register_agent_basics().await;
4673 r
4674 }
4675 };
4676
4677 let store = self
4678 .a2a_store
4679 .lock()
4680 .expect("a2a_store mutex poisoned")
4681 .take()
4682 .unwrap_or_else(|| Arc::new(car_a2a::InMemoryTaskStore::new()));
4683
4684 let card_source = self
4685 .a2a_card_source
4686 .lock()
4687 .expect("a2a_card_source mutex poisoned")
4688 .take();
4689 let card_source = match card_source {
4690 Some(c) => c,
4691 None => {
4692 let card = car_a2a::build_default_agent_card(
4693 &runtime,
4694 car_a2a::AgentCardConfig::minimal(
4695 "Common Agent Runtime",
4696 "Embedded CAR daemon — A2A v1.0 reachable over WebSocket JSON-RPC.",
4697 "ws://127.0.0.1:9100/",
4698 car_a2a::AgentProvider {
4699 organization: "Parslee".into(),
4700 url: Some("https://github.com/Parslee-ai/car".into()),
4701 },
4702 ),
4703 )
4704 .await;
4705 Arc::new(move || card.clone()) as Arc<car_a2a::AgentCardSource>
4706 }
4707 };
4708
4709 let dispatcher = Arc::new(car_a2a::A2aDispatcher::new(runtime, store, card_source));
4710 // OnceLock::set returns Err on race — accept whichever
4711 // dispatcher won and clone-return that one.
4712 let _ = self.a2a_dispatcher.set(dispatcher);
4713 self.a2a_dispatcher
4714 .get()
4715 .expect("a2a_dispatcher set or pre-existing")
4716 .clone()
4717 }
4718
4719 /// Record the start of a run and return its [`RunMeta`]. The
4720 /// `run_id` is minted by the caller (the `runs.start` handler)
4721 /// so it can set `session.current_run_id` to the same value
4722 /// **before** responding (KTD3). Idempotent collision on an
4723 /// already-present `run_id` is treated as a fresh insert (uuids
4724 /// don't collide in practice; if one did, the latest start wins).
4725 ///
4726 /// U3: also writes the `RunStarted` line to the disk store + creates
4727 /// the run file, so the run is on disk from the first record (REPLAY
4728 /// survives a restart even before any turn lands). Disk failures are
4729 /// logged, never fatal — the in-memory registry is the live path.
4730 /// Fan one `runs.trace.event` out to every live subscriber of
4731 /// `run_id` (agent run tracing, U4). Called by the lifecycle methods
4732 /// **while they hold the `runs` lock** so append-and-notify is
4733 /// serialized with subscribe's snapshot-and-register (invariant #1).
4734 ///
4735 /// Each push is a non-blocking `try_send` onto the subscriber's
4736 /// bounded channel — the WS socket is written only by that
4737 /// subscriber's dedicated drain task, never here (invariant #2). A
4738 /// full channel drops the event (the slow-subscriber case); the
4739 /// client detects the cursor gap and re-subscribes (R8). Takes the
4740 /// already-acquired subscribers guard so the caller controls the lock
4741 /// scope and the `runs` → `run_subscribers` order.
4742 fn fanout_locked(
4743 subscribers: &HashMap<(String, String), crate::host::RunTraceSubscriber>,
4744 run_id: &str,
4745 agent_id: &str,
4746 record: car_proto::RunRecord,
4747 cursor: usize,
4748 status: car_proto::RunLiveStatus,
4749 ) {
4750 for ((sub_run, _client), sub) in subscribers.iter() {
4751 if sub_run != run_id {
4752 continue;
4753 }
4754 let event = car_proto::RunTraceEvent {
4755 run_id: run_id.to_string(),
4756 agent_id: agent_id.to_string(),
4757 record: record.clone(),
4758 cursor,
4759 status,
4760 };
4761 // Best-effort: a wedged subscriber's full channel drops the
4762 // event rather than stalling the producer. Logged at debug —
4763 // the client backfills via re-subscribe.
4764 if !sub.push(event) {
4765 tracing::debug!(
4766 run_id,
4767 "run-trace: dropped event for slow subscriber (channel full)"
4768 );
4769 }
4770 }
4771 }
4772
4773 /// Atomically reserve a run id/idempotency key across every live session.
4774 /// A per-run durability lock serializes the disk absence check with the
4775 /// final process-wide map insert. The global `runs` lock is held only for
4776 /// the two bounded map checks, never for filesystem enumeration/replay.
4777 pub async fn reserve_run(&self, meta: RunMeta) -> Result<RunReservation, String> {
4778 fn existing_reservation(
4779 existing: &RunMeta,
4780 requested: &RunMeta,
4781 ) -> Result<RunReservation, String> {
4782 if existing.client_id != requested.client_id {
4783 return Err(format!(
4784 "{} run `{}` belongs to client_id `{}`",
4785 car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX,
4786 requested.run_id,
4787 existing.client_id
4788 ));
4789 }
4790 if existing.agent_id != requested.agent_id {
4791 return Err(format!(
4792 "{} run `{}` was retried under a different agent",
4793 car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX,
4794 requested.run_id
4795 ));
4796 }
4797 let changed_fields = [
4798 (existing.intent != requested.intent).then_some("intent"),
4799 (existing.outcome_description != requested.outcome_description)
4800 .then_some("outcome_description"),
4801 ]
4802 .into_iter()
4803 .flatten()
4804 .collect::<Vec<_>>();
4805 if !changed_fields.is_empty() {
4806 return Err(format!(
4807 "{} idempotency conflict for run `{}`: retry changed occurrence-defining {}",
4808 car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX,
4809 requested.run_id,
4810 changed_fields.join(" and ")
4811 ));
4812 }
4813 Ok(RunReservation::Existing(existing.clone()))
4814 }
4815
4816 let durability_lock = self.run_durability_lock(&meta.run_id).await;
4817 let _durability_guard = durability_lock.lock().await;
4818 {
4819 let runs = self.runs.lock().await;
4820 if let Some(existing) = runs.get(&meta.run_id) {
4821 return existing_reservation(existing, &meta);
4822 }
4823 }
4824 let store = self.run_store.clone();
4825 let durable_run_id = meta.run_id.clone();
4826 let durable = tokio::task::spawn_blocking(move || store.run_started(&durable_run_id))
4827 .await
4828 .map_err(|error| format!("RunStarted reservation read task failed: {error}"))?
4829 .map_err(|error| format!("RunStarted reservation read failed: {error}"))?;
4830 if let Some(started) = durable {
4831 return Err(format!(
4832 "{} persisted run `{}` belongs to client_id `{}` and cannot be adopted",
4833 car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX,
4834 meta.run_id,
4835 started.client_id.as_deref().unwrap_or("historical/unknown")
4836 ));
4837 }
4838 let mut runs = self.runs.lock().await;
4839 if let Some(existing) = runs.get(&meta.run_id) {
4840 return existing_reservation(existing, &meta);
4841 }
4842 runs.insert(meta.run_id.clone(), meta);
4843 Ok(RunReservation::New)
4844 }
4845
4846 /// Release a newly reserved candidate when `runs.start` rejects before it
4847 /// binds or persists that candidate. The exact pristine metadata and the
4848 /// absent RunStore boundary are both re-checked under the per-run durability
4849 /// lock. This deliberately does not touch `current_run_id` or the journal
4850 /// binding, which may still belong to an older live run on this session.
4851 pub(crate) async fn release_unpersisted_run_reservation(
4852 &self,
4853 session: &ClientSession,
4854 expected: &RunMeta,
4855 ) -> Result<(), String> {
4856 if expected.client_id != session.client_id
4857 || expected.start_committed
4858 || expected.termination.is_some()
4859 || expected.ended_at.is_some()
4860 || !expected.turns.is_empty()
4861 || expected.pending_terminal.is_some()
4862 {
4863 return Err(format!(
4864 "run `{}` is not a pristine reservation owned by this session",
4865 expected.run_id
4866 ));
4867 }
4868 let still_expected = |existing: &RunMeta| {
4869 existing.run_id == expected.run_id
4870 && existing.client_id == expected.client_id
4871 && existing.agent_id == expected.agent_id
4872 && existing.intent == expected.intent
4873 && existing.outcome_description == expected.outcome_description
4874 && existing.started_at == expected.started_at
4875 && existing.durability_generation == expected.durability_generation
4876 && !existing.start_committed
4877 && existing.termination.is_none()
4878 && existing.ended_at.is_none()
4879 && existing.turns.is_empty()
4880 && existing.pending_terminal.is_none()
4881 };
4882 let durability_lock = self.run_durability_lock(&expected.run_id).await;
4883 let _durability_guard = durability_lock.lock().await;
4884 {
4885 let runs = self.runs.lock().await;
4886 let existing = runs
4887 .get(&expected.run_id)
4888 .ok_or_else(|| format!("reserved run `{}` is absent", expected.run_id))?;
4889 if !still_expected(existing) {
4890 return Err(format!(
4891 "reserved run `{}` changed before pre-start release",
4892 expected.run_id
4893 ));
4894 }
4895 }
4896 let store = self.run_store.clone();
4897 let durable_run_id = expected.run_id.clone();
4898 if tokio::task::spawn_blocking(move || store.run_started(&durable_run_id))
4899 .await
4900 .map_err(|error| format!("RunStarted release read task failed: {error}"))?
4901 .map_err(|error| format!("RunStarted release read failed: {error}"))?
4902 .is_some()
4903 {
4904 return Err(format!(
4905 "reserved run `{}` reached durable storage and cannot be released",
4906 expected.run_id
4907 ));
4908 }
4909 let mut runs = self.runs.lock().await;
4910 let existing = runs
4911 .get(&expected.run_id)
4912 .ok_or_else(|| format!("reserved run `{}` is absent", expected.run_id))?;
4913 if !still_expected(existing) {
4914 return Err(format!(
4915 "reserved run `{}` changed during pre-start release",
4916 expected.run_id
4917 ));
4918 }
4919 runs.remove(&expected.run_id);
4920 Ok(())
4921 }
4922
4923 pub async fn persist_run_start(&self, run_id: &str) -> Result<car_proto::RunStarted, String> {
4924 let durability_lock = self.run_durability_lock(run_id).await;
4925 let _durability_guard = durability_lock.lock().await;
4926 let started = {
4927 let runs = self.runs.lock().await;
4928 let meta = runs
4929 .get(run_id)
4930 .ok_or_else(|| format!("unknown reserved run_id `{run_id}`"))?;
4931 car_proto::RunStarted {
4932 run_id: meta.run_id.clone(),
4933 client_id: Some(meta.client_id.clone()),
4934 agent_id: meta.agent_id.clone(),
4935 intent: meta.intent.clone(),
4936 outcome_description: meta.outcome_description.clone(),
4937 started_at: meta.started_at,
4938 }
4939 };
4940 let store = self.run_store.clone();
4941 let durable_started = started.clone();
4942 tokio::task::spawn_blocking(move || store.write_started(&durable_started))
4943 .await
4944 .map_err(|error| format!("RunStarted durability task failed: {error}"))?
4945 .map_err(|error| format!("RunStarted durability failed: {error}"))?;
4946 Ok(started)
4947 }
4948
4949 pub async fn commit_run_start(&self, run_id: &str) -> Result<(), String> {
4950 let mut runs = self.runs.lock().await;
4951 let meta = runs
4952 .get_mut(run_id)
4953 .ok_or_else(|| format!("unknown reserved run_id `{run_id}`"))?;
4954 if meta.start_committed {
4955 return Ok(());
4956 }
4957 meta.start_committed = true;
4958 meta.durability_generation = meta.durability_generation.wrapping_add(1);
4959 let started = car_proto::RunStarted {
4960 run_id: meta.run_id.clone(),
4961 client_id: Some(meta.client_id.clone()),
4962 agent_id: meta.agent_id.clone(),
4963 intent: meta.intent.clone(),
4964 outcome_description: meta.outcome_description.clone(),
4965 started_at: meta.started_at,
4966 };
4967 let subs = self.run_subscribers.lock().await;
4968 Self::fanout_locked(
4969 &subs,
4970 &started.run_id,
4971 &started.agent_id,
4972 car_proto::RunRecord::Started(started.clone()),
4973 0,
4974 car_proto::RunLiveStatus::InProgress,
4975 );
4976 Ok(())
4977 }
4978
4979 /// Resolve a `runs.start` transaction whose acknowledgement failed before
4980 /// the owning socket disconnected. Returns `true` when no durable start
4981 /// boundary existed and the reservation was released. Returns `false`
4982 /// after reconciling an exact durable `RunStarted`, which the disconnect
4983 /// sweep must then close as one historical `Incomplete` occurrence.
4984 pub(crate) async fn reconcile_or_release_unacknowledged_start(
4985 &self,
4986 session: &ClientSession,
4987 run_id: &str,
4988 ) -> Result<bool, String> {
4989 let durability_lock = self.run_durability_lock(run_id).await;
4990 let _durability_guard = durability_lock.lock().await;
4991 let expected = {
4992 let runs = self.runs.lock().await;
4993 let meta = runs
4994 .get(run_id)
4995 .ok_or_else(|| format!("unknown reserved run_id `{run_id}`"))?;
4996 if meta.client_id != session.client_id {
4997 return Err(format!(
4998 "{} run `{run_id}` belongs to client_id `{}`",
4999 car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX,
5000 meta.client_id
5001 ));
5002 }
5003 if meta.start_committed {
5004 return Ok(false);
5005 }
5006 if meta.is_terminal() || meta.pending_terminal.is_some() {
5007 return Err(format!(
5008 "uncommitted run `{run_id}` unexpectedly has terminal state"
5009 ));
5010 }
5011 car_proto::RunStarted {
5012 run_id: meta.run_id.clone(),
5013 client_id: Some(meta.client_id.clone()),
5014 agent_id: meta.agent_id.clone(),
5015 intent: meta.intent.clone(),
5016 outcome_description: meta.outcome_description.clone(),
5017 started_at: meta.started_at,
5018 }
5019 };
5020
5021 let store = self.run_store.clone();
5022 let durable_run_id = run_id.to_string();
5023 let durable = tokio::task::spawn_blocking(move || store.run_started(&durable_run_id))
5024 .await
5025 .map_err(|error| format!("RunStarted recovery read task failed: {error}"))?
5026 .map_err(|error| format!("RunStarted recovery read failed: {error}"))?;
5027 match durable {
5028 Some(durable) => {
5029 if durable != expected {
5030 return Err(format!(
5031 "durable RunStarted identity for `{run_id}` does not match its live reservation"
5032 ));
5033 }
5034 // Re-run both exact idempotent durability barriers. The first
5035 // retry turns a prior write-without-fsync into an acknowledged
5036 // RunStore boundary; the second either appends RunStarted or
5037 // fsyncs the exact journal row already pending in memory.
5038 let store = self.run_store.clone();
5039 let durable_expected = expected.clone();
5040 tokio::task::spawn_blocking(move || store.write_started(&durable_expected))
5041 .await
5042 .map_err(|error| {
5043 format!("RunStarted recovery durability task failed: {error}")
5044 })?
5045 .map_err(|error| format!("RunStarted recovery durability failed: {error}"))?;
5046 session.append_run_started_event(&expected).await?;
5047 self.commit_run_start(run_id).await?;
5048 Ok(false)
5049 }
5050 None => {
5051 // `runs.start` orders RunStore before journal append, so an
5052 // absent RunStore boundary proves no lifecycle row was ever
5053 // admitted to the journal. Remove only the zero-byte file an
5054 // open-before-write failure may have left behind.
5055 let store = self.run_store.clone();
5056 let durable_expected = expected.clone();
5057 tokio::task::spawn_blocking(move || {
5058 store.rollback_empty_run_start(&durable_expected)
5059 })
5060 .await
5061 .map_err(|error| format!("empty RunStarted rollback task failed: {error}"))?
5062 .map_err(|error| format!("empty RunStarted rollback failed: {error}"))?;
5063 {
5064 let mut runs = self.runs.lock().await;
5065 let meta = runs.get(run_id).ok_or_else(|| {
5066 format!("reserved run `{run_id}` disappeared during rollback")
5067 })?;
5068 let still_expected = meta.client_id == session.client_id
5069 && !meta.start_committed
5070 && !meta.is_terminal()
5071 && meta.pending_terminal.is_none()
5072 && meta.agent_id == expected.agent_id
5073 && meta.intent == expected.intent
5074 && meta.outcome_description == expected.outcome_description
5075 && meta.started_at == expected.started_at;
5076 if !still_expected {
5077 return Err(format!(
5078 "reserved run `{run_id}` changed during empty-start rollback"
5079 ));
5080 }
5081 runs.remove(run_id);
5082 }
5083 session.clear_run_journal_binding(run_id).await?;
5084 let mut current = session.current_run_id.lock().await;
5085 if current.as_deref() == Some(run_id) {
5086 *current = None;
5087 }
5088 Ok(true)
5089 }
5090 }
5091 }
5092
5093 /// Compatibility/internal path for CAR-owned in-process callers that do
5094 /// not expose `runs.start`. The public RPC uses reserve/persist/journal/
5095 /// commit explicitly so its acknowledgement includes both durable stores.
5096 pub async fn start_run(&self, meta: RunMeta) -> Result<(), String> {
5097 let run_id = meta.run_id.clone();
5098 self.reserve_run(meta).await?;
5099 self.persist_run_start(&run_id).await?;
5100 self.commit_run_start(&run_id).await
5101 }
5102
5103 /// Make a run terminal with a harness-reported outcome
5104 /// (`runs.complete`). Returns `Err` if the `run_id` is unknown or
5105 /// already terminal — the handler maps that to a JSON-RPC error so
5106 /// a double-complete or stale id is visible, not silently swallowed.
5107 ///
5108 /// U3: also appends the terminal `RunEnded` line to the run's JSONL
5109 /// file so REPLAY (U5) reports the right status (Completed) after a
5110 /// restart. The disk write happens after the lock is released; disk
5111 /// failures are logged, never fatal.
5112 pub async fn prepare_run_completion(
5113 &self,
5114 run_id: &str,
5115 termination: car_proto::RunTermination,
5116 ) -> Result<car_proto::RunEnded, String> {
5117 self.prepare_run_completion_for_active_owner(run_id, termination, None)
5118 .await
5119 }
5120
5121 pub(crate) async fn prepare_run_completion_for_active_owner(
5122 &self,
5123 run_id: &str,
5124 termination: car_proto::RunTermination,
5125 expected_active_client_id: Option<&str>,
5126 ) -> Result<car_proto::RunEnded, String> {
5127 let completion_digest = run_completion_digest(&termination)?;
5128 if expected_active_client_id.is_some() {
5129 if let Some(gate) = &self.run_completion_fence_gate {
5130 gate.wait_if_armed().await;
5131 }
5132 }
5133 let durability_lock = self.run_durability_lock(run_id).await;
5134 let _durability_guard = durability_lock.lock().await;
5135 let ended = {
5136 let mut runs = self.runs.lock().await;
5137 let meta = runs
5138 .get_mut(run_id)
5139 .ok_or_else(|| format!("unknown run_id `{run_id}`"))?;
5140 if let Some(error) = &meta.trace_corruption {
5141 return Err(error.clone());
5142 }
5143 if !meta.start_committed {
5144 return Err(format!("run `{run_id}` start is not durably committed"));
5145 }
5146 if expected_active_client_id.is_some_and(|expected| meta.active_client_id != expected) {
5147 return Err(format!(
5148 "run `{run_id}` active owner changed before terminalization"
5149 ));
5150 }
5151 if let Some(requested) = &meta.cancellation_pending {
5152 let matches_cancel = matches!(
5153 &termination,
5154 car_proto::RunTermination::Cancelled { cancellation }
5155 if cancellation.run_id == requested.run_id
5156 && cancellation.idempotency_key == requested.idempotency_key
5157 && cancellation.reason_digest == requested.reason_digest
5158 && cancellation.principal == requested.principal
5159 && cancellation.action_id == requested.action_id
5160 && cancellation.request_id == requested.request_id
5161 );
5162 if !matches_cancel {
5163 return Err(format!(
5164 "run `{run_id}` has a durable cancellation request and is quarantined"
5165 ));
5166 }
5167 }
5168 if let Some(existing) = &meta.termination {
5169 if !same_run_termination(existing, &termination) {
5170 return Err(format!("run `{run_id}` already has a different terminal"));
5171 }
5172 meta.resume_lease = None;
5173 car_proto::RunEnded {
5174 run_id: run_id.to_string(),
5175 client_id: Some(meta.client_id.clone()),
5176 agent_id: meta.agent_id.clone(),
5177 termination,
5178 completion_digest: Some(completion_digest),
5179 ended_at: meta
5180 .ended_at
5181 .ok_or_else(|| "terminal run is missing ended_at".to_string())?,
5182 }
5183 } else if let Some(pending) = &meta.pending_terminal {
5184 if !same_run_termination(&pending.termination, &termination)
5185 || pending.completion_digest.as_deref() != Some(completion_digest.as_str())
5186 {
5187 return Err(format!(
5188 "run `{run_id}` has a different pending terminal transaction"
5189 ));
5190 }
5191 meta.resume_lease = None;
5192 pending.clone()
5193 } else {
5194 let ended = car_proto::RunEnded {
5195 run_id: run_id.to_string(),
5196 client_id: Some(meta.client_id.clone()),
5197 agent_id: meta.agent_id.clone(),
5198 termination,
5199 completion_digest: Some(completion_digest),
5200 ended_at: chrono::Utc::now(),
5201 };
5202 meta.pending_terminal = Some(ended.clone());
5203 meta.resume_lease = None;
5204 meta.durability_generation = meta.durability_generation.wrapping_add(1);
5205 ended
5206 }
5207 };
5208 let store = self.run_store.clone();
5209 let durable_ended = ended.clone();
5210 let persisted = tokio::task::spawn_blocking(move || store.write_ended(&durable_ended))
5211 .await
5212 .map_err(|error| format!("RunEnded durability task failed: {error}"))?;
5213 if let Err(error) = persisted {
5214 if crate::run_store::is_trace_corruption_error(&error) {
5215 let mut runs = self.runs.lock().await;
5216 let message = self
5217 .quarantine_run_trace_locked(&mut runs, run_id, error.to_string())
5218 .await;
5219 return Err(message);
5220 }
5221 return Err(format!("RunEnded durability failed: {error}"));
5222 }
5223 Ok(ended)
5224 }
5225
5226 pub async fn commit_run_completion(&self, ended: &car_proto::RunEnded) -> Result<(), String> {
5227 {
5228 let mut runs = self.runs.lock().await;
5229 let meta = runs
5230 .get_mut(&ended.run_id)
5231 .ok_or_else(|| format!("unknown run_id `{}`", ended.run_id))?;
5232 if let Some(existing) = &meta.termination {
5233 if same_run_termination(existing, &ended.termination)
5234 && meta.ended_at == Some(ended.ended_at)
5235 {
5236 return Ok(());
5237 }
5238 return Err(format!(
5239 "run `{}` already has a different terminal",
5240 ended.run_id
5241 ));
5242 }
5243 if !meta
5244 .pending_terminal
5245 .as_ref()
5246 .is_some_and(|pending| same_run_ended(pending, ended))
5247 {
5248 return Err(format!(
5249 "run `{}` terminal commit does not match its prepared transaction",
5250 ended.run_id
5251 ));
5252 }
5253 meta.termination = Some(ended.termination.clone());
5254 meta.ended_at = Some(ended.ended_at);
5255 meta.pending_terminal = None;
5256 meta.resume_lease = None;
5257 meta.cancellation_pending = None;
5258 meta.cancellation_receipt = None;
5259 meta.durability_generation = meta.durability_generation.wrapping_add(1);
5260 let cursor = meta.turn_cursor();
5261 let status = meta.live_status();
5262 let agent_id = meta.agent_id.clone();
5263 let subs = self.run_subscribers.lock().await;
5264 Self::fanout_locked(
5265 &subs,
5266 &ended.run_id,
5267 &agent_id,
5268 car_proto::RunRecord::Ended(ended.clone()),
5269 cursor,
5270 status,
5271 );
5272 }
5273 // Heap hygiene: the run is now terminal and fully flushed to disk,
5274 // and the terminal `runs.trace.event` has already fanned out to any
5275 // live subscriber. Drop the in-memory per-turn buffer (each entry
5276 // holds a full prompt + CLI output, potentially MBs) so completed
5277 // runs don't pin the heaviest payloads for the daemon's lifetime.
5278 // The lightweight `RunMeta` header (agent_id, status, started_at,
5279 // termination) stays resident so `run_meta`/status lookups still
5280 // work. A late `runs.subscribe` to this terminal run re-sources its
5281 // snapshot from disk (see `subscribe_run`). Only terminal runs are
5282 // cleared; an in-progress run keeps its turns (the live snapshot
5283 // source). `Vec::new()` frees the buffer's capacity, not just its
5284 // length.
5285 self.clear_terminal_run_turns(&ended.run_id).await;
5286 Ok(())
5287 }
5288
5289 pub async fn complete_run(
5290 &self,
5291 run_id: &str,
5292 termination: car_proto::RunTermination,
5293 ) -> Result<car_proto::RunEnded, String> {
5294 let ended = self.prepare_run_completion(run_id, termination).await?;
5295 self.commit_run_completion(&ended).await?;
5296 Ok(ended)
5297 }
5298
5299 /// Commit the body-free request to RunStore and the authenticated opening
5300 /// client's journal under the same per-run durability lock used by
5301 /// terminal completion. Memory changes only after both receipts exist.
5302 pub async fn persist_run_cancellation_requested(
5303 &self,
5304 owner_session: &ClientSession,
5305 requested: &car_proto::RunCancellationRequested,
5306 ) -> Result<(), String> {
5307 let run_id = requested.run_id.as_str();
5308 let durability_lock = self.run_durability_lock(run_id).await;
5309 let _durability_guard = durability_lock.lock().await;
5310 let agent_id = {
5311 let runs = self.runs.lock().await;
5312 let meta = runs
5313 .get(run_id)
5314 .ok_or_else(|| format!("unknown run_id `{run_id}`"))?;
5315 if meta.is_terminal() || meta.pending_terminal.is_some() {
5316 return Err(format!("run `{run_id}` is already terminal"));
5317 }
5318 if meta.client_id != owner_session.client_id || !meta.start_committed {
5319 return Err("run cancellation client/run binding mismatch".into());
5320 }
5321 if let Some(existing) = &meta.cancellation_pending {
5322 if existing != requested {
5323 return Err(format!(
5324 "run `{run_id}` already has a different cancellation request"
5325 ));
5326 }
5327 return Ok(());
5328 }
5329 meta.agent_id.clone()
5330 };
5331 let store = self.run_store.clone();
5332 let durable_agent = agent_id.clone();
5333 let durable_requested = requested.clone();
5334 tokio::task::spawn_blocking(move || {
5335 store.write_cancellation_requested(&durable_agent, &durable_requested)
5336 })
5337 .await
5338 .map_err(|error| format!("cancellation request durability task failed: {error}"))?
5339 .map_err(|error| format!("cancellation request durability failed: {error}"))?;
5340 owner_session
5341 .append_run_cancellation_requested_event(requested)
5342 .await?;
5343 let mut runs = self.runs.lock().await;
5344 let meta = runs
5345 .get_mut(run_id)
5346 .ok_or_else(|| format!("unknown run_id `{run_id}`"))?;
5347 meta.cancellation_pending = Some(requested.clone());
5348 meta.durability_generation = meta.durability_generation.wrapping_add(1);
5349 let cursor = meta.turn_cursor();
5350 let agent_id = meta.agent_id.clone();
5351 let subs = self.run_subscribers.lock().await;
5352 Self::fanout_locked(
5353 &subs,
5354 run_id,
5355 &agent_id,
5356 car_proto::RunRecord::CancellationRequested(requested.clone()),
5357 cursor,
5358 car_proto::RunLiveStatus::CancellationPending,
5359 );
5360 Ok(())
5361 }
5362
5363 pub async fn persist_run_cancellation_result(
5364 &self,
5365 owner_session: &ClientSession,
5366 result: &car_proto::RunCancelResponse,
5367 ) -> Result<(), String> {
5368 let run_id = result.run_id.as_str();
5369 let durability_lock = self.run_durability_lock(run_id).await;
5370 let _durability_guard = durability_lock.lock().await;
5371 let agent_id = {
5372 let runs = self.runs.lock().await;
5373 let meta = runs
5374 .get(run_id)
5375 .ok_or_else(|| format!("unknown run_id `{run_id}`"))?;
5376 if meta.is_terminal() {
5377 return Err(format!("run `{run_id}` is already terminal"));
5378 }
5379 let requested = meta
5380 .cancellation_pending
5381 .as_ref()
5382 .ok_or_else(|| format!("run `{run_id}` has no cancellation request"))?;
5383 if requested.idempotency_key != result.idempotency_key
5384 || requested.reason_digest != result.reason_digest
5385 || requested.principal != result.principal
5386 {
5387 return Err(format!("run `{run_id}` cancellation receipt mismatch"));
5388 }
5389 if let Some(existing) = &meta.cancellation_receipt {
5390 if existing != result {
5391 return Err(format!(
5392 "run `{run_id}` has a different cancellation receipt"
5393 ));
5394 }
5395 return Ok(());
5396 }
5397 meta.agent_id.clone()
5398 };
5399 let store = self.run_store.clone();
5400 let durable_agent = agent_id.clone();
5401 let durable_result = result.clone();
5402 tokio::task::spawn_blocking(move || {
5403 store.write_cancellation_result(&durable_agent, &durable_result)
5404 })
5405 .await
5406 .map_err(|error| format!("cancellation result durability task failed: {error}"))?
5407 .map_err(|error| format!("cancellation result durability failed: {error}"))?;
5408 owner_session
5409 .append_run_cancellation_result_event(result)
5410 .await?;
5411 let mut runs = self.runs.lock().await;
5412 let meta = runs
5413 .get_mut(run_id)
5414 .ok_or_else(|| format!("unknown run_id `{run_id}`"))?;
5415 meta.cancellation_receipt = Some(result.clone());
5416 meta.durability_generation = meta.durability_generation.wrapping_add(1);
5417 let cursor = meta.turn_cursor();
5418 let agent_id = meta.agent_id.clone();
5419 let subs = self.run_subscribers.lock().await;
5420 Self::fanout_locked(
5421 &subs,
5422 run_id,
5423 &agent_id,
5424 car_proto::RunRecord::CancellationResult(result.clone()),
5425 cursor,
5426 car_proto::RunLiveStatus::CancellationPending,
5427 );
5428 Ok(())
5429 }
5430
5431 /// Finish an unconfirmed cancellation after the opening socket has gone
5432 /// away. The durable `RunStarted` client id is the journal authority: a
5433 /// reconnect may request recovery, but it may not redirect the receipt to
5434 /// its own journal. RunStore, that authenticated journal, live state, and
5435 /// subscribers advance as one retryable transaction before success is
5436 /// returned.
5437 pub async fn persist_recovered_run_cancellation_result(
5438 &self,
5439 agent_id: &str,
5440 result: &car_proto::RunCancelResponse,
5441 ) -> Result<(), String> {
5442 if result.status == car_proto::RunCancellationStatus::AlreadyTerminal
5443 && result.terminal_digest.is_some()
5444 {
5445 return self
5446 .persist_recovered_orphan_terminal_cancellation(agent_id, result)
5447 .await;
5448 }
5449 if result.status != car_proto::RunCancellationStatus::TerminationUnconfirmed
5450 || result.terminal_digest.is_some()
5451 {
5452 return Err(
5453 "recovered cancellation receipt must be termination_unconfirmed or a host orphan terminal"
5454 .into(),
5455 );
5456 }
5457 let run_id = result.run_id.as_str();
5458 let durability_lock = self.run_durability_lock(run_id).await;
5459 let _durability_guard = durability_lock.lock().await;
5460 let store = self.run_store.clone();
5461 let durable_agent = agent_id.to_string();
5462 let durable_run = run_id.to_string();
5463 let records = tokio::task::spawn_blocking(move || {
5464 store.get_run_trace_for_checked(&durable_agent, &durable_run)
5465 })
5466 .await
5467 .map_err(|error| format!("recovered cancellation trace task failed: {error}"))?
5468 .map_err(|error| format!("recovered cancellation trace read failed: {error}"))?
5469 .ok_or_else(|| "recovered cancellation durable owner is unavailable".to_string())?;
5470 let started = records
5471 .iter()
5472 .find_map(|record| match record {
5473 car_proto::RunRecord::Started(started) => Some(started),
5474 _ => None,
5475 })
5476 .ok_or_else(|| "recovered cancellation is missing durable RunStarted".to_string())?;
5477 if started.agent_id != agent_id {
5478 return Err("recovered cancellation durable agent mismatch".into());
5479 }
5480 let client_id = started
5481 .client_id
5482 .as_deref()
5483 .filter(|client_id| {
5484 !client_id.is_empty()
5485 && !client_id.contains('/')
5486 && !client_id.contains('\\')
5487 && *client_id != "."
5488 && *client_id != ".."
5489 })
5490 .ok_or_else(|| {
5491 "recovered cancellation has no authenticated durable journal owner".to_string()
5492 })?
5493 .to_string();
5494 if records
5495 .iter()
5496 .any(|record| matches!(record, car_proto::RunRecord::Ended(_)))
5497 {
5498 return Err(format!("run `{run_id}` is already terminal"));
5499 }
5500 let requested = records
5501 .iter()
5502 .find_map(|record| match record {
5503 car_proto::RunRecord::CancellationRequested(requested) => Some(requested),
5504 _ => None,
5505 })
5506 .ok_or_else(|| format!("run `{run_id}` has no durable cancellation request"))?;
5507 if requested.receipt_version != result.receipt_version
5508 || requested.run_id != result.run_id
5509 || requested.idempotency_key != result.idempotency_key
5510 || requested.reason_digest != result.reason_digest
5511 || requested.principal != result.principal
5512 || requested.action_id != result.action_id
5513 || requested.request_id != result.request_id
5514 {
5515 return Err(format!(
5516 "run `{run_id}` recovered cancellation receipt mismatch"
5517 ));
5518 }
5519 if let Some(existing) = records.iter().find_map(|record| match record {
5520 car_proto::RunRecord::CancellationResult(existing) => Some(existing),
5521 _ => None,
5522 }) {
5523 if existing != result {
5524 return Err(format!(
5525 "run `{run_id}` has a different durable cancellation receipt"
5526 ));
5527 }
5528 }
5529 {
5530 let runs = self.runs.lock().await;
5531 if let Some(meta) = runs.get(run_id) {
5532 if meta.agent_id != agent_id
5533 || meta.client_id != client_id
5534 || !meta.start_committed
5535 || meta.is_terminal()
5536 || meta.cancellation_pending.as_ref() != Some(requested)
5537 {
5538 return Err(format!(
5539 "run `{run_id}` live state does not authenticate recovered cancellation"
5540 ));
5541 }
5542 if meta
5543 .cancellation_receipt
5544 .as_ref()
5545 .is_some_and(|existing| existing != result)
5546 {
5547 return Err(format!(
5548 "run `{run_id}` has a different live cancellation receipt"
5549 ));
5550 }
5551 }
5552 }
5553
5554 let store = self.run_store.clone();
5555 let durable_agent = agent_id.to_string();
5556 let durable_result = result.clone();
5557 tokio::task::spawn_blocking(move || {
5558 store.write_cancellation_result(&durable_agent, &durable_result)
5559 })
5560 .await
5561 .map_err(|error| format!("recovered cancellation durability task failed: {error}"))?
5562 .map_err(|error| format!("recovered cancellation durability failed: {error}"))?;
5563
5564 let journal_dir = self.journal_dir.clone();
5565 let journal_failures = self.journal_failures.clone();
5566 let journal_client = client_id.clone();
5567 let journal_result = result.clone();
5568 let journal_was_new = tokio::task::spawn_blocking(move || {
5569 append_recovered_cancellation_result_journal(
5570 journal_dir,
5571 journal_failures,
5572 journal_client,
5573 journal_result,
5574 )
5575 })
5576 .await
5577 .map_err(|error| format!("recovered cancellation journal task failed: {error}"))??;
5578
5579 let mut runs = self.runs.lock().await;
5580 let (cursor, fanout) = match runs.get_mut(run_id) {
5581 Some(meta) => {
5582 let fanout = meta.cancellation_receipt.is_none();
5583 meta.cancellation_receipt = Some(result.clone());
5584 if fanout {
5585 meta.durability_generation = meta.durability_generation.wrapping_add(1);
5586 }
5587 (meta.turn_cursor(), fanout)
5588 }
5589 None => (
5590 records
5591 .iter()
5592 .filter(|record| matches!(record, car_proto::RunRecord::Turn(_)))
5593 .count(),
5594 journal_was_new,
5595 ),
5596 };
5597 if fanout {
5598 let subscribers = self.run_subscribers.lock().await;
5599 Self::fanout_locked(
5600 &subscribers,
5601 run_id,
5602 agent_id,
5603 car_proto::RunRecord::CancellationResult(result.clone()),
5604 cursor,
5605 car_proto::RunLiveStatus::CancellationPending,
5606 );
5607 }
5608 Ok(())
5609 }
5610
5611 async fn persist_recovered_orphan_terminal_cancellation(
5612 &self,
5613 agent_id: &str,
5614 result: &car_proto::RunCancelResponse,
5615 ) -> Result<(), String> {
5616 if result.principal != "host" {
5617 return Err(
5618 "only the host may terminalize a run without live cancellation control".into(),
5619 );
5620 }
5621 let termination = car_proto::RunTermination::Incomplete;
5622 let completion_digest = run_completion_digest(&termination)?;
5623 if result.terminal_digest.as_deref() != Some(completion_digest.as_str()) {
5624 return Err("recovered orphan cancellation terminal digest mismatch".into());
5625 }
5626 let requested = car_proto::RunCancellationRequested {
5627 receipt_version: result.receipt_version,
5628 run_id: result.run_id.clone(),
5629 idempotency_key: result.idempotency_key.clone(),
5630 reason_digest: result.reason_digest.clone(),
5631 principal: result.principal.clone(),
5632 action_id: result.action_id.clone(),
5633 request_id: result.request_id.clone(),
5634 };
5635 let run_id = result.run_id.as_str();
5636 let durability_lock = self.run_durability_lock(run_id).await;
5637 let _durability_guard = durability_lock.lock().await;
5638
5639 let store = self.run_store.clone();
5640 let durable_agent = agent_id.to_string();
5641 let durable_run = run_id.to_string();
5642 let records = tokio::task::spawn_blocking(move || {
5643 store.get_run_trace_for_checked(&durable_agent, &durable_run)
5644 })
5645 .await
5646 .map_err(|error| format!("recovered cancellation trace task failed: {error}"))?
5647 .map_err(|error| format!("recovered cancellation trace read failed: {error}"))?
5648 .ok_or_else(|| "recovered cancellation durable owner is unavailable".to_string())?;
5649 let started = records
5650 .iter()
5651 .find_map(|record| match record {
5652 car_proto::RunRecord::Started(started) => Some(started.clone()),
5653 _ => None,
5654 })
5655 .ok_or_else(|| "recovered cancellation is missing durable RunStarted".to_string())?;
5656 if started.agent_id != agent_id {
5657 return Err("recovered cancellation durable agent mismatch".into());
5658 }
5659 let client_id = started
5660 .client_id
5661 .as_deref()
5662 .filter(|client_id| {
5663 !client_id.is_empty()
5664 && !client_id.contains('/')
5665 && !client_id.contains('\\')
5666 && *client_id != "."
5667 && *client_id != ".."
5668 })
5669 .ok_or_else(|| {
5670 "recovered cancellation has no authenticated durable journal owner".to_string()
5671 })?
5672 .to_string();
5673 if let Some(existing) = records.iter().find_map(|record| match record {
5674 car_proto::RunRecord::CancellationRequested(existing) => Some(existing),
5675 _ => None,
5676 }) {
5677 if existing != &requested {
5678 return Err(format!(
5679 "run `{run_id}` has a different cancellation request"
5680 ));
5681 }
5682 }
5683 if records
5684 .iter()
5685 .any(|record| matches!(record, car_proto::RunRecord::CancellationResult(_)))
5686 {
5687 return Err(format!(
5688 "run `{run_id}` already has a nonterminal cancellation receipt"
5689 ));
5690 }
5691 if let Some(existing) = records.iter().find_map(|record| match record {
5692 car_proto::RunRecord::Ended(existing) => Some(existing),
5693 _ => None,
5694 }) {
5695 if !matches!(existing.termination, car_proto::RunTermination::Incomplete)
5696 || existing.completion_digest.as_deref() != Some(completion_digest.as_str())
5697 {
5698 return Err(format!("run `{run_id}` is already terminal"));
5699 }
5700 return Ok(());
5701 }
5702 {
5703 let runs = self.runs.lock().await;
5704 if let Some(meta) = runs.get(run_id) {
5705 if meta.agent_id != agent_id
5706 || meta.client_id != client_id
5707 || !meta.start_committed
5708 || meta.is_terminal()
5709 || meta.pending_terminal.is_some()
5710 || meta
5711 .cancellation_pending
5712 .as_ref()
5713 .is_some_and(|existing| existing != &requested)
5714 || meta.cancellation_receipt.is_some()
5715 {
5716 return Err(format!(
5717 "run `{run_id}` live state does not authenticate recovered cancellation"
5718 ));
5719 }
5720 }
5721 }
5722
5723 let ended = car_proto::RunEnded {
5724 run_id: run_id.to_string(),
5725 client_id: Some(client_id.clone()),
5726 agent_id: agent_id.to_string(),
5727 termination,
5728 completion_digest: Some(completion_digest),
5729 ended_at: chrono::Utc::now(),
5730 };
5731 let store = self.run_store.clone();
5732 let durable_agent = agent_id.to_string();
5733 let durable_requested = requested.clone();
5734 let durable_ended = ended.clone();
5735 tokio::task::spawn_blocking(move || {
5736 store.write_cancellation_requested(&durable_agent, &durable_requested)?;
5737 store.write_ended(&durable_ended)
5738 })
5739 .await
5740 .map_err(|error| format!("recovered terminal durability task failed: {error}"))?
5741 .map_err(|error| format!("recovered terminal durability failed: {error}"))?;
5742
5743 let journal_dir = self.journal_dir.clone();
5744 let journal_failures = self.journal_failures.clone();
5745 let journal_client = client_id;
5746 let journal_requested = requested.clone();
5747 let journal_ended = ended.clone();
5748 let journal_result = result.clone();
5749 tokio::task::spawn_blocking(move || {
5750 append_recovered_terminal_cancellation_journal(
5751 journal_dir,
5752 journal_failures,
5753 journal_client,
5754 journal_requested,
5755 journal_ended,
5756 journal_result,
5757 )
5758 })
5759 .await
5760 .map_err(|error| format!("recovered cancellation journal task failed: {error}"))??;
5761
5762 let mut runs = self.runs.lock().await;
5763 if let Some(meta) = runs.get_mut(run_id) {
5764 let cursor = meta.turn_cursor();
5765 let request_is_new = meta.cancellation_pending.is_none();
5766 meta.cancellation_pending = Some(requested.clone());
5767 meta.termination = Some(ended.termination.clone());
5768 meta.ended_at = Some(ended.ended_at);
5769 meta.pending_terminal = None;
5770 meta.resume_lease = None;
5771 meta.cancellation_pending = None;
5772 meta.cancellation_receipt = None;
5773 meta.durability_generation = meta.durability_generation.wrapping_add(1);
5774 let subscribers = self.run_subscribers.lock().await;
5775 if request_is_new {
5776 Self::fanout_locked(
5777 &subscribers,
5778 run_id,
5779 agent_id,
5780 car_proto::RunRecord::CancellationRequested(requested),
5781 cursor,
5782 car_proto::RunLiveStatus::CancellationPending,
5783 );
5784 }
5785 Self::fanout_locked(
5786 &subscribers,
5787 run_id,
5788 agent_id,
5789 car_proto::RunRecord::Ended(ended),
5790 cursor,
5791 car_proto::RunLiveStatus::Incomplete,
5792 );
5793 }
5794 drop(runs);
5795 self.clear_terminal_run_turns(run_id).await;
5796 Ok(())
5797 }
5798
5799 /// Mark a run `Incomplete` (R5) — used by disconnect cleanup when a
5800 /// harness drops without `runs.complete`. No-op if the run is
5801 /// already terminal (the common healthy-close case where
5802 /// `runs.complete` won the race). Returns `true` if it actually
5803 /// wrote the `Incomplete` marker.
5804 ///
5805 /// U3: on the transition to `Incomplete`, appends the terminal
5806 /// `RunEnded { Incomplete }` line to disk so an orphaned run reports
5807 /// `Incomplete` (not `InProgress`) on REPLAY after a restart. Disk
5808 /// failures are logged, never fatal.
5809 pub async fn prepare_run_incomplete(&self, run_id: &str) -> Option<car_proto::RunEnded> {
5810 self.prepare_run_incomplete_for_active_owner(run_id, None)
5811 .await
5812 }
5813
5814 async fn prepare_run_incomplete_for_active_owner(
5815 &self,
5816 run_id: &str,
5817 expected_active_client_id: Option<&str>,
5818 ) -> Option<car_proto::RunEnded> {
5819 // If a `runs.complete` transaction already prepared its exact outcome,
5820 // disconnect cleanup must finish that transaction, never overwrite it
5821 // with Incomplete.
5822 let termination = self
5823 .runs
5824 .lock()
5825 .await
5826 .get(run_id)
5827 .and_then(|meta| meta.pending_terminal.as_ref())
5828 .map(|ended| ended.termination.clone())
5829 .unwrap_or(car_proto::RunTermination::Incomplete);
5830 match self
5831 .prepare_run_completion_for_active_owner(run_id, termination, expected_active_client_id)
5832 .await
5833 {
5834 Ok(ended) => Some(ended),
5835 Err(error) => {
5836 tracing::error!(run_id, %error, "failed to prepare durable incomplete terminal");
5837 None
5838 }
5839 }
5840 }
5841
5842 pub async fn mark_run_incomplete(&self, run_id: &str) -> Option<car_proto::RunEnded> {
5843 let ended = self.prepare_run_incomplete(run_id).await?;
5844 if let Err(error) = self.commit_run_completion(&ended).await {
5845 tracing::error!(run_id, %error, "failed to commit incomplete terminal");
5846 return None;
5847 }
5848 Some(ended)
5849 }
5850
5851 /// Free the in-memory per-turn buffer of a TERMINAL run, keeping the
5852 /// lightweight `RunMeta` header resident (agent run tracing, heap
5853 /// hygiene follow-up). Called at the end of `complete_run` /
5854 /// `mark_run_incomplete`, AFTER the disk flush and terminal fanout, so
5855 /// the disk store remains the durable source of truth and any late
5856 /// `runs.subscribe` re-sources the snapshot from disk
5857 /// ([`Self::subscribe_run`]).
5858 ///
5859 /// Guards on `is_terminal()` so a same-id reuse that somehow re-opened
5860 /// the run (it shouldn't — ids are uuids) never has live turns dropped.
5861 /// `Vec::new()` releases the buffer's capacity, not just its length —
5862 /// the per-turn payloads (full prompt + CLI output) are the heavy part.
5863 async fn clear_terminal_run_turns(&self, run_id: &str) {
5864 let mut runs = self.runs.lock().await;
5865 if let Some(meta) = runs.get_mut(run_id) {
5866 if meta.is_terminal() {
5867 meta.turns = Vec::new();
5868 }
5869 }
5870 }
5871
5872 /// Non-acquiring read of a run's current metadata (clone). Used by
5873 /// tests and by the U2 recorder to learn a run's owning `agent_id`.
5874 ///
5875 /// NOTE: this clones the ENTIRE `RunMeta`, including its `turns` buffer
5876 /// (full prompts + CLI output — up to the per-run caps). Hot callers
5877 /// that need only the run's header facts must use [`Self::run_header`]
5878 /// instead; cloning the whole buffer per batch RPC is a ~512 MB
5879 /// worst-case copy at this PR's caps (ADV-2).
5880 pub async fn run_meta(&self, run_id: &str) -> Option<RunMeta> {
5881 self.runs.lock().await.get(run_id).cloned()
5882 }
5883
5884 /// Lightweight, non-cloning read of a run's header facts under the
5885 /// `runs` lock: `(agent_id, terminal, turn_count, trace_corruption)`.
5886 /// The owning agent id and optional error are the only heap allocations;
5887 /// This is what `handle_runs_record_turns` needs — owner for the write
5888 /// authz check, terminality for the distinguishable `run_terminal` drop,
5889 /// and the current turn count for the (fast-path) ceiling pre-check —
5890 /// without the `run_meta` deep clone of every recorded turn's payload.
5891 pub async fn run_header(&self, run_id: &str) -> Option<(String, bool, usize, Option<String>)> {
5892 self.runs.lock().await.get(run_id).map(|m| {
5893 (
5894 m.agent_id.clone(),
5895 m.is_terminal(),
5896 m.turns.len(),
5897 m.trace_corruption.clone(),
5898 )
5899 })
5900 }
5901
5902 /// Lightweight authenticated producer binding for run lifecycle handlers:
5903 /// `(client_id, terminal)`. Never falls back to disk because an on-disk row
5904 /// has no live WebSocket producer to authorize new writes.
5905 pub async fn run_lifecycle_binding(&self, run_id: &str) -> Option<(String, bool)> {
5906 self.runs.lock().await.get(run_id).map(|meta| {
5907 (
5908 meta.active_client_id.clone(),
5909 meta.is_terminal() || meta.trace_corruption.is_some(),
5910 )
5911 })
5912 }
5913
5914 /// `(client_id, terminal, start_committed, completion_pending)` for the
5915 /// authenticated producer path. Proposal writes require the exact owner,
5916 /// a committed start, and no prepared terminal transaction.
5917 pub async fn run_lifecycle_state(&self, run_id: &str) -> Option<(String, bool, bool, bool)> {
5918 self.run_lifecycle_state_with_corruption(run_id).await.map(
5919 |(client_id, terminal, committed, pending, _)| {
5920 (client_id, terminal, committed, pending)
5921 },
5922 )
5923 }
5924
5925 pub async fn run_lifecycle_state_with_corruption(
5926 &self,
5927 run_id: &str,
5928 ) -> Option<(String, bool, bool, bool, Option<String>)> {
5929 self.runs.lock().await.get(run_id).map(|meta| {
5930 (
5931 meta.active_client_id.clone(),
5932 meta.is_terminal(),
5933 meta.start_committed,
5934 meta.pending_terminal.is_some() || meta.cancellation_pending.is_some(),
5935 meta.trace_corruption.clone(),
5936 )
5937 })
5938 }
5939
5940 /// Atomically fence an orphaned live run to a replacement socket that has
5941 /// already authenticated as the run's agent. The immutable durable owner
5942 /// never changes: proposal and terminal artifacts continue to validate
5943 /// against the original `RunStarted` identity.
5944 pub(crate) async fn resume_run(
5945 &self,
5946 run_id: &str,
5947 agent_id: &str,
5948 replacement_client_id: &str,
5949 ) -> Result<RunResumeBinding, String> {
5950 // Serialize the ownership swap with every durable append/finalization
5951 // for this run. A task that already crossed this fence commits before
5952 // resume; one that has only captured the stale socket identity loses
5953 // after resume and cannot write any bytes.
5954 let durability_lock = self.run_durability_lock(run_id).await;
5955 let _durability_guard = durability_lock.lock().await;
5956 // Session creation/removal takes this same short fence around the map
5957 // mutation. Keep both guards through the run-owner compare-and-swap so
5958 // neither a late contender nor a disconnect can invalidate liveness
5959 // between validation and commit.
5960 let _liveness_guard = self.run_resume_liveness.lock().await;
5961 let live_clients = self
5962 .sessions
5963 .lock()
5964 .await
5965 .keys()
5966 .cloned()
5967 .collect::<std::collections::HashSet<_>>();
5968 if !live_clients.contains(replacement_client_id) {
5969 return Err("runs.resume replacement socket is no longer active".into());
5970 }
5971 let mut runs = self.runs.lock().await;
5972 let meta = runs
5973 .get_mut(run_id)
5974 .ok_or_else(|| "run not found or not authorized".to_string())?;
5975 if meta.agent_id != agent_id {
5976 return Err("run not found or not authorized".into());
5977 }
5978 if meta.is_terminal() {
5979 return Err(format!(
5980 "{} run `{run_id}` is already terminal",
5981 car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX
5982 ));
5983 }
5984 if !meta.start_committed {
5985 return Err(format!(
5986 "{} run `{run_id}` start is not durably committed",
5987 car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX
5988 ));
5989 }
5990 if meta.pending_terminal.is_some()
5991 || meta.cancellation_pending.is_some()
5992 || meta.trace_corruption.is_some()
5993 {
5994 return Err(format!(
5995 "{} run `{run_id}` is quarantined and cannot be resumed",
5996 car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX
5997 ));
5998 }
5999
6000 if meta.active_client_id == replacement_client_id {
6001 let resumed_from_client_id =
6002 meta.resume_predecessor_client_id.clone().ok_or_else(|| {
6003 format!(
6004 "{} run `{run_id}` is already active on this socket",
6005 car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX
6006 )
6007 })?;
6008 return Ok(RunResumeBinding {
6009 run_id: meta.run_id.clone(),
6010 agent_id: meta.agent_id.clone(),
6011 active_client_id: meta.active_client_id.clone(),
6012 resumed_from_client_id,
6013 });
6014 }
6015 if live_clients.contains(&meta.active_client_id) {
6016 return Err(format!(
6017 "{} run `{run_id}` still has a live owner",
6018 car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX
6019 ));
6020 }
6021 let lease_is_current = meta.resume_lease.as_ref().is_some_and(|lease| {
6022 lease.disconnected_client_id == meta.active_client_id
6023 && tokio::time::Instant::now() < lease.expires_at
6024 });
6025 if !lease_is_current {
6026 return Err(format!(
6027 "{} run `{run_id}` resume lease is unavailable or expired",
6028 car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX
6029 ));
6030 }
6031
6032 let resumed_from_client_id = std::mem::replace(
6033 &mut meta.active_client_id,
6034 replacement_client_id.to_string(),
6035 );
6036 meta.resume_predecessor_client_id = Some(resumed_from_client_id.clone());
6037 meta.resume_lease = None;
6038 meta.durability_generation = meta.durability_generation.wrapping_add(1);
6039 Ok(RunResumeBinding {
6040 run_id: meta.run_id.clone(),
6041 agent_id: meta.agent_id.clone(),
6042 active_client_id: meta.active_client_id.clone(),
6043 resumed_from_client_id,
6044 })
6045 }
6046
6047 /// Returns the immutable durable owner and the currently fenced live owner.
6048 pub(crate) async fn run_owner_binding(&self, run_id: &str) -> Option<(String, String)> {
6049 self.runs
6050 .lock()
6051 .await
6052 .get(run_id)
6053 .map(|meta| (meta.client_id.clone(), meta.active_client_id.clone()))
6054 }
6055
6056 /// Append per-turn trace records to a run's in-memory buffer (agent
6057 /// run tracing, U2). The recorder calls this after every
6058 /// `proposal.submit` on a session with a current run, passing the
6059 /// `RunRecord::Turn`s the recorder produced for that proposal.
6060 ///
6061 /// Returns a [`RecordRunTurnsOutcome`]:
6062 /// - `Appended { new_total }` — the batch landed; `new_total` is the
6063 /// run's new total turn count (the caller computes its next
6064 /// `start_index` from it).
6065 /// - `RefusedCeiling` — the batch would take the run past
6066 /// [`RECORD_TURNS_RUN_CEILING`] and was refused WHOLE (the under-lock
6067 /// runaway backstop, ADV-1). Nothing was appended.
6068 /// - `UnknownOrTerminal` — the `run_id` is unknown (the bracket was
6069 /// never opened) or already terminal (a turn arriving after
6070 /// `runs.complete` is dropped — the run is closed). Nothing was
6071 /// appended.
6072 ///
6073 /// U3 flushes the in-memory buffer to disk; U4 broadcasts it. Both read
6074 /// via [`Self::run_turns`].
6075 ///
6076 /// U3: the same turn records are appended to the run's JSONL file so
6077 /// REPLAY (U5) sees them after a restart. Persistence is the commit point:
6078 /// only after the exact JSONL batch is durable do memory and subscribers
6079 /// advance. The operation runs in a daemon-owned task, so dropping a
6080 /// request waiter cannot strand a persisted batch between disk and memory.
6081 pub async fn record_run_turns(
6082 self: &Arc<Self>,
6083 run_id: &str,
6084 records: Vec<car_proto::RunRecord>,
6085 ) -> RecordRunTurnsOutcome {
6086 self.record_run_turns_for_owner(run_id, None, records).await
6087 }
6088
6089 /// Append client-narrated turns only while `active_client_id` still owns
6090 /// the run. The socket id is daemon-minted and captured by the handler
6091 /// before it detaches the work into a daemon-owned task; a later resume
6092 /// therefore fences delayed work from the predecessor without trusting a
6093 /// caller-supplied credential.
6094 #[doc(hidden)]
6095 pub async fn record_run_turns_for_active_owner(
6096 self: &Arc<Self>,
6097 run_id: &str,
6098 active_client_id: &str,
6099 records: Vec<car_proto::RunRecord>,
6100 ) -> RecordRunTurnsOutcome {
6101 self.record_run_turns_for_owner(run_id, Some(active_client_id.to_string()), records)
6102 .await
6103 }
6104
6105 async fn record_run_turns_for_owner(
6106 self: &Arc<Self>,
6107 run_id: &str,
6108 expected_active_client_id: Option<String>,
6109 records: Vec<car_proto::RunRecord>,
6110 ) -> RecordRunTurnsOutcome {
6111 let state = Arc::clone(self);
6112 let run_id = run_id.to_string();
6113 match tokio::spawn(async move {
6114 state
6115 .record_run_turns_owned(&run_id, expected_active_client_id.as_deref(), records)
6116 .await
6117 })
6118 .await
6119 {
6120 Ok(outcome) => outcome,
6121 Err(error) => RecordRunTurnsOutcome::PersistenceFailed(format!(
6122 "durable turn append task failed: {error}"
6123 )),
6124 }
6125 }
6126
6127 /// Make a proposal's authenticated final trace durable exactly once and
6128 /// reflect a newly-appended batch into the live registry/subscription
6129 /// stream before proposal finalization can acknowledge success.
6130 pub async fn ensure_proposal_run_turns(
6131 self: &Arc<Self>,
6132 pending: &crate::run_store::PendingProposalFinalization,
6133 ) -> Result<(), String> {
6134 let state = Arc::clone(self);
6135 let pending = pending.clone();
6136 tokio::spawn(async move { state.ensure_proposal_run_turns_owned(&pending).await })
6137 .await
6138 .map_err(|error| format!("proposal trace durability task failed: {error}"))?
6139 }
6140
6141 async fn ensure_proposal_run_turns_owned(
6142 &self,
6143 pending: &crate::run_store::PendingProposalFinalization,
6144 ) -> Result<(), String> {
6145 let durability_lock = self.run_durability_lock(&pending.run_id).await;
6146 let _durability_guard = durability_lock.lock().await;
6147 let (agent_id, status, generation) = {
6148 let runs = self.runs.lock().await;
6149 let meta = runs
6150 .get(&pending.run_id)
6151 .ok_or_else(|| format!("active run `{}` is absent", pending.run_id))?;
6152 if let Some(error) = &meta.trace_corruption {
6153 return Err(error.clone());
6154 }
6155 if !meta.accepts_proposals() {
6156 return Err(format!("active run `{}` is not writable", pending.run_id));
6157 }
6158 (
6159 meta.agent_id.clone(),
6160 meta.live_status(),
6161 meta.durability_generation,
6162 )
6163 };
6164 let store = self.run_store.clone();
6165 let durable = pending.clone();
6166 let durable_result =
6167 tokio::task::spawn_blocking(move || store.ensure_proposal_turns(&durable))
6168 .await
6169 .map_err(|error| format!("proposal trace durability task failed: {error}"))?;
6170 let mut runs = self.runs.lock().await;
6171 let ensured = match durable_result {
6172 Ok(ensured) => ensured,
6173 Err(error) if crate::run_store::is_trace_corruption_error(&error) => {
6174 let message = self
6175 .quarantine_run_trace_locked(&mut runs, &pending.run_id, error.to_string())
6176 .await;
6177 return Err(message);
6178 }
6179 Err(error) => return Err(format!("proposal trace durability failed: {error}")),
6180 };
6181 let meta = runs.get(&pending.run_id).ok_or_else(|| {
6182 format!(
6183 "active run `{}` disappeared after durability",
6184 pending.run_id
6185 )
6186 })?;
6187 if let Some(error) = &meta.trace_corruption {
6188 return Err(error.clone());
6189 }
6190 if meta.durability_generation != generation || !meta.accepts_proposals() {
6191 return Err(format!(
6192 "active run `{}` changed during proposal trace durability",
6193 pending.run_id
6194 ));
6195 }
6196 let already_live = !ensured.appended
6197 && meta.turns.iter().any(|record| match record {
6198 car_proto::RunRecord::Turn(turn) => {
6199 turn.proposal_id.as_deref() == Some(pending.final_proposal_id.as_str())
6200 }
6201 _ => false,
6202 });
6203 if ensured.appended || !already_live {
6204 let meta = runs
6205 .get_mut(&pending.run_id)
6206 .expect("run registry remains locked during proposal trace commit");
6207 let base = ensured
6208 .records
6209 .first()
6210 .and_then(|record| match record {
6211 car_proto::RunRecord::Turn(turn) => Some(turn.index),
6212 _ => None,
6213 })
6214 .unwrap_or(meta.turns.len());
6215 meta.turns.extend(ensured.records.iter().cloned());
6216 meta.durability_generation = meta.durability_generation.wrapping_add(1);
6217 let subs = self.run_subscribers.lock().await;
6218 for (offset, record) in ensured.records.into_iter().enumerate() {
6219 Self::fanout_locked(
6220 &subs,
6221 &pending.run_id,
6222 &agent_id,
6223 record,
6224 base + offset + 1,
6225 status,
6226 );
6227 }
6228 }
6229 Ok(())
6230 }
6231
6232 async fn record_run_turns_owned(
6233 &self,
6234 run_id: &str,
6235 expected_active_client_id: Option<&str>,
6236 mut records: Vec<car_proto::RunRecord>,
6237 ) -> RecordRunTurnsOutcome {
6238 // Serialize durability only with this run. The global registry lock is
6239 // held for the admission snapshot and memory/fanout commit, never for
6240 // trace scanning, append, flush, or fsync.
6241 let durability_lock = self.run_durability_lock(run_id).await;
6242 let _durability_guard = durability_lock.lock().await;
6243 let (agent_id, base, new_total, status, generation) = {
6244 let mut runs = self.runs.lock().await;
6245 match runs.get_mut(run_id) {
6246 Some(meta) => {
6247 if let Some(error) = &meta.trace_corruption {
6248 return RecordRunTurnsOutcome::PersistenceFailed(error.clone());
6249 }
6250 if expected_active_client_id
6251 .is_some_and(|expected| meta.active_client_id != expected)
6252 {
6253 return RecordRunTurnsOutcome::UnknownOrTerminal;
6254 }
6255 if !meta.accepts_proposals() {
6256 return RecordRunTurnsOutcome::UnknownOrTerminal;
6257 }
6258 // ADV-1: hard ceiling, under the lock. Count only the
6259 // `Turn` records — Started/Ended are not buffered here,
6260 // and the ceiling is a turn-count cap. Refuse the whole
6261 // batch if appending it would cross the ceiling.
6262 let incoming_turns = records
6263 .iter()
6264 .filter(|r| matches!(r, car_proto::RunRecord::Turn(_)))
6265 .count();
6266 if meta.turns.len() + incoming_turns > RECORD_TURNS_RUN_CEILING {
6267 return RecordRunTurnsOutcome::RefusedCeiling;
6268 } else {
6269 // Position of the first record about to be appended —
6270 // its post-append cursor is `base + 1`.
6271 let base = meta.turns.len();
6272 // FIX 6: re-stamp each turn's `index` from the LIVE
6273 // append position under the `runs` lock. The caller
6274 // computes a provisional `start_index` from a pre-read
6275 // `run_turn_count` OUTSIDE this lock — a TOCTOU: two
6276 // concurrent proposals on one connection can read the
6277 // same `start_index` and bake overlapping indices.
6278 // Authoritatively assigning the index here, from
6279 // `base + offset`, makes it contiguous and correct
6280 // regardless of the racing reads upstream.
6281 for (offset, record) in records.iter_mut().enumerate() {
6282 if let car_proto::RunRecord::Turn(turn) = record {
6283 turn.index = base + offset;
6284 }
6285 }
6286 let agent_id = meta.agent_id.clone();
6287 let status = meta.live_status();
6288 (
6289 agent_id,
6290 base,
6291 base + records.len(),
6292 status,
6293 meta.durability_generation,
6294 )
6295 }
6296 }
6297 None => return RecordRunTurnsOutcome::UnknownOrTerminal,
6298 }
6299 };
6300
6301 let store = self.run_store.clone();
6302 let durable_agent = agent_id.clone();
6303 let durable_run = run_id.to_string();
6304 let durable_records = records.clone();
6305 let append = tokio::task::spawn_blocking(move || {
6306 store.append_turns(&durable_agent, &durable_run, &durable_records)
6307 })
6308 .await;
6309 let mut runs = self.runs.lock().await;
6310 match append {
6311 Ok(Ok(())) => {}
6312 Ok(Err(error)) => {
6313 tracing::warn!(run_id, %error, "run-store: failed to persist turns");
6314 if crate::run_store::is_trace_corruption_error(&error) {
6315 let message = self
6316 .quarantine_run_trace_locked(&mut runs, run_id, error.to_string())
6317 .await;
6318 return RecordRunTurnsOutcome::PersistenceFailed(message);
6319 }
6320 return RecordRunTurnsOutcome::PersistenceFailed(error.to_string());
6321 }
6322 Err(error) => {
6323 tracing::warn!(run_id, %error, "run-store: durable turn task failed");
6324 return RecordRunTurnsOutcome::PersistenceFailed(error.to_string());
6325 }
6326 }
6327
6328 let meta = runs
6329 .get_mut(run_id)
6330 .expect("run registry entry survives a committed turn append");
6331 if let Some(error) = &meta.trace_corruption {
6332 return RecordRunTurnsOutcome::PersistenceFailed(error.clone());
6333 }
6334 if meta.durability_generation != generation || !meta.accepts_proposals() {
6335 return RecordRunTurnsOutcome::PersistenceFailed(format!(
6336 "run `{run_id}` changed during durable turn append"
6337 ));
6338 }
6339 meta.turns.extend(records.iter().cloned());
6340 meta.durability_generation = meta.durability_generation.wrapping_add(1);
6341 let subs = self.run_subscribers.lock().await;
6342 for (offset, record) in records.into_iter().enumerate() {
6343 Self::fanout_locked(&subs, run_id, &agent_id, record, base + offset + 1, status);
6344 }
6345 RecordRunTurnsOutcome::Appended { new_total }
6346 }
6347
6348 /// Number of turns already recorded for a run — the `start_index` the
6349 /// recorder passes so per-turn `index` stays monotonic across the
6350 /// run's proposals. `0` for an unknown run.
6351 pub async fn run_turn_count(&self, run_id: &str) -> usize {
6352 self.runs
6353 .lock()
6354 .await
6355 .get(run_id)
6356 .map(|m| m.turns.len())
6357 .unwrap_or(0)
6358 }
6359
6360 /// Clone of a run's ordered per-turn trace (agent run tracing, U2).
6361 /// This is the accessor U3 reads to flush turns to disk and U4 reads
6362 /// to broadcast them. Empty Vec for an unknown run or a run with no
6363 /// turns yet.
6364 pub async fn run_turns(&self, run_id: &str) -> Vec<car_proto::RunRecord> {
6365 self.runs
6366 .lock()
6367 .await
6368 .get(run_id)
6369 .map(|m| m.turns.clone())
6370 .unwrap_or_default()
6371 }
6372
6373 /// Atomically snapshot a run's turns AND register a live
6374 /// `runs.trace.event` subscriber for `(run_id, host_client_id)` —
6375 /// the load-bearing invariant #1 (agent run tracing, U4).
6376 ///
6377 /// Holds the `runs` lock across BOTH (a) reading the run's current
6378 /// turn buffer as the snapshot at cursor `C = turns.len()` and (b)
6379 /// inserting the subscriber into `run_subscribers`. Because the
6380 /// lifecycle path (`record_run_turns` / `complete_run` /
6381 /// `mark_run_incomplete`) appends-and-notifies under the SAME lock,
6382 /// the snapshot contains exactly the turns ≤ C and every turn appended
6383 /// after registration is delivered — no turn in the snapshot/register
6384 /// window is dropped (gap) or double-delivered (dup).
6385 ///
6386 /// Spawns the subscriber's drain task (bounded channel → WS) so the
6387 /// producer never writes the socket directly (invariant #2). Returns
6388 /// the snapshot response shape. `None` if the `run_id` is unknown (the
6389 /// handler maps that to a not-found error). A re-subscribe by the same
6390 /// `(run_id, host_client_id)` replaces the prior subscriber (its drain
6391 /// task ends when the old sender drops) and re-snapshots — the R8
6392 /// reconnect path: the fresh snapshot covers any turns emitted during
6393 /// the gap with no dup.
6394 ///
6395 /// Terminal-run disk fallback: a completed/incomplete run has its
6396 /// in-memory `turns` evicted for heap hygiene (see
6397 /// [`Self::clear_terminal_run_turns`]). When the run is terminal AND its
6398 /// in-memory buffer is empty, the snapshot is re-sourced from the disk
6399 /// store ([`crate::run_store::RunStore::get_run_trace`], the same source
6400 /// `runs.get_trace` reads), filtered to `RunRecord::Turn` so the shape
6401 /// matches the live in-memory snapshot. A terminal run takes no further
6402 /// appends, so the disk trail is final and there is no gap/dup concern —
6403 /// the snapshot/register atomicity the `runs` lock provides is only
6404 /// load-bearing for the in-progress path, which is unchanged.
6405 pub(crate) async fn subscribe_run_page(
6406 &self,
6407 run_id: &str,
6408 host_client_id: &str,
6409 channel: Arc<WsChannel>,
6410 cursor: usize,
6411 limit: usize,
6412 ) -> Result<Option<RunSubscribePageResult>, String> {
6413 // Build the subscriber (and its drain task) OUTSIDE the runs lock
6414 // so spawning never happens under the hot lock; registration
6415 // itself is the only thing serialized.
6416 let subscriber =
6417 crate::host::RunTraceSubscriber::spawn(host_client_id.to_string(), channel);
6418
6419 let (agent_id, mut expected_state) = {
6420 let runs = self.runs.lock().await;
6421 let Some(meta) = runs.get(run_id) else {
6422 return Ok(None);
6423 };
6424 if let Some(error) = &meta.trace_corruption {
6425 return Err(error.clone());
6426 }
6427 if meta.is_terminal() {
6428 return Ok(Some(RunSubscribePageResult::Durable {
6429 agent_id: meta.agent_id.clone(),
6430 status: meta.live_status(),
6431 }));
6432 }
6433 (
6434 meta.agent_id.clone(),
6435 (meta.turns.len(), meta.pending_terminal.is_some()),
6436 )
6437 };
6438
6439 for attempt in 0..=RUN_SUBSCRIBE_SUMMARY_STATE_RETRY_LIMIT {
6440 // Keep disk I/O off the global run-registry lock. A concurrent
6441 // mutation advances the captured state under that lock; retrying
6442 // the summary read then gives snapshot/registration one coherent
6443 // durable/live boundary.
6444 let store = self.run_store.clone();
6445 let durable_agent = agent_id.clone();
6446 let durable_run = run_id.to_string();
6447 let durable_corruption = tokio::task::spawn_blocking(move || {
6448 store.run_trace_corruption_for(&durable_agent, &durable_run)
6449 })
6450 .await
6451 .map_err(|error| format!("run trace corruption check task failed: {error}"))?
6452 .map_err(|error| format!("run trace corruption check failed: {error}"))?;
6453 let mut runs = self.runs.lock().await;
6454 let Some(meta) = runs.get(run_id) else {
6455 return Ok(None);
6456 };
6457 if let Some(error) = &meta.trace_corruption {
6458 return Err(error.clone());
6459 }
6460 if meta.agent_id != agent_id {
6461 return Err(format!("run `{run_id}` changed ownership during subscribe"));
6462 }
6463 if let Some(corruption) = durable_corruption {
6464 let message = self
6465 .quarantine_run_trace_locked(
6466 &mut runs,
6467 run_id,
6468 format!("malformed run trace record at line {}", corruption.line),
6469 )
6470 .await;
6471 return Err(message);
6472 }
6473 let status = meta.live_status();
6474 if meta.is_terminal() {
6475 return Ok(Some(RunSubscribePageResult::Durable { agent_id, status }));
6476 }
6477 let current_state = (meta.turns.len(), meta.pending_terminal.is_some());
6478 if current_state != expected_state {
6479 if attempt == RUN_SUBSCRIBE_SUMMARY_STATE_RETRY_LIMIT {
6480 return Err(format!(
6481 "run `{run_id}` changed repeatedly during durable subscribe validation; retry"
6482 ));
6483 }
6484 expected_state = current_state;
6485 drop(runs);
6486 continue;
6487 }
6488 let live_cursor = meta.turns.len();
6489 if cursor > live_cursor {
6490 return Err(format!(
6491 "runs.subscribe cursor {cursor} exceeds live_cursor {live_cursor}"
6492 ));
6493 }
6494 let end = cursor.saturating_add(limit).min(live_cursor);
6495 let turns = meta.turns[cursor..end].to_vec();
6496 let next_cursor = (end < live_cursor).then_some(end);
6497 let subscribed = next_cursor.is_none();
6498 if subscribed {
6499 let mut subs = self.run_subscribers.lock().await;
6500 subs.insert((run_id.to_string(), host_client_id.to_string()), subscriber);
6501 }
6502 drop(runs);
6503
6504 return Ok(Some(RunSubscribePageResult::Ready(
6505 car_proto::RunSubscribeResponse {
6506 run_id: run_id.to_string(),
6507 agent_id,
6508 turns,
6509 cursor,
6510 limit,
6511 next_cursor,
6512 live_cursor,
6513 subscribed,
6514 status,
6515 },
6516 )));
6517 }
6518 unreachable!("bounded subscribe validation loop always returns")
6519 }
6520
6521 /// Remove a live run-trace subscriber for `(run_id, host_client_id)`
6522 /// (agent run tracing, U4). Returns `true` if a subscription existed.
6523 /// Dropping the [`crate::host::RunTraceSubscriber`] drops its channel
6524 /// sender, which ends the drain task.
6525 pub async fn unsubscribe_run(&self, run_id: &str, host_client_id: &str) -> bool {
6526 self.run_subscribers
6527 .lock()
6528 .await
6529 .remove(&(run_id.to_string(), host_client_id.to_string()))
6530 .is_some()
6531 }
6532
6533 /// Drop every live run-trace subscription owned by `host_client_id`
6534 /// (agent run tracing, U4 — R8 cleanup). Called from
6535 /// [`remove_session`](crate::session::ServerState::remove_session) on disconnect so a CarHost that drops doesn't
6536 /// leave dangling drain tasks. Reconnect-durability is client-side:
6537 /// the run stays subscribable while it lives, and the CarHost re-issues
6538 /// `runs.subscribe {run_id}` on its new connection — the server never
6539 /// synthesizes a failure on subscriber drop.
6540 pub async fn drop_run_subscribers_for_client(&self, host_client_id: &str) {
6541 self.run_subscribers
6542 .lock()
6543 .await
6544 .retain(|(_run, client), _| client != host_client_id);
6545 }
6546
6547 /// Disconnect cleanup for agent runs (R5). Called from
6548 /// [`remove_session`] with the `client_id` of the dropping
6549 /// connection. For every run this connection owns that is **not**
6550 /// yet terminal, wait a short grace window then re-check: if a
6551 /// concurrently-dispatched `runs.complete` made it terminal in the
6552 /// meantime, leave it; otherwise mark it `Incomplete`. The grace
6553 /// window is the fix for the serve-mode false-positive where a
6554 /// healthy `runs.complete` is still in flight (in a spawned
6555 /// dispatch task) when the close frame arrives (Risk: run identity
6556 /// races).
6557 async fn sweep_runs_for_disconnect(&self, session: &ClientSession) {
6558 let client_id = session.client_id.as_str();
6559 // Snapshot the still-open runs owned by this connection.
6560 let pending: Vec<(String, Option<tokio::time::Instant>)> = {
6561 let runs = self.runs.lock().await;
6562 runs.values()
6563 .filter(|m| m.active_client_id == client_id && !m.is_terminal())
6564 .map(|m| {
6565 let resume_deadline = m.resume_lease.as_ref().and_then(|lease| {
6566 (lease.disconnected_client_id == client_id).then_some(lease.expires_at)
6567 });
6568 (m.run_id.clone(), resume_deadline)
6569 })
6570 .collect()
6571 };
6572 if pending.is_empty() {
6573 return;
6574 }
6575 for (run_id, resume_deadline) in pending {
6576 // The lease deadline was installed atomically with removal from
6577 // `sessions`; sleeping to that absolute instant keeps teardown
6578 // work before this point from silently extending the contract.
6579 match resume_deadline {
6580 Some(deadline) => tokio::time::sleep_until(deadline).await,
6581 None => tokio::time::sleep(RUN_COMPLETE_GRACE).await,
6582 }
6583 match self.run_lifecycle_state(&run_id).await {
6584 None | Some((_, true, _, _)) => continue,
6585 Some((owner, false, _, _)) if owner != client_id => {
6586 // An authenticated replacement claimed this run during the
6587 // disconnect grace window. The old owner is fenced and
6588 // must not be allowed to close the replacement's run.
6589 continue;
6590 }
6591 Some((_, false, true, _)) => {}
6592 Some((_, false, false, _)) => {
6593 match self
6594 .reconcile_or_release_unacknowledged_start(session, &run_id)
6595 .await
6596 {
6597 Ok(true) => continue,
6598 Ok(false) => {}
6599 Err(error) => {
6600 tracing::error!(run_id, client_id, %error, "failed to reconcile or release an unacknowledged RunStarted transaction");
6601 continue;
6602 }
6603 }
6604 }
6605 }
6606 match self.run_store.pending_proposal(&run_id) {
6607 Ok(Some(_)) => {
6608 tracing::warn!(
6609 run_id,
6610 client_id,
6611 "leaving run open because proposal finalization is pending durable replay"
6612 );
6613 continue;
6614 }
6615 Err(error) => {
6616 tracing::error!(run_id, client_id, %error, "leaving run open because proposal finalization state is unreadable and outcome is unknown");
6617 continue;
6618 }
6619 Ok(None) => {}
6620 }
6621 match self.run_store.execution_marker(&run_id) {
6622 Ok(Some(marker)) => {
6623 tracing::warn!(
6624 run_id,
6625 client_id,
6626 original_proposal_id = marker.original_proposal_id,
6627 "leaving run open because proposal execution outcome is unknown"
6628 );
6629 continue;
6630 }
6631 Err(error) => {
6632 tracing::error!(run_id, client_id, %error, "leaving run open because proposal execution marker is unreadable and outcome is unknown");
6633 continue;
6634 }
6635 Ok(None) => {}
6636 }
6637 if let Some(ended) = self
6638 .prepare_run_incomplete_for_active_owner(&run_id, Some(client_id))
6639 .await
6640 {
6641 let durable_client_id = ended.client_id.as_deref().unwrap_or(client_id);
6642 let append_result = match session
6643 .append_run_terminal_event_once(&ended, durable_client_id)
6644 .await
6645 {
6646 Err(error) if error.is_retry_safe() => {
6647 tracing::warn!(
6648 run_id,
6649 client_id,
6650 %error,
6651 "disconnect terminal durability is unknown; retrying the exact journal row once"
6652 );
6653 session
6654 .append_run_terminal_event_once(&ended, durable_client_id)
6655 .await
6656 }
6657 result => result,
6658 };
6659 if let Err(error) = append_result {
6660 tracing::error!(run_id, client_id, %error, "failed to bind disconnect terminal journal event");
6661 continue;
6662 }
6663 if let Err(error) = self.commit_run_completion(&ended).await {
6664 tracing::error!(run_id, client_id, %error, "failed to commit disconnect terminal");
6665 continue;
6666 }
6667 if let Err(error) = session.clear_run_journal_binding(&run_id).await {
6668 tracing::error!(run_id, client_id, %error, "failed to clear durable disconnect journal binding");
6669 continue;
6670 }
6671 let mut current = session.current_run_id.lock().await;
6672 if current.as_deref() == Some(run_id.as_str()) {
6673 *current = None;
6674 }
6675 }
6676 }
6677 }
6678
6679 /// Fails only when `<home>/.car/policies/` holds a malformed rule file —
6680 /// see [`apply_project_policies`] for why that is fatal rather than a
6681 /// warning.
6682 pub async fn create_session(
6683 &self,
6684 client_id: &str,
6685 channel: Arc<WsChannel>,
6686 ) -> Result<Arc<ClientSession>, String> {
6687 let journal_path = self.journal_dir.join(format!("{}.jsonl", client_id));
6688 let event_log = match self.journal_failures.clone() {
6689 Some(failures) => EventLog::with_journal_failure_injector(journal_path, failures),
6690 None => EventLog::with_journal(journal_path),
6691 };
6692
6693 let negotiated_capabilities =
6694 Arc::new(std::sync::RwLock::new(std::collections::BTreeSet::new()));
6695 let halted = Arc::new(AtomicBool::new(false));
6696 let ws_executor = Arc::new(WsToolExecutor::new(
6697 channel.clone(),
6698 negotiated_capabilities.clone(),
6699 halted.clone(),
6700 ));
6701
6702 // Compose: connector (remote MCP) tools route to the shared
6703 // process-wide executor; everything else falls back to this
6704 // session's WS tool executor. Both views share the same routing
6705 // table, so a connector enabled in any session is callable here.
6706 let executor: Arc<dyn ToolExecutor> =
6707 Arc::new(self.mcp_executor.share_with_fallback(ws_executor));
6708
6709 // Outbound human messaging. Built-in iMessage adapter over the same
6710 // `RealMessageSender` + `~/.car` config store the daemon's
6711 // `messaging.*` handlers use, plus a host fallback so a channel CAR has
6712 // no built-in transport for (Teams, per Parslee-ai/car#885) is
6713 // delivered by the host rather than compiled into the runtime.
6714 //
6715 // NO APPROVAL LOOP. `RealMessageSender` deliberately calls the *un-gated*
6716 // `car_ffi_common::integrations::messages_send` so that the approval
6717 // transport's own prompt does not raise an approval for itself. That
6718 // choice is safe to reuse here, and the recursion this new path might
6719 // look like it opens does not exist, because the two directions never
6720 // meet: `messaging.send` → registry → adapter → `messages_send` is a
6721 // straight line into the JXA bridge with no re-entry into the runtime,
6722 // and an approval raised *for* `messaging.send` is delivered by the
6723 // orchestrator/fanout path, which calls `MessageSender` directly and
6724 // never dispatches a runtime tool. Neither leg can call the other, so
6725 // there is no cycle to bound. Keep it that way: routing approval
6726 // delivery through `messaging.send` WOULD create one.
6727 let outbound = Arc::new(car_messaging::outbound::OutboundRegistry::new());
6728 outbound.register(Arc::new(
6729 car_messaging::outbound::ImessageOutboundAdapter::new(
6730 Arc::new(crate::messaging_orchestrator::RealMessageSender),
6731 crate::messaging_config::MessagingConfigStore::from_home(),
6732 ),
6733 ));
6734 outbound.set_fallback(Arc::new(crate::host_channel::HostChannelAdapter::new(
6735 executor.clone(),
6736 )));
6737
6738 // The trajectory store is the daemon-wide one, so traces from every
6739 // session accumulate in a single history. Without this the executor's
6740 // `persist_trajectory` short-circuits on its `self.trajectory_store
6741 // .as_ref()?` and nothing is ever recorded — which is why the
6742 // per-tool success rates had no data behind them.
6743 let runtime = Runtime::new()
6744 .with_event_log(event_log)
6745 .with_executor(executor)
6746 .with_trajectory_store(self.trajectory_store.clone())
6747 .with_message_sink(outbound);
6748
6749 // Declarative rules from `~/.car/policies/*.toml`. Same `~/.car`
6750 // convention the messaging config and the approval journal use — the
6751 // daemon serves whatever project a client happens to be working in, so
6752 // its policy set is the operator's, not a per-project one. (The
6753 // assistant, which IS rooted in a project, loads `<root>/.car`.)
6754 if let Some(car_dir) = car_home_dir() {
6755 apply_project_policies(&runtime, &car_dir).await?;
6756 }
6757
6758 // The session's permission-tier gate. Built here rather than inline in
6759 // the `ClientSession` literal below because the admission gate
6760 // registered a few lines down has to hold the SAME object the
6761 // `permission.*` RPCs mutate — see the field doc on
6762 // `ClientSession::permission_gate`.
6763 //
6764 // `SandboxEdit` is the default `docs/websocket-protocol.md` publishes,
6765 // and the code had drifted to `ReadOnly` while nothing enforced the
6766 // tier. `RiskClassifier::baseline` maps both `StateWrite` and
6767 // `ToolCall` to `SandboxEdit`, so keeping the drifted `ReadOnly`
6768 // default the moment enforcement lands would escalate essentially
6769 // every action every existing binding client submits. `full_access`
6770 // actions still escalate — mandatory HITL, exactly as documented.
6771 let permission_gate = Arc::new(tokio::sync::RwLock::new(car_policy::PermissionGate::new(
6772 car_policy::PermissionTier::SandboxEdit,
6773 )));
6774 // These handles are shared with the permission admission gate so it
6775 // observes the identity established later by `session.auth`; taking a
6776 // snapshot here would leave every newly-created session unbound.
6777 let authenticated = Arc::new(std::sync::atomic::AtomicBool::new(false));
6778 let agent_id = Arc::new(tokio::sync::Mutex::new(None));
6779 let callback_tool_schema_digests = Arc::new(tokio::sync::RwLock::new(HashMap::new()));
6780
6781 // If the embedder supplied a shared memgine, every session uses it.
6782 // Otherwise each session gets its own — matches pre-extraction behavior.
6783 // Create it before the admission gate so proposal.submit and the
6784 // permission advisory RPCs consult the exact same live skill ceilings.
6785 let memgine = match &self.shared_memgine {
6786 Some(eng) => eng.clone(),
6787 None => Arc::new(Mutex::new(car_memgine::MemgineEngine::new(None))),
6788 };
6789 // A non-shared session engine has no startup constructor hook, so
6790 // rehydrate the authoritative identity profile here too. Shared engines
6791 // were initialized in `try_with_config`; repeating the idempotent upsert
6792 // makes an externally edited identity.json visible to the next session.
6793 match self.identity_store.load() {
6794 Ok(identity) => {
6795 let mut engine = memgine.lock().await;
6796 mirror_identity_into_memgine(&mut engine, &identity);
6797 }
6798 Err(error) => tracing::warn!(
6799 error = %error,
6800 "identity profile could not be mirrored into a new session"
6801 ),
6802 }
6803
6804 // A terminal callback poisons this WebSocket session after the current
6805 // proposal's engine-level abort and rollback. Register the latch as an
6806 // ordinary admission gate so a later proposal gets the same complete
6807 // rejected-result shape as every other pre-execution refusal.
6808 runtime
6809 .register_admission_gate(Arc::new(SessionHaltAdmissionGate {
6810 halted: halted.clone(),
6811 }))
6812 .await;
6813
6814 // Statically verify submitted proposals before any action dispatches.
6815 //
6816 // This is the surface that matters for it: `proposal.submit` accepts
6817 // caller-authored, **multi-action** proposals, and per-action validation
6818 // only rejects an action once execution reaches it — so a bad tool name
6819 // in action 5 is caught after actions 1–4 have already had their side
6820 // effects. The gate refuses the whole proposal up front, so nothing
6821 // partial happens. (On a runtime that submits one action at a time, like
6822 // the assistant loop, it adds nothing the validator wasn't already
6823 // doing; see `car_engine::verify_gate` for what blocks and why.)
6824 runtime
6825 .register_admission_gate(Arc::new(car_engine::StaticVerificationGate::new(
6826 runtime.tools.clone(),
6827 )))
6828 .await;
6829
6830 // Out-of-process supervision (proposal item 5). Registered
6831 // unconditionally: with no supervisor subscribed the gate short-circuits
6832 // to Allow on an empty-map read, so leaving it on costs nothing and
6833 // means a supervisor can attach to a session that was already running
6834 // rather than only to sessions created after it connected.
6835 runtime
6836 .register_admission_gate(Arc::new(crate::supervision::SupervisionGate::new(
6837 self.supervision.clone(),
6838 )))
6839 .await;
6840
6841 // Enforce the session's granted permission tier before any action
6842 // dispatches (Parslee-ai/car#890).
6843 //
6844 // Everything this needs already existed — the per-session tier gate,
6845 // the durable daemon-wide approval ledger, `permission.approve` — but
6846 // nothing consulted any of it on the way to the executor, so an action
6847 // the daemon's own `permission.evaluate` called `needs_approval`
6848 // executed anyway and its state write persisted. This is the missing
6849 // call site: it asks the SAME gate the same question `permission.
6850 // evaluate` asks, against the SAME shared ledger, so the advisory
6851 // answer and the enforced one cannot diverge. An escalation blocks the
6852 // whole proposal until an operator approves the named fingerprint;
6853 // approving it makes the next submit come back `Allow` with no
6854 // escalation raised at all.
6855 runtime
6856 .register_admission_gate(Arc::new(
6857 crate::permission_gate::PermissionAdmissionGate::new(
6858 permission_gate.clone(),
6859 self.approval_ledger.clone(),
6860 )
6861 .with_authenticated_agent(authenticated.clone(), agent_id.clone())
6862 .with_callback_tools(
6863 callback_tool_schema_digests.clone(),
6864 runtime.registry.clone(),
6865 )
6866 .with_skill_memgine(memgine.clone())
6867 .with_event_log(runtime.log.clone()),
6868 ))
6869 .await;
6870
6871 let session = Arc::new(ClientSession {
6872 client_id: client_id.to_string(),
6873 runtime: Arc::new(runtime),
6874 channel,
6875 host: self.host.clone(),
6876 memgine,
6877 browser: car_ffi_common::browser::BrowserSessionSlot::new(),
6878 // When auth is disabled (no token installed), every
6879 // session is "authenticated" by default — preserves the
6880 // pre-#32 behaviour. When auth is enabled, the value is
6881 // ignored on creation; the dispatcher's gate checks
6882 // `ServerState::auth_token.is_some()` to decide whether
6883 // to enforce.
6884 authenticated,
6885 negotiated_protocol_version: std::sync::atomic::AtomicU32::new(0),
6886 negotiated_capabilities,
6887 inference_control: Arc::new(crate::inference_control::InferenceRegistry::default()),
6888 is_host: std::sync::atomic::AtomicBool::new(false),
6889 agent_id,
6890 agent_method_allowlist: Arc::new(std::sync::RwLock::new(None)),
6891 callback_tool_schema_digests,
6892 memory_namespace: tokio::sync::Mutex::new(None),
6893 bound_memgine: tokio::sync::Mutex::new(None),
6894 current_run_id: tokio::sync::Mutex::new(None),
6895 run_lifecycle_guard: Arc::new(tokio::sync::Mutex::new(())),
6896 permission_gate,
6897 halted,
6898 evolution_guard: crate::evolution::CycleGuard::default(),
6899 last_chat_turn: tokio::sync::Mutex::new(None),
6900 chat_inflight: std::sync::atomic::AtomicUsize::new(0),
6901 tenant: tokio::sync::Mutex::new(None),
6902 tool_stream_subscribed: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)),
6903 });
6904
6905 // Seed connector tools already enabled (in any session) into this
6906 // new session's runtime registry so the model sees them at once.
6907 // Only touches the manager if connectors have been initialized —
6908 // embedders that never use connectors pay nothing here.
6909 if self.connectors.get().is_some() {
6910 for entry in self.connectors().enabled_tool_entries().await {
6911 session.runtime.register_tool_entry(entry).await;
6912 }
6913 }
6914
6915 {
6916 let _liveness_guard = self.run_resume_liveness.lock().await;
6917 self.sessions
6918 .lock()
6919 .await
6920 .insert(client_id.to_string(), session.clone());
6921 }
6922
6923 Ok(session)
6924 }
6925
6926 /// Bind a session's runtime to the [`car_engine::McpSubstrate`] backed by an
6927 /// already-connected MCP connector, making that session **coherent**:
6928 /// its commodity built-ins (`read_file`/`write_file`/…/`run_command`)
6929 /// act on the *same* environment as its `mcp_{slug}_*` connector tools,
6930 /// instead of being split between the host/WS client and the remote MCP
6931 /// server (`docs/execution-substrate.md` §3, phase 3).
6932 ///
6933 /// This is **opt-in and explicit** — it is reached only via the
6934 /// `session.bindSubstrate` JSON-RPC method, never inferred from a
6935 /// connector merely being enabled. Sessions that never call it keep the
6936 /// exact historic composition (`share_with_fallback(ws_executor)` over a
6937 /// default `LocalSubstrate`), so GUI / A2A-over-WS / connectors-as-tools
6938 /// / coder behavior is unchanged.
6939 ///
6940 /// On bind we:
6941 /// 1. Wrap the live connector session (shared with the
6942 /// `mcp_{slug}_*` routes — no second dial) as an [`car_engine::McpSubstrate`] and
6943 /// `set_substrate` it on the session runtime.
6944 /// 2. `register_agent_basics()` so the model sees portable names
6945 /// (`read_file`/`run_command`/…) that the engine routes to the bound
6946 /// substrate.
6947 /// 3. Swap the session executor to a [`SubstrateShadowExecutor`] over the
6948 /// same `share_with_fallback(ws_executor)` composition, so the bare
6949 /// substrate-owned names fall through to the substrate while connector
6950 /// routes and host tool callbacks keep working unchanged (option (a)).
6951 ///
6952 /// Returns the substrate's environment name (the connector slug) on
6953 /// success, or an error string if the connector is not connected.
6954 pub async fn bind_substrate_to_connector(
6955 &self,
6956 session: &Arc<ClientSession>,
6957 slug: &str,
6958 ) -> Result<String, String> {
6959 self.ensure_connectors_loaded().await;
6960
6961 // The connector's live MCP session is the same handle the
6962 // `mcp_{slug}_*` routes dispatch through — wrap it, don't re-dial.
6963 let mcp_session = self
6964 .mcp_executor
6965 .session(slug)
6966 .await
6967 .ok_or_else(|| format!("connector '{slug}' is not connected"))?;
6968
6969 let substrate: Arc<dyn car_engine::Substrate> =
6970 Arc::new(car_engine::McpSubstrate::new(mcp_session, slug.to_string()));
6971 let name = substrate.name().to_string();
6972
6973 // 1. Bind the substrate; 2. register substrate-backed built-ins.
6974 //
6975 // These three steps (set_substrate → register_agent_basics →
6976 // set_executor below) are not one critical section, so a tool `execute`
6977 // racing on this same session mid-swap could observe a half-applied
6978 // state. That's benign here: a WS connection processes its JSON-RPC
6979 // requests sequentially, and `bindSubstrate` is a client control call
6980 // issued before tool work — so no in-flight built-in overlaps the swap.
6981 session.runtime.set_substrate(substrate).await;
6982 session.runtime.register_agent_basics().await;
6983
6984 // 3. Shadow the substrate-owned names so they fall through to the
6985 // bound substrate, keeping connector routes + WS host callbacks.
6986 let ws_executor = Arc::new(WsToolExecutor::new(
6987 session.channel.clone(),
6988 session.negotiated_capabilities.clone(),
6989 session.halted.clone(),
6990 ));
6991 let composed: Arc<dyn ToolExecutor> =
6992 Arc::new(self.mcp_executor.share_with_fallback(ws_executor));
6993 let shadowed: Arc<dyn ToolExecutor> = Arc::new(SubstrateShadowExecutor::new(composed));
6994 session.runtime.set_executor(shadowed).await;
6995
6996 Ok(name)
6997 }
6998
6999 /// Remove a per-client session from the registry on disconnect.
7000 /// Returns the removed session if present so callers can drop any
7001 /// remaining strong refs (e.g. drain pending tool callbacks). Fix
7002 /// for MULTI-4 / WS-3 — without this, `state.sessions` retains
7003 /// `Arc<ClientSession>` for every connection that ever existed.
7004 pub async fn remove_session(&self, client_id: &str) -> Option<Arc<ClientSession>> {
7005 // Cancel any live detached tool invocations on this session's
7006 // runtime (linus review D3): after disconnect the per-session
7007 // registry is unreachable, so a still-running tool would be
7008 // uncancellable until daemon restart. Same teardown discipline
7009 // as run subscribers / chat sessions below.
7010 if let Some(session) = self.sessions.lock().await.get(client_id).cloned() {
7011 let n = session.runtime.tool_handles.cancel_all().await;
7012 if n > 0 {
7013 tracing::info!(
7014 client_id,
7015 cancelled = n,
7016 "cancelled detached tools at session teardown"
7017 );
7018 }
7019 }
7020
7021 let removed = {
7022 let _liveness_guard = self.run_resume_liveness.lock().await;
7023 let removed = self.sessions.lock().await.remove(client_id);
7024 let resume_negotiated = removed.as_ref().is_some_and(|session| {
7025 session
7026 .negotiated_capabilities
7027 .read()
7028 .map(|caps| caps.contains(car_proto::RUNS_RESUME_CAPABILITY))
7029 .unwrap_or(false)
7030 });
7031 if resume_negotiated {
7032 let expires_at = tokio::time::Instant::now() + self.run_resume_lease;
7033 let mut runs = self.runs.lock().await;
7034 for meta in runs
7035 .values_mut()
7036 .filter(|meta| meta.active_client_id == client_id && !meta.is_terminal())
7037 {
7038 meta.resume_lease = Some(RunResumeLease {
7039 disconnected_client_id: client_id.to_string(),
7040 expires_at,
7041 });
7042 }
7043 }
7044 removed
7045 };
7046 if let Some(session) = &removed {
7047 // #169: drop the agent_id → client_id binding so a
7048 // disconnected lifecycle agent can reconnect (or its
7049 // supervisor-respawned replacement can take the slot)
7050 // without colliding with the stale claim.
7051 let bound = session.agent_id.lock().await.clone();
7052 if let Some(id) = bound {
7053 let mut attached = self.attached_agents.lock().await;
7054 if attached.get(&id).map(String::as_str) == Some(client_id) {
7055 attached.remove(&id);
7056 }
7057 }
7058 // Drop the peer-messaging guard with the binding. The guard holds a
7059 // rate budget and a dedupe window keyed by agent *name*, so a
7060 // supervisor-respawned replacement taking the same name would
7061 // otherwise inherit the dead process's history — arriving
7062 // pre-throttled, or silently deduped against messages it never saw.
7063 let bound_for_guards = session.agent_id.lock().await.clone();
7064 if let Some(id) = bound_for_guards {
7065 self.peer_guards.lock().await.remove(&id);
7066 }
7067 // Drop any in-flight `agents.chat` sessions bound to this
7068 // client — either side disconnecting orphans the stream,
7069 // and a respawned agent's stray `agent.chat.event`
7070 // notifications must not race into a stale routing entry.
7071 // See `docs/proposals/agent-chat-surface.md`.
7072 let bound_agent = session.agent_id.lock().await.clone();
7073 let mut chats = self.chat_sessions.lock().await;
7074 chats.retain(|_, s| {
7075 if s.host_client_id == client_id {
7076 return false;
7077 }
7078 if let Some(agent_id) = &bound_agent {
7079 if &s.agent_id == agent_id {
7080 return false;
7081 }
7082 }
7083 true
7084 });
7085 // Drop the chat lock before the run sweep, which awaits the
7086 // grace window and re-locks `runs` — keeping locks disjoint
7087 // avoids holding `chat_sessions` across the sleep.
7088 drop(chats);
7089 // Same for in-process A2A chat collectors owned by this client.
7090 // Dropping the entry drops the sole sender, so the collecting task's
7091 // `recv()` returns `None` and it falls back immediately instead of
7092 // waiting out its 180s cap. Done after `drop(chats)` so we never
7093 // hold `chat_sessions` while locking `chat_collectors` (the
7094 // interceptor locks them collectors→sessions; staying disjoint here
7095 // avoids inverting that order).
7096 self.chat_collectors
7097 .lock()
7098 .await
7099 .retain(|_, c| c.host_client_id != client_id);
7100 // Agent run tracing (U4): drop this connection's live
7101 // run-trace subscriptions so its drain tasks end and the
7102 // registry doesn't accumulate stale `(run_id, client)` entries
7103 // across reconnects. Reconnect-durability is client-side: the
7104 // run stays subscribable; CarHost re-subscribes by `run_id` on
7105 // its new connection (R8). This is exempt from the chat
7106 // drain-and-synthesize-error path — a dropped trace subscriber
7107 // never marks the underlying run failed.
7108 self.drop_run_subscribers_for_client(client_id).await;
7109 // Browser-drawer subscriptions are per-connection too. Dropping
7110 // them also starts the control grace period on any view this
7111 // connection was DRIVING, so an agent is never parked forever
7112 // behind a controller whose app just quit (see
7113 // `browser_view::CONTROL_GRACE`).
7114 self.browser_views
7115 .drop_subscriptions_for_client(client_id)
7116 .await;
7117 // Only a host-client removal can change host connectivity. This
7118 // used to fan a redundant reverse call to every producer for every
7119 // agent/CLI disconnect too. The session is already out of
7120 // `sessions`, so a real host removal reads the post-removal truth.
7121 if session.is_host.load(std::sync::atomic::Ordering::Acquire) {
7122 let host_connected = self.any_host_connected().await;
7123 self.browser_views
7124 .broadcast_host_connected(host_connected)
7125 .await;
7126 }
7127 // And if this connection was a supervised agent PUBLISHING a
7128 // browser, that browser went with the process. Its views stay
7129 // registered reporting an empty browser, so the drawer sees the
7130 // change and a restarted process can replace them in place (see
7131 // `browser_relay::RelayProducer::note_disconnected`).
7132 self.browser_views
7133 .note_producer_disconnected(client_id)
7134 .await;
7135 // Coder event subscriptions are per-connection too; the session
7136 // itself keeps running (reconnect + `coder.subscribe { from_seq }`
7137 // replays what was missed).
7138 // Board watchers (`coder.watch`) and discussion streams are
7139 // per-connection too, and are dropped on the same boundary.
7140 crate::coder::rpc::drop_subscriptions_for_client(self, client_id).await;
7141 crate::coder::discuss::drop_subscriptions_for_client(self, client_id).await;
7142 // Agent run tracing (R5): any run this connection owns that
7143 // has no terminal record yet is swept to `Incomplete` after
7144 // a short grace window — long enough for an in-flight
7145 // `runs.complete` to land first so a healthy close is never
7146 // mislabeled.
7147 let _run_guard = session.run_lifecycle_guard.lock().await;
7148 self.sweep_runs_for_disconnect(session).await;
7149 }
7150 removed
7151 }
7152}
7153
7154#[cfg(test)]
7155mod durable_task_tests {
7156 use super::*;
7157
7158 #[tokio::test]
7159 async fn dropping_response_waiter_does_not_cancel_operation_or_release_its_guard() {
7160 let temp = tempfile::tempdir().unwrap();
7161 let state = Arc::new(ServerState::with_config(ServerStateConfig::new(
7162 temp.path().to_path_buf(),
7163 )));
7164 let overlap_guard = Arc::new(tokio::sync::Mutex::new(()));
7165 let operation_guard = overlap_guard.clone();
7166 let (started_tx, started_rx) = tokio::sync::oneshot::channel();
7167 let (release_tx, release_rx) = tokio::sync::oneshot::channel();
7168
7169 let response = state
7170 .spawn_durable_operation("auth.test", async move {
7171 let _held = operation_guard.lock().await;
7172 let _ = started_tx.send(());
7173 let _ = release_rx.await;
7174 Ok::<_, String>(serde_json::json!({ "ok": true }))
7175 })
7176 .await;
7177 started_rx.await.unwrap();
7178 drop(response);
7179
7180 assert!(
7181 overlap_guard.try_lock().is_err(),
7182 "dropping only the connection waiter must not release the operation guard"
7183 );
7184 release_tx.send(()).unwrap();
7185 let joined = tokio::time::timeout(
7186 std::time::Duration::from_secs(1),
7187 state.durable_tasks.lock().await.join_next(),
7188 )
7189 .await
7190 .expect("daemon-owned operation should finish")
7191 .expect("task should exist")
7192 .expect("task should join");
7193 assert_eq!(joined, "auth.test");
7194 assert!(
7195 overlap_guard.try_lock().is_ok(),
7196 "the guard releases only after the daemon-owned operation finishes"
7197 );
7198 }
7199}
7200
7201#[cfg(test)]
7202mod tool_timeout_tests {
7203 use super::*;
7204
7205 #[test]
7206 fn honors_action_timeout_over_default() {
7207 // The action's budget bounds the wait (+grace so the executor's own
7208 // deadline reaps first) — no 60s ceiling, no env read.
7209 assert_eq!(
7210 tool_callback_timeout(Some(180_000)),
7211 std::time::Duration::from_millis(180_000 + TOOL_TIMEOUT_GRACE_MS)
7212 );
7213 // A budget far above the old 60s ceiling is honored (the #259 bug),
7214 // and the wait always exceeds the budget (so the executor wins).
7215 assert!(tool_callback_timeout(Some(600_000)) > std::time::Duration::from_secs(600));
7216 assert!(tool_callback_timeout(Some(180_000)) >= std::time::Duration::from_millis(180_000));
7217 }
7218
7219 #[test]
7220 fn default_is_not_the_old_60s() {
7221 // With no action budget and no env override, the fallback is the
7222 // raised default — not the hardcoded 60s that reaped real tools.
7223 // (Asserted only when CAR_TOOL_TIMEOUT is unset, which it is in CI.)
7224 if std::env::var_os("CAR_TOOL_TIMEOUT").is_none() {
7225 assert_eq!(
7226 tool_callback_timeout(None),
7227 std::time::Duration::from_millis(DEFAULT_TOOL_TIMEOUT_MS)
7228 );
7229 assert!(
7230 DEFAULT_TOOL_TIMEOUT_MS > 60_000,
7231 "default must exceed the old 60s"
7232 );
7233 }
7234 }
7235}
7236
7237#[cfg(test)]
7238mod observer_mode_tests {
7239 use super::*;
7240
7241 fn journal_dir() -> PathBuf {
7242 let target = std::env::var_os("CARGO_TARGET_DIR")
7243 .map(std::path::PathBuf::from)
7244 .unwrap_or_else(|| {
7245 std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
7246 .join("..")
7247 .join("..")
7248 .join("target")
7249 });
7250 std::fs::create_dir_all(&target).ok();
7251 let target = std::fs::canonicalize(&target).unwrap_or(target);
7252 let tmp = tempfile::TempDir::new_in(&target).unwrap();
7253 let p = tmp.path().to_path_buf();
7254 std::mem::forget(tmp); // keep the dir alive for the test
7255 p
7256 }
7257
7258 #[test]
7259 fn supervisor_returns_observer_error_when_marker_set() {
7260 // Closes Parslee-ai/car-releases#44: the second car-server on
7261 // a host installs the observer marker after `with_paths`
7262 // returns AlreadyRunning. Subsequent `state.supervisor()`
7263 // calls must return a clear "observe-only" error mentioning
7264 // the manifest path — they must NOT retry user_default()
7265 // (which would re-acquire the lock and likely also fail).
7266 let state = ServerState::standalone(journal_dir());
7267 let fake_manifest = PathBuf::from("/tmp/fake-manifest-for-test.json");
7268 state
7269 .install_observer_manifest(fake_manifest.clone())
7270 .expect("install_observer_manifest succeeds on fresh state");
7271 assert_eq!(state.observer_manifest_path(), Some(&fake_manifest));
7272
7273 let err = state.supervisor().map(|_| ()).unwrap_err();
7274 assert!(
7275 err.contains("observe-only"),
7276 "error must mention observe-only mode: {err}"
7277 );
7278 assert!(
7279 err.contains("fake-manifest-for-test.json"),
7280 "error must surface the manifest path so operators know which daemon owns it: {err}"
7281 );
7282 }
7283
7284 #[test]
7285 fn install_observer_manifest_is_idempotent_per_path_collision() {
7286 let state = ServerState::standalone(journal_dir());
7287 let p = PathBuf::from("/tmp/manifest-a.json");
7288 let q = PathBuf::from("/tmp/manifest-b.json");
7289 state.install_observer_manifest(p.clone()).unwrap();
7290 // OnceLock::set returns the value back on collision.
7291 let err = state.install_observer_manifest(q.clone()).unwrap_err();
7292 assert_eq!(err, q);
7293 assert_eq!(state.observer_manifest_path(), Some(&p));
7294 }
7295
7296 #[test]
7297 fn supervisor_if_installed_does_not_lazy_init() {
7298 // The Heisenberg-subscribe guard: `host.subscribe`'s
7299 // identity path must use the non-acquiring read so a
7300 // purely observational client can't cause the daemon to
7301 // claim `<manifest>.lock` as a side effect of asking
7302 // about it. Fresh state has no supervisor installed.
7303 let state = ServerState::standalone(journal_dir());
7304 assert!(state.supervisor_if_installed().is_none());
7305 // observer_manifest_path should remain unset too — no
7306 // implicit init.
7307 assert!(state.observer_manifest_path().is_none());
7308 }
7309}
7310
7311#[cfg(test)]
7312mod substrate_binding_tests {
7313 use super::*;
7314 use car_engine::{McpSession, McpToolInfo};
7315 use car_ir::ActionProposal;
7316 use serde_json::json;
7317 use std::sync::Mutex as StdMutex;
7318
7319 fn journal_dir() -> PathBuf {
7320 let target = std::env::var_os("CARGO_TARGET_DIR")
7321 .map(std::path::PathBuf::from)
7322 .unwrap_or_else(|| {
7323 std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
7324 .join("..")
7325 .join("..")
7326 .join("target")
7327 });
7328 std::fs::create_dir_all(&target).ok();
7329 let target = std::fs::canonicalize(&target).unwrap_or(target);
7330 let tmp = tempfile::TempDir::new_in(&target).unwrap();
7331 let p = tmp.path().to_path_buf();
7332 std::mem::forget(tmp);
7333 p
7334 }
7335
7336 /// A minimal in-memory MCP session standing in for a connected `vm`
7337 /// connector. `read_text`/`write_text`/`run_command` behave like the
7338 /// `vm` bridge so we can assert a substrate-bound session's built-ins
7339 /// land here rather than on the WS client (which has no responder in a
7340 /// test and would hang/fail).
7341 struct FakeVmSession {
7342 name: String,
7343 files: Arc<StdMutex<std::collections::HashMap<String, String>>>,
7344 }
7345
7346 #[async_trait::async_trait]
7347 impl McpSession for FakeVmSession {
7348 async fn list_tools(&mut self) -> Result<Vec<McpToolInfo>, String> {
7349 Ok(vec![])
7350 }
7351 async fn call_tool(
7352 &mut self,
7353 name: &str,
7354 args: serde_json::Value,
7355 ) -> Result<serde_json::Value, String> {
7356 match name {
7357 "write_text" => {
7358 let p = args["path"].as_str().unwrap().to_string();
7359 let c = args["content"].as_str().unwrap().to_string();
7360 self.files.lock().unwrap().insert(p, c);
7361 Ok(json!("ok"))
7362 }
7363 "read_text" => {
7364 let p = args["path"].as_str().unwrap();
7365 let c = self
7366 .files
7367 .lock()
7368 .unwrap()
7369 .get(p)
7370 .cloned()
7371 .ok_or_else(|| "not found".to_string())?;
7372 Ok(json!(c))
7373 }
7374 "run_command" => {
7375 let command = args["command"].as_str().unwrap_or_default();
7376 if let Some(path) = command
7377 .split_once("[ -e '")
7378 .map(|(_, path)| path)
7379 .and_then(|path| path.split_once("' ]").map(|(path, _)| path))
7380 {
7381 let exists = self.files.lock().unwrap().contains_key(path);
7382 return Ok(json!({
7383 "stdout": "",
7384 "stderr": "",
7385 "exit_code": if exists { 0 } else { 1 }
7386 }));
7387 }
7388 Ok(json!({
7389 "stdout": "from-vm",
7390 "stderr": "",
7391 "exit_code": 0
7392 }))
7393 }
7394 other => Err(format!("unknown tool {other}")),
7395 }
7396 }
7397 fn name(&self) -> &str {
7398 &self.name
7399 }
7400 }
7401
7402 fn read_file_proposal(path: &str) -> ActionProposal {
7403 serde_json::from_value(json!({
7404 "source": "test",
7405 "actions": [{
7406 "id": "r0",
7407 "type": "tool_call",
7408 "tool": "read_file",
7409 "parameters": { "path": path },
7410 "dependencies": [],
7411 }],
7412 }))
7413 .expect("proposal deserializes")
7414 }
7415
7416 /// The outbound message sink is attached to every session runtime, so
7417 /// `messaging.send` is a real, executable tool rather than a schema the
7418 /// runtime would refuse. `with_message_sink` registers the sink and the
7419 /// schema together, so seeing the schema proves the sink is there.
7420 #[tokio::test]
7421 async fn session_runtime_has_the_messaging_send_tool() {
7422 let state = Arc::new(ServerState::standalone(journal_dir()));
7423 let session = state
7424 .create_session("c-messaging", Arc::new(WsChannel::test_stub()))
7425 .await
7426 .unwrap();
7427
7428 assert!(
7429 session
7430 .runtime
7431 .tools
7432 .read()
7433 .await
7434 .contains_key("messaging.send"),
7435 "every session must be able to execute messaging.send"
7436 );
7437 }
7438
7439 /// Default (no bind): the session runtime keeps a `LocalSubstrate`.
7440 /// This is the backward-compat invariant — every existing consumer
7441 /// (GUI, A2A-over-WS, connectors-as-tools, coder) sees no change.
7442 #[tokio::test]
7443 async fn default_session_substrate_is_local() {
7444 let state = Arc::new(ServerState::standalone(journal_dir()));
7445 let channel = Arc::new(WsChannel::test_stub());
7446 let session = state.create_session("c-default", channel).await.unwrap();
7447
7448 let sub = session.runtime.substrate().await;
7449 assert_eq!(
7450 sub.name(),
7451 "local",
7452 "an un-bound session must keep the default LocalSubstrate"
7453 );
7454 assert!(sub.is_local(), "default substrate must be the host");
7455 }
7456
7457 /// Binding fails cleanly when the named connector is not connected —
7458 /// no panic, a descriptive error, and the substrate stays Local.
7459 #[tokio::test]
7460 async fn bind_unknown_connector_errors_and_keeps_local() {
7461 let state = Arc::new(ServerState::standalone(journal_dir()));
7462 let channel = Arc::new(WsChannel::test_stub());
7463 let session = state.create_session("c-missing", channel).await.unwrap();
7464
7465 let err = state
7466 .bind_substrate_to_connector(&session, "nope")
7467 .await
7468 .unwrap_err();
7469 assert!(err.contains("not connected"), "got: {err}");
7470 assert_eq!(session.runtime.substrate().await.name(), "local");
7471 }
7472
7473 /// A connector-driven session that binds its substrate routes the bare
7474 /// commodity built-ins (`read_file`/`write_file`) to that connector's
7475 /// environment via the engine fall-through, NOT to the WS client. We
7476 /// prove it end-to-end: write+read through the runtime land on the fake
7477 /// VM session's file map.
7478 #[tokio::test]
7479 async fn bound_session_routes_builtins_to_substrate() {
7480 let state = Arc::new(ServerState::standalone(journal_dir()));
7481 let channel = Arc::new(WsChannel::test_stub());
7482 let session = state.create_session("c-vm", channel).await.unwrap();
7483
7484 // Register a connected "vm" connector session + its routes the way
7485 // the connector manager does on dial.
7486 let files = Arc::new(StdMutex::new(std::collections::HashMap::new()));
7487 let fake: Arc<Mutex<dyn McpSession>> = Arc::new(Mutex::new(FakeVmSession {
7488 name: "vm".into(),
7489 files: files.clone(),
7490 }));
7491 state.mcp_executor.add_session("vm", fake).await;
7492
7493 // Bind the session to the vm connector's substrate.
7494 let bound = state
7495 .bind_substrate_to_connector(&session, "vm")
7496 .await
7497 .expect("bind succeeds for a connected connector");
7498 assert_eq!(bound, "vm");
7499 assert_eq!(session.runtime.substrate().await.name(), "vm");
7500
7501 // write_file through the runtime → must hit the fake VM session,
7502 // not the WS client (which would hang/fail with no responder).
7503 let write = serde_json::from_value::<ActionProposal>(json!({
7504 "source": "test",
7505 "actions": [{
7506 "id": "w0",
7507 "type": "tool_call",
7508 "tool": "write_file",
7509 "parameters": { "path": "/vm/a.txt", "content": "vm-bytes" },
7510 "dependencies": [],
7511 }],
7512 }))
7513 .unwrap();
7514 let wres = session.runtime.execute(&write).await;
7515 assert!(
7516 wres.results[0].error.is_none(),
7517 "write_file via substrate failed: {:?}",
7518 wres.results[0].error
7519 );
7520 assert_eq!(
7521 files.lock().unwrap().get("/vm/a.txt").map(String::as_str),
7522 Some("vm-bytes"),
7523 "write_file must land on the bound VM substrate"
7524 );
7525
7526 // read_file back must return the VM's content.
7527 let rres = session
7528 .runtime
7529 .execute(&read_file_proposal("/vm/a.txt"))
7530 .await;
7531 assert!(
7532 rres.results[0].error.is_none(),
7533 "read_file via substrate failed: {:?}",
7534 rres.results[0].error
7535 );
7536 let out = rres.results[0].output.clone().unwrap_or(Value::Null);
7537 let content = out.get("content").and_then(|v| v.as_str()).unwrap_or("");
7538 // read_file output is line-numbered `cat -n` style (H1/F4-remainder,
7539 // audit 2026-07-06); the VM's single-line content becomes " 1\t<line>".
7540 assert_eq!(
7541 content, " 1\tvm-bytes",
7542 "read_file must come from the VM substrate"
7543 );
7544 }
7545
7546 /// The shadow executor: substrate-owned built-in names return the
7547 /// `"unknown tool"` prefix (so the engine falls through to the bound
7548 /// substrate), while any other tool name is delegated to the inner
7549 /// executor unchanged (connector `mcp_*` routes + host WS callbacks).
7550 #[tokio::test]
7551 async fn shadow_executor_only_shadows_builtin_names() {
7552 struct Inner;
7553 #[async_trait::async_trait]
7554 impl ToolExecutor for Inner {
7555 async fn execute(&self, tool: &str, _p: &Value) -> Result<Value, String> {
7556 Ok(json!({ "delegated": tool }))
7557 }
7558 }
7559 let shadow = SubstrateShadowExecutor::new(Arc::new(Inner));
7560
7561 for name in SUBSTRATE_OWNED_TOOLS {
7562 let err = shadow.execute(name, &json!({})).await.unwrap_err();
7563 assert!(
7564 err.starts_with("unknown tool"),
7565 "{name} must be shadowed so the engine falls through: {err}"
7566 );
7567 }
7568
7569 // A connector tool name and a pure tool pass through untouched.
7570 for passthrough in ["mcp_vm_run_command", "calculate", "browser.run"] {
7571 let out = shadow.execute(passthrough, &json!({})).await.unwrap();
7572 assert_eq!(out["delegated"], json!(passthrough));
7573 }
7574 }
7575}
7576
7577/// `.car/policies/*.toml` reaching the runtime at all — the whole point of
7578/// wiring `load_project_policies` to a real call site. Exercised against a
7579/// temp directory rather than through `create_session`, which would read the
7580/// developer's real `$HOME`.
7581#[cfg(test)]
7582mod project_policy_loading_tests {
7583 use super::*;
7584
7585 fn runtime() -> Runtime {
7586 Runtime::new()
7587 }
7588
7589 #[tokio::test]
7590 async fn a_missing_policies_directory_is_silent() {
7591 let dir = tempfile::tempdir().unwrap();
7592 // No `.car` at all, let alone `.car/policies`.
7593 apply_project_policies(&runtime(), &dir.path().join(".car"))
7594 .await
7595 .expect("a project with no rules must start normally");
7596 }
7597
7598 #[tokio::test]
7599 async fn an_empty_policies_directory_is_silent() {
7600 let dir = tempfile::tempdir().unwrap();
7601 let car = dir.path().join(".car");
7602 std::fs::create_dir_all(car.join("policies")).unwrap();
7603 apply_project_policies(&runtime(), &car).await.unwrap();
7604 }
7605
7606 #[tokio::test]
7607 async fn a_well_formed_rule_file_loads_and_takes_effect() {
7608 let dir = tempfile::tempdir().unwrap();
7609 let car = dir.path().join(".car");
7610 std::fs::create_dir_all(car.join("policies")).unwrap();
7611 // Exactly the shape `car_policy::rules`' module docs advertise: a
7612 // top-level list for `deny_tool`, a table array for the per-parameter
7613 // rules.
7614 std::fs::write(
7615 car.join("policies").join("messaging.toml"),
7616 "deny_tool = [\"messaging.send\"]\n\n\
7617 [[deny_tool_param]]\n\
7618 tool = \"messaging.send\"\n\
7619 param = \"channel\"\n\
7620 equals = \"slack\"\n",
7621 )
7622 .unwrap();
7623
7624 let rt = runtime();
7625 apply_project_policies(&rt, &car).await.unwrap();
7626
7627 // Registered on the runtime's engine, not merely parsed.
7628 let names: Vec<String> = rt
7629 .list_policies(None)
7630 .await
7631 .unwrap()
7632 .into_iter()
7633 .map(|(name, _)| name)
7634 .collect();
7635 assert!(
7636 names.len() >= 2,
7637 "both declarative rules must reach the policy engine, got {names:?}"
7638 );
7639 }
7640
7641 #[tokio::test]
7642 async fn a_malformed_rule_file_fails_loudly_and_names_the_file() {
7643 let dir = tempfile::tempdir().unwrap();
7644 let car = dir.path().join(".car");
7645 std::fs::create_dir_all(car.join("policies")).unwrap();
7646 std::fs::write(
7647 car.join("policies").join("broken.toml"),
7648 "[[deny_tool]\ntool = \"shell\"",
7649 )
7650 .unwrap();
7651
7652 let err = apply_project_policies(&runtime(), &car)
7653 .await
7654 .expect_err("a malformed policy file must NOT be swallowed");
7655 assert!(
7656 err.contains("broken.toml"),
7657 "the operator has to know which file to fix: {err}"
7658 );
7659 assert!(err.contains("refusing to start"), "{err}");
7660 }
7661}
7662
7663#[cfg(test)]
7664mod org_scope_wiring_tests {
7665 use super::*;
7666
7667 fn granter_hex_for(secret: &[u8], user: &str) -> String {
7668 let master = car_sync::StretchedMaster::from_issued_high_entropy(secret, user);
7669 let vk = car_sync::ed25519_verifying(&car_sync::derive_ed25519_identity(&master, user));
7670 vk.to_bytes().iter().map(|b| format!("{b:02x}")).collect()
7671 }
7672 fn granter_hex() -> String {
7673 granter_hex_for(b"granter-login", "acc_granter")
7674 }
7675
7676 #[test]
7677 fn unset_env_is_the_personal_only_path() {
7678 // The real-tenants-OFF invariant: no value → Ok(None) → base provider.
7679 assert!(ServerState::parse_org_scope_config(None).unwrap().is_none());
7680 assert!(ServerState::parse_org_scope_config(Some(String::new()))
7681 .unwrap()
7682 .is_none());
7683 }
7684
7685 #[test]
7686 fn wellformed_env_parses_org_and_granter() {
7687 let hex = granter_hex();
7688 let (org, granters) = ServerState::parse_org_scope_config(Some(format!("orgtest:{hex}")))
7689 .unwrap()
7690 .expect("well-formed opt-in parses");
7691 assert_eq!(org, "orgtest");
7692 assert_eq!(granters.len(), 1);
7693 let expected_master =
7694 car_sync::StretchedMaster::from_issued_high_entropy(b"granter-login", "acc_granter");
7695 let expected = car_sync::ed25519_verifying(&car_sync::derive_ed25519_identity(
7696 &expected_master,
7697 "acc_granter",
7698 ));
7699 assert_eq!(granters[0].to_bytes(), expected.to_bytes());
7700 }
7701
7702 #[test]
7703 fn multiple_granters_parse_in_order() {
7704 // The trusted set is plural: an org can designate several granters.
7705 let g1 = granter_hex_for(b"g1", "acc_g1");
7706 let g2 = granter_hex_for(b"g2", "acc_g2");
7707 let (org, granters) =
7708 ServerState::parse_org_scope_config(Some(format!("orgtest:{g1},{g2}")))
7709 .unwrap()
7710 .expect("well-formed multi-granter opt-in parses");
7711 assert_eq!(org, "orgtest");
7712 assert_eq!(granters.len(), 2);
7713 assert_ne!(granters[0].to_bytes(), granters[1].to_bytes());
7714 // Surrounding whitespace on a granter entry is tolerated.
7715 let spaced = ServerState::parse_org_scope_config(Some(format!("orgtest:{g1} , {g2}")))
7716 .unwrap()
7717 .unwrap();
7718 assert_eq!(spaced.1.len(), 2);
7719 }
7720
7721 #[test]
7722 fn one_bad_granter_in_the_list_fails_the_whole_value() {
7723 // A typo in ANY granter is a loud failure — never silently drop it and
7724 // proceed with a partial trusted set.
7725 let good = granter_hex();
7726 assert!(
7727 ServerState::parse_org_scope_config(Some(format!("orgtest:{good},not-hex"))).is_err(),
7728 "a malformed granter among valid ones must be a hard error"
7729 );
7730 // An org id with no granters at all is rejected.
7731 assert!(ServerState::parse_org_scope_config(Some("orgtest:".into())).is_err());
7732 assert!(ServerState::parse_org_scope_config(Some("orgtest: , ".into())).is_err());
7733 }
7734
7735 #[test]
7736 fn malformed_env_is_a_hard_error_never_silent() {
7737 // A typo must NOT silently fall through to a wrong/unauthenticated state.
7738 let hex = granter_hex();
7739 let bads = [
7740 "no-colon-here".to_string(), // missing ':'
7741 ":deadbeef".to_string(), // empty org
7742 "orgtest:not-hex".to_string(), // bad hex
7743 "orgtest:aabb".to_string(), // wrong length
7744 format!("bad org:{hex}"), // non-canonical org (space)
7745 format!("org/evil:{hex}"), // non-canonical org (delimiter)
7746 ];
7747 for bad in &bads {
7748 assert!(
7749 ServerState::parse_org_scope_config(Some(bad.clone())).is_err(),
7750 "malformed value {bad:?} must be a hard error"
7751 );
7752 }
7753 // NOTE: an all-zero (identity/small-order) key PARSES — from_bytes accepts
7754 // the encoding; verify_strict rejects it at verification, so such a granter
7755 // simply grants nothing (fail-closed), which is correct, not a parse error.
7756 }
7757
7758 #[test]
7759 fn scope_routing_sends_only_matching_org_to_the_org_subsystem() {
7760 use car_sync::Scope;
7761 let acme = Scope::Shared { org: "acme".into() };
7762 let globex = Scope::Shared {
7763 org: "globex".into(),
7764 };
7765
7766 // org-scope OFF (no holder): EVERYTHING routes to the personal subsystem —
7767 // the byte-identical invariant.
7768 assert_eq!(route_for_scope(&Scope::Personal, None), SyncRoute::User);
7769 assert_eq!(route_for_scope(&acme, None), SyncRoute::User);
7770
7771 // opted into "acme": only Scope::Shared{acme} routes to the org subsystem.
7772 assert_eq!(route_for_scope(&acme, Some("acme")), SyncRoute::Org);
7773 assert_eq!(
7774 route_for_scope(&Scope::Personal, Some("acme")),
7775 SyncRoute::User
7776 );
7777 assert_eq!(
7778 route_for_scope(&globex, Some("acme")),
7779 SyncRoute::User,
7780 "a different org must NOT go to acme's delivery subsystem"
7781 );
7782 }
7783}