Skip to main content

car_server_core/
session.rs

1//! Server-side session state — shared across all connections.
2
3use car_engine::{AdmissionGate, GateContext, GateOutcome, Runtime, ToolExecutor};
4use car_eventlog::EventLog;
5use car_ir::ToolFailure;
6use car_proto::{ToolCancelRequest, ToolExecuteRequest, ToolExecuteResponse};
7use futures::Sink;
8use serde::{Deserialize, Serialize};
9use serde_json::Value;
10use sha2::{Digest, Sha256};
11use std::collections::HashMap;
12use std::path::{Path, PathBuf};
13use std::pin::Pin;
14use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
15use std::sync::Arc;
16use tokio::sync::{mpsc, oneshot, Mutex};
17use tokio_tungstenite::tungstenite::{Error as WsError, Message};
18
19/// Type-erased WebSocket sink. The dispatch loop accepts either a
20/// `WebSocketStream<TcpStream>` (the legacy car-server TCP listener)
21/// or a `WebSocketStream<UnixStream>` (the daemon-as-default UDS
22/// listener) — both implement `Sink<Message, Error = WsError>` after
23/// the tungstenite handshake. Erasing the type here avoids cascading
24/// a generic parameter through every WsChannel / Session / ServerState
25/// touchpoint in the dispatcher.
26pub type WsSink = Pin<Box<dyn Sink<Message, Error = WsError> + Send + Unpin + 'static>>;
27
28/// Grace window applied on disconnect before a still-open run is marked
29/// `Incomplete` (agent run tracing, U1 — R5). Short on purpose: it only
30/// has to cover the gap between a healthy `runs.complete` being
31/// dispatched on a spawned task and its terminal record landing, not any
32/// real work. Long enough to absorb that scheduling jitter, short enough
33/// that a genuinely abandoned run is reported `Incomplete` promptly.
34pub const RUN_COMPLETE_GRACE: std::time::Duration = std::time::Duration::from_millis(250);
35
36/// Bounded in-process lease for a disconnected run whose producer negotiated
37/// `runs.resume.v1`. Ten seconds leaves practical headroom for a loaded Mac's
38/// Python controller to establish a fresh local socket, authenticate, and
39/// negotiate capabilities without turning this into restart recovery. Runs
40/// without the capability keep the original 250 ms behavior; an unclaimed
41/// resumable run still becomes `Incomplete`.
42pub const RUN_RESUME_LEASE: std::time::Duration = std::time::Duration::from_secs(10);
43
44/// Maximum time an async run-lifecycle mutation waits for the journal writer's
45/// durability acknowledgement. Five seconds matches the server's existing
46/// acknowledgement/grace budget and remains well inside the default mutating
47/// handler deadline. If an operator override cancels earlier, the async append
48/// still marks the exact row pending before its first yield.
49const CRITICAL_JOURNAL_ACKNOWLEDGEMENT_TIMEOUT: std::time::Duration =
50    std::time::Duration::from_secs(5);
51
52/// Startup uses the same durability budget as live lifecycle mutations. A
53/// stalled journal must fail the constructor before the daemon listens rather
54/// than occupying the startup thread forever.
55const STARTUP_RECONCILIATION_ACKNOWLEDGEMENT_TIMEOUT: std::time::Duration =
56    CRITICAL_JOURNAL_ACKNOWLEDGEMENT_TIMEOUT;
57
58/// Per-run observation bound for disconnect cleanup: the ten-second resume
59/// lease, two bounded five-second journal acknowledgement attempts (the second
60/// is an exact retry after durability-unknown), and scheduler headroom. Cleanup
61/// never clears the run or binding unless an acknowledgement arrives.
62pub const RUN_DISCONNECT_CLEANUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(22);
63
64const RUN_SUBSCRIBE_SUMMARY_STATE_RETRY_LIMIT: usize = 3;
65
66/// Stable admission reason returned after a callback terminally halts its
67/// WebSocket session. The halt is intentionally connection-local and dies on
68/// reconnect; durable run outcomes remain the responsibility of `runs.complete`.
69pub const SESSION_HALTED_REASON: &str =
70    "session halted by a terminal tool failure; reconnect or ask a host to call session.clear_halt";
71
72struct SessionHaltAdmissionGate {
73    halted: Arc<AtomicBool>,
74}
75
76#[async_trait::async_trait]
77impl AdmissionGate for SessionHaltAdmissionGate {
78    fn name(&self) -> &str {
79        "session_halt"
80    }
81
82    async fn check(
83        &self,
84        _proposal: &car_ir::ActionProposal,
85        _ctx: &GateContext<'_>,
86    ) -> GateOutcome {
87        if self.halted.load(Ordering::Acquire) {
88            GateOutcome::reject_all(SESSION_HALTED_REASON)
89        } else {
90            GateOutcome::Allow
91        }
92    }
93}
94
95/// Server-side credentials for continuing an A2A-owned A2UI surface.
96///
97/// This intentionally lives outside `car_a2ui::A2uiSurfaceOwner` so
98/// renderers can inspect surface ownership without receiving secrets.
99#[derive(Debug, Clone, Serialize, Deserialize)]
100#[serde(rename_all = "camelCase", tag = "type")]
101pub enum A2aRouteAuth {
102    None,
103    Bearer { token: String },
104    Header { name: String, value: String },
105}
106
107#[cfg(test)]
108mod run_reservation_liveness_tests {
109    use super::*;
110    use crate::run_store::RunStoreLookupGate;
111    use std::time::Duration;
112
113    fn pristine_run(run_id: &str, client_id: &str) -> RunMeta {
114        RunMeta {
115            run_id: run_id.to_string(),
116            agent_id: "agent-release".to_string(),
117            client_id: client_id.to_string(),
118            active_client_id: client_id.to_string(),
119            resume_predecessor_client_id: None,
120            resume_lease: None,
121            intent: "release without blocking the registry".to_string(),
122            outcome_description: None,
123            started_at: chrono::Utc::now(),
124            termination: None,
125            ended_at: None,
126            turns: Vec::new(),
127            start_committed: false,
128            pending_terminal: None,
129            cancellation_pending: None,
130            cancellation_receipt: None,
131            trace_corruption: None,
132            durability_generation: 0,
133        }
134    }
135
136    #[tokio::test]
137    async fn blocked_release_lookup_does_not_hold_global_runs_lock() {
138        let tmp = tempfile::TempDir::new().unwrap();
139        let gate = RunStoreLookupGate::default();
140        let state = Arc::new(ServerState::with_config(
141            ServerStateConfig::new(tmp.path().join("journals"))
142                .with_run_store_lookup_gate(gate.clone()),
143        ));
144        let session = state
145            .create_session("release-client", Arc::new(WsChannel::test_stub()))
146            .await
147            .unwrap();
148        let expected = pristine_run("release-candidate", &session.client_id);
149        assert!(matches!(
150            state.reserve_run(expected.clone()).await,
151            Ok(RunReservation::New)
152        ));
153
154        gate.block_next();
155        let release_state = state.clone();
156        let release_session = session.clone();
157        let release_expected = expected.clone();
158        let release = tokio::spawn(async move {
159            release_state
160                .release_unpersisted_run_reservation(&release_session, &release_expected)
161                .await
162        });
163        let wait_gate = gate.clone();
164        assert!(
165            tokio::task::spawn_blocking(move || {
166                wait_gate.wait_until_entered(Duration::from_secs(3))
167            })
168            .await
169            .unwrap(),
170            "release must reach the blocked durable lookup"
171        );
172
173        let registry = tokio::time::timeout(Duration::from_secs(1), state.runs.lock())
174            .await
175            .expect("blocked release lookup must not hold the global runs lock");
176        drop(registry);
177        gate.release();
178        assert_eq!(release.await.unwrap(), Ok(()));
179        assert!(state.run_meta(&expected.run_id).await.is_none());
180    }
181}
182
183/// Shared write half of the WebSocket, plus pending callback channels.
184/// `write` is type-erased via [`WsSink`] so the dispatcher can run
185/// against any transport-specific WebSocketStream (TCP or UDS today;
186/// axum-bridged in future) without templatizing every consumer.
187pub struct WsChannel {
188    pub write: Mutex<WsSink>,
189    /// Pending tool execution callbacks: request_id → oneshot sender
190    pub pending: Mutex<HashMap<String, oneshot::Sender<ToolExecuteResponse>>>,
191    /// Body-free correlation for active callbacks. A session owns at most one
192    /// active run, so every entry belongs to that run while its lifecycle
193    /// guard is held by proposal execution.
194    pub active_actions: Mutex<HashMap<String, String>>,
195    pub next_id: AtomicU64,
196}
197
198/// A WS connection is a host event sink (#418): `HostState` broadcasts
199/// `host.event` frames to subscribers through this trait, so it never has to
200/// know about the concrete `WsChannel` / tungstenite types.
201#[async_trait::async_trait]
202impl crate::host::EventSubscriber for WsChannel {
203    async fn send_text(&self, json: String) {
204        use futures::SinkExt;
205        // Bound both lock acquisition and the socket write. Host events are
206        // emitted from reducer transitions; a suspended client must not hold
207        // that reducer's announcement lock (and browser controls behind it)
208        // indefinitely through TCP backpressure.
209        let _ = tokio::time::timeout(std::time::Duration::from_secs(10), async {
210            self.write
211                .lock()
212                .await
213                .send(Message::Text(json.into()))
214                .await
215        })
216        .await;
217    }
218}
219
220impl WsChannel {
221    pub fn next_request_id(&self) -> String {
222        let id = self.next_id.fetch_add(1, Ordering::SeqCst);
223        format!("cb-{}", id)
224    }
225
226    /// Test-only stub that returns a WsChannel whose write sink drains
227    /// to nowhere. Used by `host.rs` tests that need a real
228    /// `Arc<WsChannel>` in the subscribers map (to exercise membership
229    /// checks like the cross-session resolve fan-out) without
230    /// constructing a tungstenite handshake. Never writes are
231    /// performed against this stub; if anything tries, the drain sink
232    /// quietly absorbs.
233    #[cfg(test)]
234    pub fn test_stub() -> Self {
235        use futures::sink::SinkExt;
236        let sink: WsSink = Box::pin(
237            futures::sink::drain()
238                .sink_map_err(|_| tokio_tungstenite::tungstenite::Error::ConnectionClosed),
239        );
240        WsChannel {
241            write: Mutex::new(sink),
242            pending: Mutex::new(HashMap::new()),
243            active_actions: Mutex::new(HashMap::new()),
244            next_id: AtomicU64::new(0),
245        }
246    }
247
248    /// Test-only channel whose write sink is *observable*: every
249    /// `Message::Text` payload written to it is appended to the returned
250    /// shared Vec. `test_stub` drains to nowhere, which makes it useless
251    /// for asserting that a specific frame — e.g. the `tools.cancel`
252    /// notification — actually reached the wire.
253    ///
254    /// The Vec is behind a `std::sync::Mutex` (not tokio's) because the
255    /// sink closure is synchronous; the lock is taken and released inside
256    /// the closure before the returned future is handed back, so a test
257    /// holding the lock to assert can never deadlock a concurrent write.
258    #[cfg(test)]
259    pub fn test_capture() -> (Self, std::sync::Arc<std::sync::Mutex<Vec<String>>>) {
260        let frames = std::sync::Arc::new(std::sync::Mutex::new(Vec::<String>::new()));
261        let sink: WsSink = Box::pin(futures::sink::unfold(
262            frames.clone(),
263            |frames: std::sync::Arc<std::sync::Mutex<Vec<String>>>, msg: Message| {
264                if let Message::Text(text) = &msg {
265                    // Scoped so the lock is released before we yield the
266                    // (already-ready) future back to the sink machinery.
267                    if let Ok(mut held) = frames.lock() {
268                        held.push(text.as_str().to_string());
269                    }
270                }
271                futures::future::ready(Ok::<_, WsError>(frames))
272            },
273        ));
274        (
275            WsChannel {
276                write: Mutex::new(sink),
277                pending: Mutex::new(HashMap::new()),
278                active_actions: Mutex::new(HashMap::new()),
279                next_id: AtomicU64::new(0),
280            },
281            frames,
282        )
283    }
284}
285
286/// In-flight `agents.chat` session bookkeeping. Created when a host
287/// client calls `agents.chat`, removed when the agent emits a terminal
288/// `agent.chat.event` (`kind: "done"` or `"error"`), when either side
289/// disconnects, or when the host cancels via `agents.chat.cancel`.
290///
291/// The session_id is host-supplied (or server-generated when omitted)
292/// and threads through every `agent.chat.event` notification so the
293/// server can route streamed deltas back to the originating host
294/// without needing per-session subscriptions. See
295/// `docs/proposals/agent-chat-surface.md` for the wire contract.
296#[derive(Debug, Clone)]
297pub struct ChatSession {
298    /// Agent that owns this chat — populated from
299    /// `attached_agents` at `agents.chat` dispatch time.
300    pub agent_id: String,
301    /// Client id of the host that issued `agents.chat`. The server
302    /// forwards `agent.chat.event` notifications back to *this* host
303    /// only, so two CarHost windows chatting with the same agent are
304    /// independent streams.
305    pub host_client_id: String,
306    /// Unix-seconds creation time — used by the future stale-session
307    /// sweeper to drop sessions whose agent died without emitting a
308    /// terminal event.
309    pub created_at: u64,
310    /// In-process cancellation flag for daemon-owned chat runs. External
311    /// attached agents receive `agent.chat.cancel` instead.
312    pub local_cancel: Option<Arc<AtomicBool>>,
313}
314
315/// Host-visible standing goal for a chat session. `goal.set` stores this keyed
316/// by `session_id`; `agents.chat` reuses it for that session when the turn does
317/// not pass an inline `goal`.
318#[derive(Debug, Clone, Serialize, Deserialize)]
319pub struct ChatGoalState {
320    pub session_id: String,
321    pub check: String,
322    pub max_iterations: u32,
323    pub status: String,
324    pub last_iteration: Option<u32>,
325    pub last_met: Option<bool>,
326    pub last_grounded: Option<bool>,
327    pub last_reason: Option<String>,
328    pub terminal_kind: Option<String>,
329    pub terminal_message: Option<String>,
330    pub updated_at: u64,
331}
332
333fn chat_goals_path_from_journal_dir(journal_dir: &Path) -> PathBuf {
334    let car_dir = journal_dir
335        .parent()
336        .map(PathBuf::from)
337        .unwrap_or_else(|| PathBuf::from("."));
338    car_dir.join("chat-goals.json")
339}
340
341fn load_chat_goals_from_disk(journal_dir: &Path) -> HashMap<String, ChatGoalState> {
342    let path = chat_goals_path_from_journal_dir(journal_dir);
343    let Ok(text) = std::fs::read_to_string(&path) else {
344        return HashMap::new();
345    };
346    match serde_json::from_str::<HashMap<String, ChatGoalState>>(&text) {
347        Ok(mut goals) => {
348            for goal in goals.values_mut() {
349                if goal.status == "running" {
350                    goal.status = "active".to_string();
351                }
352            }
353            goals
354        }
355        Err(e) => {
356            tracing::warn!(
357                path = %path.display(),
358                error = %e,
359                "chat-goals store was unreadable; starting with an empty in-memory goal registry"
360            );
361            HashMap::new()
362        }
363    }
364}
365
366fn write_chat_goals_atomic(
367    path: &std::path::Path,
368    goals: &HashMap<String, ChatGoalState>,
369) -> std::io::Result<()> {
370    use std::sync::atomic::{AtomicU64, Ordering};
371    static SEQ: AtomicU64 = AtomicU64::new(0);
372    if let Some(parent) = path.parent() {
373        std::fs::create_dir_all(parent)?;
374    }
375    let json = serde_json::to_vec_pretty(goals)?;
376    let seq = SEQ.fetch_add(1, Ordering::Relaxed);
377    let mut tmp_os = path.as_os_str().to_owned();
378    tmp_os.push(format!(".tmp.{}.{}", std::process::id(), seq));
379    let tmp = PathBuf::from(tmp_os);
380    std::fs::write(&tmp, json)?;
381    std::fs::rename(&tmp, path)
382}
383
384/// One normalized chunk of an `agent.chat` stream, delivered to an in-process
385/// [`chat_collectors`](ServerState::chat_collectors) sink (the A2A bridge). A
386/// `token` chunk carries a `delta`; a terminal chunk has `kind` `"done"`
387/// (success) or `"error"` (with `error` text).
388#[derive(Debug, Clone)]
389pub struct ChatStreamChunk {
390    pub kind: String,
391    pub delta: Option<String>,
392    pub error: Option<String>,
393}
394
395/// An in-process `agent.chat` collector entry: the sink that
396/// [`try_forward_agent_chat_event`](crate::handler::try_forward_agent_chat_event)
397/// feeds, plus the host client whose disconnect should abandon it. Holding the
398/// sole sender here means dropping the entry closes the stream, so the
399/// collecting task wakes immediately instead of waiting out its timeout.
400pub struct ChatCollector {
401    pub tx: tokio::sync::mpsc::UnboundedSender<ChatStreamChunk>,
402    pub host_client_id: String,
403}
404
405#[derive(Debug, Clone)]
406#[doc(hidden)]
407pub struct RunResumeLease {
408    pub disconnected_client_id: String,
409    pub expires_at: tokio::time::Instant,
410}
411
412/// Deterministic test seam at the authenticated completion boundary. It is
413/// inert unless explicitly armed through [`ServerStateConfig`].
414#[derive(Debug, Clone, Default)]
415#[doc(hidden)]
416pub struct RunCompletionFenceGate {
417    armed: Arc<std::sync::atomic::AtomicBool>,
418    entered: Arc<tokio::sync::Notify>,
419    release: Arc<tokio::sync::Notify>,
420}
421
422impl RunCompletionFenceGate {
423    pub fn block_next(&self) {
424        self.armed.store(true, std::sync::atomic::Ordering::Release);
425    }
426
427    pub async fn wait_until_entered(&self, timeout: std::time::Duration) -> bool {
428        tokio::time::timeout(timeout, self.entered.notified())
429            .await
430            .is_ok()
431    }
432
433    pub fn release(&self) {
434        self.armed
435            .store(false, std::sync::atomic::Ordering::Release);
436        self.release.notify_one();
437    }
438
439    async fn wait_if_armed(&self) {
440        if !self.armed.load(std::sync::atomic::Ordering::Acquire) {
441            return;
442        }
443        self.entered.notify_one();
444        if self.armed.load(std::sync::atomic::Ordering::Acquire) {
445            self.release.notified().await;
446        }
447    }
448}
449
450/// Daemon-side record of a single agent run (agent run tracing, U1).
451///
452/// Keyed by `run_id` in the process-wide [`ServerState::runs`] registry
453/// so the record survives the WS connection that produced it — needed
454/// both for the disconnect grace window (R5) and so the disk store (U3)
455/// can flush a run even after its session is gone. U1 keeps this purely
456/// in memory; persistence is U3.
457#[derive(Debug, Clone)]
458pub struct RunMeta {
459    pub run_id: String,
460    /// Owning agent. Set at `runs.start` from the resolved id; used by
461    /// the read RPCs' ownership check (U5/KTD10) and the disk key (U3).
462    pub agent_id: String,
463    /// Immutable WS `client_id` that durably called `runs.start`. This remains
464    /// the provenance identity written to `RunStarted`, proposal receipts, and
465    /// `RunEnded`, even if an authenticated replacement socket resumes the
466    /// in-memory run.
467    pub client_id: String,
468    /// The one live socket currently authorized to continue the run. Equal to
469    /// `client_id` for a fresh run; changed only by authenticated orphan
470    /// reclaim after the previous owner has disappeared from `sessions`.
471    pub active_client_id: String,
472    /// Stable predecessor returned by an idempotent `runs.resume` retry on the
473    /// winning replacement socket.
474    pub resume_predecessor_client_id: Option<String>,
475    /// Ephemeral, in-process authority window installed atomically when the
476    /// active socket leaves the session registry. It is intentionally absent
477    /// from RunStore/journals and therefore never survives daemon restart.
478    #[doc(hidden)]
479    pub resume_lease: Option<RunResumeLease>,
480    pub intent: String,
481    pub outcome_description: Option<String>,
482    pub started_at: chrono::DateTime<chrono::Utc>,
483    /// `None` while the run is in progress; `Some` once a terminal
484    /// record (a reported outcome or an `Incomplete` marker) is
485    /// written. The presence of this field is the "is this run still
486    /// open?" signal the grace window checks.
487    pub termination: Option<car_proto::RunTermination>,
488    /// When the terminal record was written, if any.
489    pub ended_at: Option<chrono::DateTime<chrono::Utc>>,
490    /// The ordered per-turn trace recorded for this run (agent run
491    /// tracing, U2). Each entry is a [`car_proto::RunRecord::Turn`]
492    /// produced by the recorder from a submitted proposal + its
493    /// `ActionResult`s. U1 leaves this empty; the U2 recorder appends to
494    /// it via [`ServerState::record_run_turns`]. U3 flushes this buffer
495    /// to disk and U4 broadcasts it — both read it through
496    /// [`ServerState::run_turns`]. The `turn.index` field is monotonic
497    /// across the run's proposals, so this Vec is the clean ordered
498    /// stream those units consume.
499    pub turns: Vec<car_proto::RunRecord>,
500    /// True only after the exact RunStarted row reached flush+fsync and the
501    /// authenticated run_started journal event received the same durability
502    /// acknowledgement. A reserved-but-uncommitted run blocks all proposal
503    /// writes and can only be resumed by its owning client.
504    pub start_committed: bool,
505    /// Stable terminal transaction preimage retained across a failed
506    /// RunStore or journal durability boundary. Retries must present the same
507    /// RunTermination and finish this exact RunEnded; a conflicting terminal
508    /// is rejected.
509    pub pending_terminal: Option<car_proto::RunEnded>,
510    /// Durable cancellation request. Presence quarantines new proposals and
511    /// `runs.complete` until CAR either confirms a Cancelled terminal or an
512    /// operator independently resolves the unconfirmed stop.
513    pub cancellation_pending: Option<car_proto::RunCancellationRequested>,
514    /// Stable response for same-key idempotent retries.
515    pub cancellation_receipt: Option<car_proto::RunCancelResponse>,
516    /// Stable fail-closed error once the durable JSONL is known corrupt.
517    /// A quarantined run accepts no more proposals/turns, cannot complete,
518    /// and cannot install or retain live subscribers.
519    pub trace_corruption: Option<String>,
520    /// Monotonic in-memory commit generation for durability work which drops
521    /// the global run-registry lock while filesystem I/O executes.
522    #[doc(hidden)]
523    pub durability_generation: u64,
524}
525
526#[derive(Debug, Clone)]
527pub enum RunReservation {
528    New,
529    Existing(RunMeta),
530}
531
532/// In-memory ownership result for the authenticated WebSocket resume path.
533/// `active_client_id` is the fenced socket allowed to use the original
534/// durable run provenance now.
535#[derive(Debug, Clone, PartialEq, Eq)]
536pub(crate) struct RunResumeBinding {
537    pub run_id: String,
538    pub agent_id: String,
539    pub active_client_id: String,
540    pub resumed_from_client_id: String,
541}
542
543/// Canonical terminal identity shared by the durable `RunEnded` row, terminal
544/// journal event, and `runs.complete` response. The input is exactly the RFC
545/// 8785/JCS serialization of `RunTermination`; it deliberately does not include
546/// timestamps, run ids, client ids, or Daily artifact bytes.
547pub(crate) fn run_completion_digest(
548    termination: &car_proto::RunTermination,
549) -> Result<String, String> {
550    let canonical = car_inference::catalog_identity::canonical_json(termination)?;
551    Ok(format!("{:x}", Sha256::digest(canonical.as_bytes())))
552}
553
554fn run_trace_corruption_message(run_id: &str, detail: impl std::fmt::Display) -> String {
555    format!(
556        "{} run `{run_id}`: {detail}",
557        car_proto::RUN_TRACE_CORRUPTION_MESSAGE_PREFIX
558    )
559}
560
561fn same_run_termination(
562    left: &car_proto::RunTermination,
563    right: &car_proto::RunTermination,
564) -> bool {
565    match (run_completion_digest(left), run_completion_digest(right)) {
566        (Ok(left), Ok(right)) => left == right,
567        _ => false,
568    }
569}
570
571fn same_run_ended(left: &car_proto::RunEnded, right: &car_proto::RunEnded) -> bool {
572    left.run_id == right.run_id
573        && left.client_id == right.client_id
574        && left.agent_id == right.agent_id
575        && left.completion_digest == right.completion_digest
576        && left.ended_at == right.ended_at
577        && same_run_termination(&left.termination, &right.termination)
578}
579
580struct StartupJournalCache {
581    journal_dir: PathBuf,
582    failures: Option<car_eventlog::JournalFailureInjector>,
583    logs: HashMap<String, EventLog>,
584}
585
586impl StartupJournalCache {
587    fn new(journal_dir: &Path, failures: Option<&car_eventlog::JournalFailureInjector>) -> Self {
588        Self {
589            journal_dir: journal_dir.to_path_buf(),
590            failures: failures.cloned(),
591            logs: HashMap::new(),
592        }
593    }
594
595    fn get(&mut self, client_id: &str) -> Result<&mut EventLog, String> {
596        if !self.logs.contains_key(client_id) {
597            let path = self.journal_dir.join(format!("{client_id}.jsonl"));
598            let log = match (path.exists(), self.failures.as_ref()) {
599                (true, Some(failures)) => {
600                    EventLog::load_with_journal_failure_injector(&path, failures.clone())
601                        .map_err(|error| error.to_string())
602                }
603                (true, None) => EventLog::load(&path).map_err(|error| error.to_string()),
604                (false, Some(failures)) => Ok(EventLog::with_journal_failure_injector(
605                    path,
606                    failures.clone(),
607                )),
608                (false, None) => Ok(EventLog::with_journal(path)),
609            }?;
610            self.logs.insert(client_id.to_string(), log);
611        }
612        self.logs
613            .get_mut(client_id)
614            .ok_or_else(|| "startup journal cache insertion failed".to_string())
615    }
616}
617
618fn append_recovered_cancellation_result_journal(
619    journal_dir: PathBuf,
620    failures: Option<car_eventlog::JournalFailureInjector>,
621    client_id: String,
622    result: car_proto::RunCancelResponse,
623) -> Result<bool, String> {
624    let mut journals = StartupJournalCache::new(&journal_dir, failures.as_ref());
625    let log = journals.get(&client_id)?;
626    log.bind_run(&result.run_id, &client_id)?;
627    let Value::Object(data) = serde_json::to_value(&result).map_err(|error| error.to_string())?
628    else {
629        return Err("recovered cancellation result did not serialize as an object".into());
630    };
631    let data: HashMap<String, Value> = data.into_iter().collect();
632    let already_durable = log.events().iter().any(|event| {
633        event.kind == car_eventlog::EventKind::RunCancellationResult
634            && event.run_id.as_deref() == Some(result.run_id.as_str())
635            && event.client_id.as_deref() == Some(client_id.as_str())
636            && event.action_id.as_deref() == result.action_id.as_deref()
637            && event.proposal_id.is_none()
638            && event.data == data
639    });
640    log.append_critical_bounded(
641        car_eventlog::EventKind::RunCancellationResult,
642        result.action_id.as_deref(),
643        None,
644        data,
645        CRITICAL_JOURNAL_ACKNOWLEDGEMENT_TIMEOUT,
646    )
647    .map_err(|error| format!("recovered cancellation journal append failed: {error}"))?;
648    Ok(!already_durable)
649}
650
651fn reconcile_durable_run_journals(
652    journals: &mut StartupJournalCache,
653    store: &crate::run_store::RunStore,
654    acknowledgement_timeout: std::time::Duration,
655) -> Result<(), String> {
656    type DurableRunBoundaries = (
657        car_proto::RunStarted,
658        Option<car_proto::RunEnded>,
659        Option<car_proto::RunCancellationRequested>,
660        Option<car_proto::RunCancelResponse>,
661    );
662    let mut by_client: HashMap<String, Vec<DurableRunBoundaries>> = HashMap::new();
663    store.visit_run_boundaries(|started, ended, requested, result| {
664        let Some(client_id) = started.client_id.as_deref() else {
665            return;
666        };
667        if client_id.is_empty()
668            || client_id.contains('/')
669            || client_id.contains('\\')
670            || client_id == "."
671            || client_id == ".."
672        {
673            tracing::error!(run_id = %started.run_id, "run-journal outbox has unsafe client_id");
674            return;
675        }
676        by_client
677            .entry(client_id.to_string())
678            .or_default()
679            .push((started, ended, requested, result));
680    });
681
682    for (client_id, mut runs) in by_client {
683        runs.sort_by_key(|(started, _, _, _)| started.started_at);
684        let log = match journals.get(&client_id) {
685            Ok(log) => log,
686            Err(error) => {
687                tracing::error!(%client_id, %error, "cannot load run journal for outbox reconciliation");
688                continue;
689            }
690        };
691        for (started, ended, requested, result) in runs {
692            if let Err(error) = log.bind_run(&started.run_id, &client_id) {
693                tracing::error!(run_id = %started.run_id, %error, "cannot bind run journal during outbox reconciliation");
694                continue;
695            }
696            let mut start_data = HashMap::from([
697                (
698                    "agent_id".to_string(),
699                    Value::from(started.agent_id.clone()),
700                ),
701                ("intent".to_string(), Value::from(started.intent.clone())),
702                (
703                    "started_at".to_string(),
704                    serde_json::to_value(started.started_at).unwrap_or(Value::Null),
705                ),
706            ]);
707            if let Some(description) = &started.outcome_description {
708                start_data.insert(
709                    "outcome_description".to_string(),
710                    Value::from(description.clone()),
711                );
712            }
713            log.append_critical_bounded(
714                car_eventlog::EventKind::RunStarted,
715                None,
716                None,
717                start_data,
718                acknowledgement_timeout,
719            )
720            .map_err(|error| {
721                format!(
722                    "startup reconciliation failed for run {} run_started: {error}",
723                    started.run_id
724                )
725            })?;
726
727            if let Some(requested) = requested {
728                let Value::Object(data) = serde_json::to_value(&requested).unwrap_or(Value::Null)
729                else {
730                    continue;
731                };
732                log.append_critical_bounded(
733                    car_eventlog::EventKind::RunCancellationRequested,
734                    requested.action_id.as_deref(),
735                    None,
736                    data.into_iter().collect(),
737                    acknowledgement_timeout,
738                )
739                .map_err(|error| {
740                    format!(
741                        "startup reconciliation failed for run {} cancellation request: {error}",
742                        started.run_id
743                    )
744                })?;
745            }
746
747            if ended.is_none() {
748                if let Some(result) = result {
749                    let Value::Object(data) = serde_json::to_value(&result).unwrap_or(Value::Null)
750                    else {
751                        continue;
752                    };
753                    log.append_critical_bounded(
754                        car_eventlog::EventKind::RunCancellationResult,
755                        result.action_id.as_deref(),
756                        None,
757                        data.into_iter().collect(),
758                        acknowledgement_timeout,
759                    )
760                    .map_err(|error| {
761                        format!(
762                            "startup reconciliation failed for run {} cancellation result: {error}",
763                            started.run_id
764                        )
765                    })?;
766                }
767            }
768
769            let Some(ended) = ended else {
770                continue;
771            };
772            let Some(completion_digest) = ended.completion_digest.as_deref() else {
773                log.clear_run_binding(&started.run_id, &client_id)?;
774                continue;
775            };
776            if ended.client_id.as_deref() != Some(client_id.as_str()) {
777                tracing::error!(run_id = %started.run_id, "RunEnded owner does not match RunStarted during outbox reconciliation");
778                log.clear_run_binding(&started.run_id, &client_id)?;
779                continue;
780            }
781            let termination_kind = match &ended.termination {
782                car_proto::RunTermination::Outcome { .. } => "outcome",
783                car_proto::RunTermination::Incomplete => "incomplete",
784                car_proto::RunTermination::Cancelled { .. } => "cancelled",
785            };
786            let data = HashMap::from([
787                (
788                    "termination_kind".to_string(),
789                    Value::from(termination_kind),
790                ),
791                (
792                    "completion_digest".to_string(),
793                    Value::from(completion_digest),
794                ),
795                (
796                    "termination".to_string(),
797                    serde_json::to_value(&ended.termination).unwrap_or(Value::Null),
798                ),
799            ]);
800            log.append_critical_bounded(
801                car_eventlog::EventKind::RunCompleted,
802                None,
803                None,
804                data,
805                acknowledgement_timeout,
806            )
807            .map_err(|error| {
808                format!(
809                    "startup reconciliation failed for run {} run_completed: {error}",
810                    started.run_id
811                )
812            })?;
813            log.clear_run_binding(&started.run_id, &client_id)?;
814        }
815    }
816    Ok(())
817}
818
819/// Finish proposal terminals whose actions already ran before the daemon lost
820/// the journal acknowledgement. The outbox owns the exact normal-serde result
821/// and JCS digest, so startup only replays that terminal row; it never invokes
822/// the runtime or a tool executor.
823fn reconcile_pending_proposal_journals(
824    journals: &mut StartupJournalCache,
825    store: &crate::run_store::RunStore,
826    acknowledgement_timeout: std::time::Duration,
827) -> Result<(), String> {
828    let pending_proposals = match store.all_pending_proposals() {
829        Ok(pending) => pending,
830        Err(error) => {
831            tracing::error!(%error, "cannot enumerate proposal-finalization outbox; all unresolved runs remain quarantined");
832            return Ok(());
833        }
834    };
835    for pending in pending_proposals {
836        match store.completed_proposal(
837            &pending.run_id,
838            &pending.client_id,
839            pending.requested_policy_session_id.as_deref(),
840            &pending.original_submission,
841        ) {
842            Ok(Some(receipt)) => {
843                if let Err(error) = store.cleanup_completed_proposal_guards(&receipt) {
844                    tracing::error!(run_id = %pending.run_id, %error, "completed proposal response is durable; startup guard cleanup remains pending");
845                }
846                continue;
847            }
848            Ok(None) => {}
849            Err(error) => {
850                tracing::error!(run_id = %pending.run_id, %error, "cannot validate exact completed proposal response during startup cleanup");
851                continue;
852            }
853        }
854        let (started, marker) = match store.pending_provenance(&pending) {
855            Ok(provenance) => provenance,
856            Err(error) => {
857                tracing::error!(run_id = %pending.run_id, %error, "proposal-finalization provenance is invalid; preserving outcome-unknown quarantine");
858                continue;
859            }
860        };
861        let Some(client_id) = started.client_id.as_deref() else {
862            tracing::error!(run_id = %pending.run_id, "durable RunStarted is missing client_id");
863            continue;
864        };
865        if client_id.is_empty()
866            || client_id.contains('/')
867            || client_id.contains('\\')
868            || client_id == "."
869            || client_id == ".."
870        {
871            tracing::error!(run_id = %pending.run_id, "proposal-finalization outbox has unsafe client_id");
872            continue;
873        }
874        let canonical = match car_inference::catalog_identity::canonical_json(
875            &pending.proposal_result,
876        ) {
877            Ok(canonical) => canonical,
878            Err(error) => {
879                tracing::error!(run_id = %pending.run_id, %error, "proposal-finalization result is not RFC 8785 canonicalizable");
880                continue;
881            }
882        };
883        let digest = format!("{:x}", Sha256::digest(canonical.as_bytes()));
884        if digest != pending.result_digest {
885            tracing::error!(run_id = %pending.run_id, "proposal-finalization digest does not match its durable result preimage");
886            continue;
887        }
888
889        let log = match journals.get(client_id) {
890            Ok(log) => log,
891            Err(error) => {
892                tracing::error!(run_id = %pending.run_id, %error, "cannot load proposal journal for outbox reconciliation");
893                continue;
894            }
895        };
896        if let Err(error) = log.bind_run(&started.run_id, client_id) {
897            tracing::error!(run_id = %pending.run_id, %error, "cannot bind proposal journal during outbox reconciliation");
898            continue;
899        }
900        if let Some(policy_session_id) = marker.policy_session_id.as_deref() {
901            if let Err(error) = log.bind_policy_session(policy_session_id) {
902                tracing::error!(run_id = %pending.run_id, %error, "cannot restore proposal policy binding during outbox reconciliation");
903                continue;
904            }
905        }
906        store.ensure_proposal_turns(&pending).map_err(|error| {
907            format!(
908                "startup reconciliation failed for run {} proposal trace: {error}",
909                pending.run_id
910            )
911        })?;
912        log.append_critical_bounded(
913            car_eventlog::EventKind::ProposalCompleted,
914            None,
915            Some(&pending.final_proposal_id),
916            pending.event_data(),
917            acknowledgement_timeout,
918        )
919        .map_err(|error| {
920            format!(
921                "startup reconciliation failed for run {} proposal_completed: {error}",
922                pending.run_id
923            )
924        })?;
925        if let Some(policy_session_id) = marker.policy_session_id.as_deref() {
926            log.clear_policy_session(policy_session_id)?;
927        }
928        let receipt = match store.write_completed_proposal(&pending) {
929            Ok(receipt) => receipt,
930            Err(error) => {
931                tracing::error!(run_id = %pending.run_id, %error, "cannot persist reconciled completed proposal response");
932                continue;
933            }
934        };
935        if let Err(error) = store.cleanup_completed_proposal_guards(&receipt) {
936            tracing::error!(run_id = %pending.run_id, %error, "completed proposal response is durable; startup guard cleanup remains pending");
937            continue;
938        }
939    }
940    Ok(())
941}
942
943/// A completed response can outlive either cleanup guard when the daemon dies
944/// between independently-fsynced unlink boundaries. Reconcile only under the
945/// receipt's exact typed authority; never dispatch or append another event.
946fn reconcile_completed_proposal_guards(store: &crate::run_store::RunStore) {
947    if let Err(error) = store.reconcile_completed_proposal_migration() {
948        tracing::error!(%error, "cannot migrate completed proposal response index; guards remain quarantined");
949    }
950}
951
952/// Per-run turn ceiling — a runaway-loop backstop sized well above any
953/// healthy main-agent-only cycle (tens of turns), never a trimmer. This is
954/// a TRUE hard cap, and it is enforced HERE — inside
955/// [`ServerState::record_run_turns`], under the `runs` lock — not only in
956/// the WS handler's pre-check. The handler keeps a fast-path pre-check off a
957/// lock-free snapshot, but the dispatcher spawns a task per frame, so
958/// pipelined `runs.record_turns` batches can all read a sub-ceiling snapshot
959/// and pass that pre-check before any of them appends. Only the under-lock
960/// check below is atomic with the append, so it is the one that actually
961/// bounds the run (ADV-1). A batch that would take the run PAST this many
962/// recorded turns is refused WHOLE.
963pub const RECORD_TURNS_RUN_CEILING: usize = 2000;
964
965/// Outcome of [`ServerState::record_run_turns`] — distinguishes the three
966/// reasons a batch can fail to land so the caller maps each to the right
967/// `runs.record_turns` drop reason (ADV-1). Before this enum the function
968/// returned a bare `usize` (the new total, or `0` for "nothing appended"),
969/// which collapsed an under-lock CEILING refusal and an unknown/terminal run
970/// into the same `0` — the handler then mislabeled a ceiling refusal as
971/// `run_terminal`.
972#[derive(Debug, Clone, PartialEq, Eq)]
973pub enum RecordRunTurnsOutcome {
974    /// The batch was appended; the run's new total turn count.
975    Appended { new_total: usize },
976    /// The batch was refused WHOLE because it would take the run past
977    /// [`RECORD_TURNS_RUN_CEILING`] (a runaway backstop). Maps to
978    /// `dropped: "run_turn_limit"`.
979    RefusedCeiling,
980    /// The run id is unknown to this process OR already terminal — nothing
981    /// was appended. Maps to `dropped: "run_terminal"` (the benign TOCTOU:
982    /// the run went terminal between the handler's pre-check and the
983    /// append), matching the prior silent-zero semantics.
984    UnknownOrTerminal,
985    /// The batch cannot advance canonical memory/subscribers. Ordinary I/O
986    /// failures mean the durable append boundary was not acknowledged and an
987    /// exact retry is safe. A `run trace corruption:` error instead means the
988    /// bytes exposed committed middle corruption; the run is quarantined and
989    /// must not be retried or completed.
990    PersistenceFailed(String),
991}
992
993pub(crate) enum RunSubscribePageResult {
994    Ready(car_proto::RunSubscribeResponse),
995    Durable {
996        agent_id: String,
997        status: car_proto::RunLiveStatus,
998    },
999}
1000
1001impl RunMeta {
1002    /// True once a terminal record (outcome or `Incomplete`) is set.
1003    pub fn is_terminal(&self) -> bool {
1004        self.termination.is_some()
1005    }
1006
1007    pub fn accepts_proposals(&self) -> bool {
1008        self.start_committed
1009            && self.pending_terminal.is_none()
1010            && self.cancellation_pending.is_none()
1011            && self.trace_corruption.is_none()
1012            && !self.is_terminal()
1013    }
1014
1015    /// The coarse live status of this run for the U4 subscribe snapshot /
1016    /// `runs.trace.event` (agent run tracing). Mirrors `RunStore`'s
1017    /// `RunStatus` but reads off the in-memory `RunMeta` so the live path
1018    /// never touches disk.
1019    pub fn live_status(&self) -> car_proto::RunLiveStatus {
1020        match &self.termination {
1021            None if self.cancellation_pending.is_some() => {
1022                car_proto::RunLiveStatus::CancellationPending
1023            }
1024            None => car_proto::RunLiveStatus::InProgress,
1025            Some(car_proto::RunTermination::Outcome { .. }) => car_proto::RunLiveStatus::Completed,
1026            Some(car_proto::RunTermination::Incomplete) => car_proto::RunLiveStatus::Incomplete,
1027            Some(car_proto::RunTermination::Cancelled { .. }) => {
1028                car_proto::RunLiveStatus::Cancelled
1029            }
1030        }
1031    }
1032
1033    /// Count of `RunRecord::Turn` entries in this run's buffer — the live
1034    /// turn cursor. Equals `turns.len()` because the buffer holds only
1035    /// `Turn` records (Started/Ended are not buffered here).
1036    pub fn turn_cursor(&self) -> usize {
1037        self.turns.len()
1038    }
1039}
1040
1041/// Default ceiling for the daemon→host `tools.execute` callback wait when
1042/// an action carries no explicit `timeout_ms`. Overridable via
1043/// `CAR_TOOL_TIMEOUT` (seconds). Raised from the old hardcoded 60s — real
1044/// tools (build steps, CLI drivers, slow APIs) routinely run longer, and a
1045/// 60s ceiling reaped them regardless of the agent's budget (Parslee-ai/car#259).
1046pub const DEFAULT_TOOL_TIMEOUT_MS: u64 = 300_000;
1047
1048/// Grace added to the callback wait above an action's own `timeout_ms`. The
1049/// executor (`car-engine`) already wraps each attempt in
1050/// `timeout(action.timeout_ms, dispatch)`; if this wait used the *same*
1051/// value it would race that outer deadline with zero slack. By waiting a
1052/// little longer, the executor's deadline fires first and reaps+rolls back
1053/// cleanly with its structured "action timed out" error — this wait is only
1054/// a transport backstop for the case the executor deadline doesn't apply.
1055const TOOL_TIMEOUT_GRACE_MS: u64 = 5_000;
1056
1057/// The callback-wait budget for one tool call.
1058///
1059/// - **`Some(X)`**: the executor is the authority (it bounds the attempt at
1060///   `X`); this wait is `X + grace`, a backstop that lets the executor reap
1061///   first. So the agent's per-action budget takes effect — the bug was that
1062///   this wait used a hardcoded 60s and ignored `X` entirely (car#259).
1063/// - **`None`**: the executor applies **no** deadline, so this wait is the
1064///   *sole* bound on the call — the `CAR_TOOL_TIMEOUT`-overridable
1065///   [`DEFAULT_TOOL_TIMEOUT_MS`].
1066fn tool_callback_timeout(action_timeout_ms: Option<u64>) -> std::time::Duration {
1067    let ms = match action_timeout_ms {
1068        Some(budget) => budget.saturating_add(TOOL_TIMEOUT_GRACE_MS),
1069        None => std::env::var("CAR_TOOL_TIMEOUT")
1070            .ok()
1071            .and_then(|s| s.trim().parse::<u64>().ok())
1072            .map(|secs| secs.saturating_mul(1000))
1073            .unwrap_or(DEFAULT_TOOL_TIMEOUT_MS),
1074    };
1075    std::time::Duration::from_millis(ms)
1076}
1077
1078/// Build and write the fire-and-forget `tools.cancel` notification for a
1079/// cancelled `tools.execute` callback (Parslee-ai/car#264).
1080///
1081/// This is the ONE place that constructs a [`ToolCancelRequest`] frame and
1082/// puts it on the wire; every cancellation path routes through
1083/// [`PendingToolCall`]'s drop, which calls this. Best-effort by design — the
1084/// daemon has already given up on the call, so a failed write changes nothing
1085/// it can act on.
1086pub(crate) async fn write_tool_cancel(
1087    channel: &WsChannel,
1088    request_id: String,
1089    action_id: String,
1090    reason: String,
1091) {
1092    use futures::SinkExt;
1093    let cancel = ToolCancelRequest {
1094        request_id,
1095        action_id,
1096        reason,
1097    };
1098    let notification = serde_json::json!({
1099        "jsonrpc": "2.0",
1100        "method": "tools.cancel",
1101        "params": cancel,
1102    });
1103    let Ok(text) = serde_json::to_string(&notification) else {
1104        return;
1105    };
1106    let _ = channel
1107        .write
1108        .lock()
1109        .await
1110        .send(Message::Text(text.into()))
1111        .await;
1112}
1113
1114/// RAII guard covering one in-flight daemon→host `tools.execute` callback
1115/// (Parslee-ai/car#264).
1116///
1117/// Armed from the moment the oneshot sender lands in
1118/// [`WsChannel::pending`]; disarmed only when a `ToolExecuteResponse` is
1119/// actually received. If it is still armed when it drops, the call was
1120/// cancelled and the guard both releases the `pending` entry and tells the
1121/// host to abort the in-flight child.
1122///
1123/// Why RAII rather than an explicit cleanup branch: `tokio::time::timeout`
1124/// **drops** the inner future on expiry, and the daemon's callback wait is
1125/// deliberately the *outermost* of three deadlines
1126/// (`TOOL_TIMEOUT_GRACE_MS` above the executor's per-action budget). So on
1127/// every real reap the whole dispatch future is dropped while parked on
1128/// `rx`, and any code written after the `await` — including the explicit
1129/// timeout arm — never runs. A `Drop` impl is the only cleanup that survives
1130/// being dropped.
1131struct PendingToolCall {
1132    channel: Arc<WsChannel>,
1133    request_id: String,
1134    action_id: String,
1135    reason: String,
1136    armed: bool,
1137}
1138
1139impl PendingToolCall {
1140    fn new(channel: Arc<WsChannel>, request_id: String, action_id: String, reason: String) -> Self {
1141        Self {
1142            channel,
1143            request_id,
1144            action_id,
1145            reason,
1146            armed: true,
1147        }
1148    }
1149
1150    /// A response was delivered — this call completed, so no cancel is owed.
1151    fn disarm(&mut self) {
1152        self.armed = false;
1153    }
1154}
1155
1156impl Drop for PendingToolCall {
1157    fn drop(&mut self) {
1158        if !self.armed {
1159            return;
1160        }
1161        // `pending` is a `tokio::sync::Mutex` and the write is async, but
1162        // `Drop` is sync — so the cleanup has to be a detached task. Outside
1163        // a runtime (a plain `drop` in sync code) there is nothing to spawn
1164        // onto; skip quietly rather than panicking out of a destructor.
1165        let Ok(handle) = tokio::runtime::Handle::try_current() else {
1166            return;
1167        };
1168        let channel = self.channel.clone();
1169        let request_id = std::mem::take(&mut self.request_id);
1170        let action_id = std::mem::take(&mut self.action_id);
1171        let reason = std::mem::take(&mut self.reason);
1172        handle.spawn(async move {
1173            // Only cancel if WE claimed the pending entry. The response
1174            // demux in `handler.rs` removes it on the success path, and
1175            // connection teardown clears the whole map — in either case the
1176            // host is not waiting on anything we should abort.
1177            let claimed = channel.pending.lock().await.remove(&request_id).is_some();
1178            channel.active_actions.lock().await.remove(&request_id);
1179            if claimed {
1180                write_tool_cancel(&channel, request_id, action_id, reason).await;
1181            }
1182        });
1183    }
1184}
1185
1186/// Tool executor that sends callbacks to the client over WebSocket.
1187pub struct WsToolExecutor {
1188    pub channel: Arc<WsChannel>,
1189    negotiated_capabilities: Arc<std::sync::RwLock<std::collections::BTreeSet<String>>>,
1190    halted: Arc<AtomicBool>,
1191}
1192
1193#[async_trait::async_trait]
1194impl ToolExecutor for WsToolExecutor {
1195    async fn execute(&self, tool: &str, params: &Value) -> Result<Value, String> {
1196        // Legacy callers that don't have a proposal-level Action.id
1197        // (e.g. internal `executor.execute` chains in tests) — emit an
1198        // empty action_id so the client-side handler can still see the
1199        // payload shape and decide whether to fail loudly.
1200        self.execute_with_action(tool, params, "", None).await
1201    }
1202
1203    async fn execute_with_action(
1204        &self,
1205        tool: &str,
1206        params: &Value,
1207        action_id: &str,
1208        timeout_ms: Option<u64>,
1209    ) -> Result<Value, String> {
1210        self.execute_with_action_in_session(tool, params, action_id, timeout_ms, None, 1)
1211            .await
1212    }
1213
1214    async fn execute_with_action_in_session(
1215        &self,
1216        tool: &str,
1217        params: &Value,
1218        action_id: &str,
1219        timeout_ms: Option<u64>,
1220        session_id: Option<&str>,
1221        attempt: u32,
1222    ) -> Result<Value, String> {
1223        self.execute_callback(
1224            tool,
1225            params,
1226            action_id,
1227            timeout_ms,
1228            session_id,
1229            attempt,
1230            &HashMap::new(),
1231            None,
1232            false,
1233        )
1234        .await
1235        .map(|execution| execution.output)
1236        .map_err(|failure| failure.message)
1237    }
1238
1239    async fn execute_with_action_state_in_session(
1240        &self,
1241        tool: &str,
1242        params: &Value,
1243        action_id: &str,
1244        timeout_ms: Option<u64>,
1245        session_id: Option<&str>,
1246        attempt: u32,
1247        expected_effects: &HashMap<String, Value>,
1248        return_schema: Option<&Value>,
1249    ) -> Result<car_engine::ToolExecution, String> {
1250        self.execute_callback(
1251            tool,
1252            params,
1253            action_id,
1254            timeout_ms,
1255            session_id,
1256            attempt,
1257            expected_effects,
1258            return_schema,
1259            self.callback_state_negotiated(),
1260        )
1261        .await
1262        .map_err(|failure| failure.message)
1263    }
1264
1265    async fn execute_classified(
1266        &self,
1267        tool: &str,
1268        params: &Value,
1269        action_id: &str,
1270        timeout_ms: Option<u64>,
1271        session_id: Option<&str>,
1272        attempt: u32,
1273        expected_effects: &HashMap<String, Value>,
1274        return_schema: Option<&Value>,
1275    ) -> Result<car_engine::ToolExecution, ToolFailure> {
1276        self.execute_callback(
1277            tool,
1278            params,
1279            action_id,
1280            timeout_ms,
1281            session_id,
1282            attempt,
1283            expected_effects,
1284            return_schema,
1285            self.callback_state_negotiated(),
1286        )
1287        .await
1288    }
1289}
1290
1291impl WsToolExecutor {
1292    pub fn new(
1293        channel: Arc<WsChannel>,
1294        negotiated_capabilities: Arc<std::sync::RwLock<std::collections::BTreeSet<String>>>,
1295        halted: Arc<AtomicBool>,
1296    ) -> Self {
1297        Self {
1298            channel,
1299            negotiated_capabilities,
1300            halted,
1301        }
1302    }
1303
1304    #[cfg(test)]
1305    fn legacy(channel: Arc<WsChannel>) -> Self {
1306        Self::new(
1307            channel,
1308            Arc::new(std::sync::RwLock::new(std::collections::BTreeSet::new())),
1309            Arc::new(AtomicBool::new(false)),
1310        )
1311    }
1312
1313    fn callback_state_negotiated(&self) -> bool {
1314        self.negotiated_capabilities
1315            .read()
1316            .map(|capabilities| capabilities.contains(car_proto::TOOLS_CALLBACK_STATE_CAPABILITY))
1317            .unwrap_or(false)
1318    }
1319
1320    #[allow(clippy::too_many_arguments)]
1321    async fn execute_callback(
1322        &self,
1323        tool: &str,
1324        params: &Value,
1325        action_id: &str,
1326        timeout_ms: Option<u64>,
1327        session_id: Option<&str>,
1328        attempt: u32,
1329        expected_effects: &HashMap<String, Value>,
1330        return_schema: Option<&Value>,
1331        callback_state_negotiated: bool,
1332    ) -> Result<car_engine::ToolExecution, ToolFailure> {
1333        use futures::SinkExt;
1334
1335        // The JSON-RPC request id is the daemon's callback-routing key
1336        // (used by the pending-response map below). The `action_id`
1337        // FIELD on the payload is the originating proposal Action.id
1338        // surfaced to the host so process-wide handlers can route
1339        // concurrent callbacks back to per-call dispatchers
1340        // (Parslee-ai/car-releases#43 follow-up). They serve different
1341        // purposes and must stay distinct: routing id is daemon-side,
1342        // action id is host-side.
1343        let request_id = self.channel.next_request_id();
1344
1345        let callback = ToolExecuteRequest {
1346            action_id: action_id.to_string(),
1347            tool: tool.to_string(),
1348            parameters: params.clone(),
1349            // Surface the action's budget to the host so its own tool runner
1350            // can bound the work too (Parslee-ai/car#259) — was always None.
1351            timeout_ms,
1352            // The engine's real retry counter, 1-based. This was hardcoded to
1353            // `1`, which made the field a constant on the wire: a host building
1354            // the retry-disambiguating join the field exists for got a key that
1355            // never varied, and would only find out under the concurrent-retry
1356            // conditions the join is for (Parslee-ai/car#928).
1357            attempt,
1358            // Surface the routing id so the host can key a per-call abort
1359            // registry on it; a reap emits `tools.cancel` carrying the same id
1360            // (Parslee-ai/car#264).
1361            request_id: request_id.clone(),
1362            // Server-stamped correlation. The executor has always passed this
1363            // in; the parameter was `_session_id` and the value was discarded,
1364            // so every host that needed attribution had to rebuild it from
1365            // client-side convention (Parslee-ai/car#904).
1366            session_id: session_id.map(str::to_string),
1367        };
1368
1369        // Create a oneshot channel for the response
1370        let (tx, rx) = oneshot::channel();
1371        self.channel
1372            .pending
1373            .lock()
1374            .await
1375            .insert(request_id.clone(), tx);
1376        self.channel
1377            .active_actions
1378            .lock()
1379            .await
1380            .insert(request_id.clone(), action_id.to_string());
1381
1382        // From here on the call is in flight. The guard makes cleanup +
1383        // host-abort survive ANY cancellation of this future — the
1384        // executor's per-action `timeout_ms` deadline (the common case, it
1385        // fires `TOOL_TIMEOUT_GRACE_MS` before our own wait below), the
1386        // server's per-method deadline in `handler.rs`, or the callback wait
1387        // here — not just the one arm we can write code in
1388        // (Parslee-ai/car#264).
1389        let mut pending_call = PendingToolCall::new(
1390            self.channel.clone(),
1391            request_id.clone(),
1392            action_id.to_string(),
1393            format!("tool '{tool}' call cancelled before completion (request {request_id})"),
1394        );
1395
1396        // Send the callback to the client as a JSON-RPC request
1397        let rpc_request = serde_json::json!({
1398            "jsonrpc": "2.0",
1399            "method": "tools.execute",
1400            "params": callback,
1401            "id": request_id,
1402        });
1403
1404        let msg = Message::Text(
1405            serde_json::to_string(&rpc_request)
1406                .map_err(|e| e.to_string())?
1407                .into(),
1408        );
1409        self.channel
1410            .write
1411            .lock()
1412            .await
1413            .send(msg)
1414            .await
1415            .map_err(|e| format!("failed to send tool callback: {}", e))?;
1416
1417        // Wait for the client to respond, bounded by the action's budget
1418        // (or the configurable default) — NOT a hardcoded 60s. This is the
1419        // ceiling that previously ignored `Action.timeout_ms` and reaped
1420        // legitimately-long tool calls (Parslee-ai/car#259).
1421        let wait = tool_callback_timeout(timeout_ms);
1422        let response = match tokio::time::timeout(wait, rx).await {
1423            Ok(inner) => inner.map_err(|_| format!("tool '{}' callback channel closed", tool))?,
1424            Err(_) => {
1425                // Reaped by our own callback wait. Sharpen the guard's reason
1426                // to name the timeout, then let it do the cleanup + host abort
1427                // on the way out — the same path a dropped future takes
1428                // (Parslee-ai/car#264).
1429                let reason = format!("tool '{}' callback timed out ({}s)", tool, wait.as_secs());
1430                pending_call.reason = reason.clone();
1431                return Err(ToolFailure::ordinary(reason));
1432            }
1433        };
1434
1435        // A response was delivered (error field set or not): the call is
1436        // finished, so no cancel is owed to the host.
1437        self.channel.active_actions.lock().await.remove(&request_id);
1438        pending_call.disarm();
1439
1440        if let Some(err) = response.error {
1441            let failure = match response.terminal {
1442                false => ToolFailure::ordinary(err),
1443                true => {
1444                    // Latch at the callback boundary, before returning to the
1445                    // engine. Handler-level result inspection repeats this as
1446                    // defense in depth, but this write closes the cancellation
1447                    // gap between typed evidence arriving and execute() ending.
1448                    self.halted.store(true, Ordering::Release);
1449                    ToolFailure::terminal(err)
1450                }
1451            };
1452            return Err(failure);
1453        }
1454        decode_callback_execution(
1455            tool,
1456            action_id,
1457            response.output.unwrap_or(Value::Null),
1458            expected_effects,
1459            return_schema,
1460            callback_state_negotiated,
1461        )
1462        .map_err(ToolFailure::ordinary)
1463    }
1464}
1465
1466fn decode_callback_execution(
1467    tool: &str,
1468    action_id: &str,
1469    result: Value,
1470    expected_effects: &HashMap<String, Value>,
1471    return_schema: Option<&Value>,
1472    callback_state_negotiated: bool,
1473) -> Result<car_engine::ToolExecution, String> {
1474    if !callback_state_negotiated {
1475        return Ok(car_engine::ToolExecution::output_only(result));
1476    }
1477    let envelope = result.as_object().filter(|object| {
1478        object.len() == 2 && object.contains_key("output") && object.contains_key("state_changes")
1479    });
1480
1481    if expected_effects.is_empty() && envelope.is_none() {
1482        return Ok(car_engine::ToolExecution::output_only(result));
1483    }
1484    let envelope = envelope.ok_or_else(|| {
1485        format!(
1486            "tool '{tool}' callback for action '{action_id}' must return exact envelope {{output,state_changes}}"
1487        )
1488    })?;
1489    let output = envelope
1490        .get("output")
1491        .cloned()
1492        .expect("exact envelope contains output");
1493    let state_changes: HashMap<String, Value> = serde_json::from_value(
1494        envelope
1495            .get("state_changes")
1496            .cloned()
1497            .expect("exact envelope contains state_changes"),
1498    )
1499    .map_err(|_| {
1500        format!("tool '{tool}' callback for action '{action_id}' state_changes must be an object")
1501    })?;
1502    let expected_keys: std::collections::BTreeSet<&str> =
1503        expected_effects.keys().map(String::as_str).collect();
1504    let actual_keys: std::collections::BTreeSet<&str> =
1505        state_changes.keys().map(String::as_str).collect();
1506    if expected_keys != actual_keys {
1507        return Err(format!(
1508            "tool '{tool}' callback state_changes keys do not match action '{action_id}': expected {:?}, got {:?}",
1509            expected_keys, actual_keys
1510        ));
1511    }
1512    if let Some(schema) = return_schema {
1513        car_engine::validate_tool_output(tool, schema, &output)?;
1514    }
1515    let changes_value = serde_json::to_value(&state_changes).map_err(|error| {
1516        format!("tool '{tool}' callback state_changes serialization failed: {error}")
1517    })?;
1518    car_inference::catalog_identity::canonical_json(&changes_value).map_err(|error| {
1519        format!("tool '{tool}' callback state_changes failed JCS/I-JSON validation: {error}")
1520    })?;
1521    Ok(car_engine::ToolExecution {
1522        output,
1523        state_changes,
1524    })
1525}
1526
1527#[cfg(test)]
1528mod tool_cancel_tests {
1529    use super::*;
1530    use serde_json::json;
1531    use std::time::Duration;
1532
1533    /// Every `Message::Text` the executor wrote, parsed as JSON.
1534    fn written(frames: &std::sync::Arc<std::sync::Mutex<Vec<String>>>) -> Vec<Value> {
1535        frames
1536            .lock()
1537            .unwrap()
1538            .iter()
1539            .map(|t| serde_json::from_str::<Value>(t).expect("frame is JSON"))
1540            .collect()
1541    }
1542
1543    fn frame_with_method<'a>(frames: &'a [Value], method: &str) -> Option<&'a Value> {
1544        frames.iter().find(|f| f["method"] == json!(method))
1545    }
1546
1547    /// Let detached cleanup tasks (the `PendingToolCall` drop spawn) run.
1548    async fn settle() {
1549        for _ in 0..8 {
1550            tokio::task::yield_now().await;
1551        }
1552        tokio::time::sleep(Duration::from_millis(100)).await;
1553        for _ in 0..8 {
1554            tokio::task::yield_now().await;
1555        }
1556    }
1557
1558    /// The daemon's own callback wait expiring still cancels — the arm that
1559    /// used to hold the inline cleanup now delegates to the drop guard, and
1560    /// the observable behaviour (Err text, cancel frame, drained `pending`)
1561    /// must be unchanged.
1562    #[tokio::test(start_paused = true)]
1563    async fn callback_wait_expiry_cancels() {
1564        let (channel, frames) = WsChannel::test_capture();
1565        let channel = Arc::new(channel);
1566        let executor = WsToolExecutor::legacy(channel.clone());
1567
1568        // 1ms budget → a ~5s callback wait; paused time makes that instant.
1569        let err = executor
1570            .execute_with_action_in_session("drive_cli", &json!({}), "a0", Some(1), None, 1)
1571            .await
1572            .expect_err("the callback wait must expire");
1573        assert!(
1574            err.contains("tool 'drive_cli' callback timed out"),
1575            "unexpected error text: {err}"
1576        );
1577
1578        settle().await;
1579
1580        let frames = written(&frames);
1581        let execute = frame_with_method(&frames, "tools.execute").expect("tools.execute was sent");
1582        let cancel = frame_with_method(&frames, "tools.cancel")
1583            .expect("a tools.cancel notification must reach the wire");
1584        assert_eq!(
1585            cancel["params"]["request_id"], execute["params"]["request_id"],
1586            "cancel must correlate to the originating request_id"
1587        );
1588        assert_eq!(cancel["params"]["action_id"], json!("a0"));
1589        assert!(
1590            cancel["id"].is_null(),
1591            "cancel is a notification, not a request"
1592        );
1593        assert!(
1594            channel.pending.lock().await.is_empty(),
1595            "the pending entry must be released"
1596        );
1597    }
1598
1599    /// Parslee-ai/car#904 — the execution session must reach the wire.
1600    ///
1601    /// The executor has always passed a session id in; the parameter was named
1602    /// `_session_id` and the value was dropped on the floor, so every host that
1603    /// needed to know which mission a callback belonged to had to rebuild it
1604    /// from client-side convention. Asserted on the actual `tools.execute`
1605    /// frame rather than on the struct, because the struct field existing is
1606    /// not the property that was missing.
1607    #[tokio::test(start_paused = true)]
1608    async fn tools_execute_carries_the_execution_session() {
1609        let (channel, frames) = WsChannel::test_capture();
1610        let channel = Arc::new(channel);
1611        let executor = WsToolExecutor::legacy(channel.clone());
1612
1613        let _ = tokio::time::timeout(
1614            Duration::from_millis(50),
1615            executor.execute_with_action_in_session(
1616                "search",
1617                &json!({}),
1618                "a0",
1619                Some(10_000),
1620                Some("sess-42"),
1621                1,
1622            ),
1623        )
1624        .await;
1625        settle().await;
1626
1627        let frames = written(&frames);
1628        let execute = frame_with_method(&frames, "tools.execute").expect("tools.execute was sent");
1629        assert_eq!(
1630            execute["params"]["session_id"],
1631            json!("sess-42"),
1632            "the daemon must stamp the session it already knows, not leave \
1633             attribution to the host"
1634        );
1635    }
1636
1637    /// Parslee-ai/car#928 — the retry counter on the wire must be the engine's
1638    /// real one.
1639    ///
1640    /// It was hardcoded to `1` here, so the field a host would build its
1641    /// retry-disambiguating join on never varied. That is worse than the field
1642    /// being absent: the join looks correct and degenerates silently, and only
1643    /// under the concurrent-retry conditions it exists to handle.
1644    #[tokio::test(start_paused = true)]
1645    async fn tools_execute_carries_the_real_attempt_number() {
1646        for attempt in [1u32, 2, 7] {
1647            let (channel, frames) = WsChannel::test_capture();
1648            let channel = Arc::new(channel);
1649            let executor = WsToolExecutor::legacy(channel.clone());
1650
1651            let _ = tokio::time::timeout(
1652                Duration::from_millis(50),
1653                executor.execute_with_action_in_session(
1654                    "search",
1655                    &json!({}),
1656                    "a0",
1657                    Some(10_000),
1658                    Some("sess-1"),
1659                    attempt,
1660                ),
1661            )
1662            .await;
1663            settle().await;
1664
1665            let frames = written(&frames);
1666            let execute =
1667                frame_with_method(&frames, "tools.execute").expect("tools.execute was sent");
1668            assert_eq!(
1669                execute["params"]["attempt"],
1670                json!(attempt),
1671                "the payload must carry the engine's attempt, not a constant"
1672            );
1673        }
1674    }
1675
1676    /// The other half: a caller with no session must omit the key rather than
1677    /// send a null. `execute()` and in-process executors legitimately have
1678    /// none, and a null would make every such payload differ from the pre-#904
1679    /// shape for no gain.
1680    #[tokio::test(start_paused = true)]
1681    async fn a_sessionless_call_omits_the_session_key() {
1682        let (channel, frames) = WsChannel::test_capture();
1683        let channel = Arc::new(channel);
1684        let executor = WsToolExecutor::legacy(channel.clone());
1685
1686        let _ = tokio::time::timeout(
1687            Duration::from_millis(50),
1688            executor.execute_with_action_in_session(
1689                "search",
1690                &json!({}),
1691                "a0",
1692                Some(10_000),
1693                None,
1694                1,
1695            ),
1696        )
1697        .await;
1698        settle().await;
1699
1700        let frames = written(&frames);
1701        let execute = frame_with_method(&frames, "tools.execute").expect("tools.execute was sent");
1702        assert!(
1703            execute["params"].get("session_id").is_none(),
1704            "a sessionless call must omit the key entirely, got: {}",
1705            execute["params"]
1706        );
1707    }
1708
1709    /// REGRESSION (Parslee-ai/car#264): the executor's per-action deadline
1710    /// fires `TOOL_TIMEOUT_GRACE_MS` BEFORE the daemon's callback wait, so it
1711    /// drops this whole future while it is parked on `rx`. Before the drop
1712    /// guard, nothing ran on that path: no `tools.cancel` frame was ever
1713    /// written and the `pending` entry leaked until connection teardown,
1714    /// orphaning the host's in-flight child.
1715    #[tokio::test(start_paused = true)]
1716    async fn engine_deadline_drop_cancels() {
1717        let (channel, frames) = WsChannel::test_capture();
1718        let channel = Arc::new(channel);
1719        let executor = WsToolExecutor::legacy(channel.clone());
1720
1721        // Outer deadline (standing in for the engine's per-action timeout)
1722        // at 50ms; the callback wait would be 10s + 5s grace.
1723        let outcome = tokio::time::timeout(
1724            Duration::from_millis(50),
1725            executor.execute_with_action_in_session(
1726                "drive_cli",
1727                &json!({}),
1728                "a0",
1729                Some(10_000),
1730                None,
1731                1,
1732            ),
1733        )
1734        .await;
1735        assert!(
1736            outcome.is_err(),
1737            "the outer deadline must fire first and drop the dispatch future"
1738        );
1739
1740        settle().await;
1741
1742        let frames = written(&frames);
1743        let execute = frame_with_method(&frames, "tools.execute").expect("tools.execute was sent");
1744        let cancel = frame_with_method(&frames, "tools.cancel").expect(
1745            "dropping the dispatch future must still emit tools.cancel so the host child isn't orphaned",
1746        );
1747        assert_eq!(
1748            cancel["params"]["request_id"], execute["params"]["request_id"],
1749            "cancel must correlate to the originating request_id"
1750        );
1751        assert!(
1752            channel.pending.lock().await.is_empty(),
1753            "the pending entry must not leak"
1754        );
1755    }
1756
1757    /// The happy path must stay silent: a delivered response disarms the
1758    /// guard, so no host abort is emitted for a call that completed.
1759    #[tokio::test]
1760    async fn success_emits_no_cancel() {
1761        let (channel, frames) = WsChannel::test_capture();
1762        let channel = Arc::new(channel);
1763        let executor = WsToolExecutor::legacy(channel.clone());
1764
1765        let responder = channel.clone();
1766        let host = tokio::spawn(async move {
1767            loop {
1768                let claimed = {
1769                    let mut pending = responder.pending.lock().await;
1770                    let key = pending.keys().next().cloned();
1771                    key.and_then(|k| pending.remove(&k).map(|tx| (k, tx)))
1772                };
1773                if let Some((request_id, tx)) = claimed {
1774                    let _ = tx.send(ToolExecuteResponse {
1775                        action_id: request_id,
1776                        output: Some(json!({"ok": true})),
1777                        error: None,
1778                        terminal: false,
1779                    });
1780                    return;
1781                }
1782                tokio::task::yield_now().await;
1783            }
1784        });
1785
1786        let out = executor
1787            .execute_with_action_in_session("drive_cli", &json!({}), "a0", Some(10_000), None, 1)
1788            .await
1789            .expect("the host responded, so the call succeeds");
1790        assert_eq!(out, json!({"ok": true}));
1791        host.await.expect("responder task finishes");
1792
1793        settle().await;
1794
1795        let frames = written(&frames);
1796        assert!(
1797            frame_with_method(&frames, "tools.execute").is_some(),
1798            "tools.execute was sent"
1799        );
1800        assert!(
1801            frame_with_method(&frames, "tools.cancel").is_none(),
1802            "a completed call must not emit tools.cancel"
1803        );
1804        assert!(channel.pending.lock().await.is_empty());
1805    }
1806}
1807
1808/// The bare commodity-tool names owned by the engine's bound substrate.
1809/// When a session binds a non-Local substrate (`session.bindSubstrate`),
1810/// these names must reach the engine's substrate-after-fall_through path
1811/// (`executor.rs`), NOT the WS tool callback. Everything else — connector
1812/// `mcp_{slug}_*` routes and host-executed client tools — is untouched.
1813///
1814/// `calculate` is deliberately excluded: it is a pure, environment-free
1815/// tool that the engine already handles in-process via `agent_basics`, so
1816/// it never needs to be re-routed to a substrate.
1817pub(crate) const SUBSTRATE_OWNED_TOOLS: &[&str] = &[
1818    "read_file",
1819    "write_file",
1820    "edit_file",
1821    "list_dir",
1822    "find_files",
1823    "grep_files",
1824];
1825
1826/// Executor wrapper used **only** by substrate-bound sessions
1827/// (`session.bindSubstrate`). It is the low-blast-radius "option (a)" from
1828/// `docs/execution-substrate.md` §3: keep the existing
1829/// `share_with_fallback(ws_executor)` composition intact for connector
1830/// routes and host tool callbacks, but for the bare substrate-owned
1831/// built-in names ([`SUBSTRATE_OWNED_TOOLS`]) return an `"unknown tool:
1832/// <name>"` error so the engine's dispatch (`executor.rs`) takes the
1833/// `fall_through` branch and routes those names to the runtime's bound
1834/// substrate (the connector/VM [`car_engine::McpSubstrate`]) instead of the WS client.
1835///
1836/// This preserves every existing contract — GUI/connector/coder sessions
1837/// never construct one of these — and is what makes a connector-driven
1838/// session *coherent*: its file/exec built-ins land on the same machine as
1839/// its `mcp_{slug}_*` tools, not split between the WS client and a remote
1840/// MCP server.
1841pub struct SubstrateShadowExecutor {
1842    inner: Arc<dyn ToolExecutor>,
1843}
1844
1845impl SubstrateShadowExecutor {
1846    pub fn new(inner: Arc<dyn ToolExecutor>) -> Self {
1847        Self { inner }
1848    }
1849}
1850
1851#[async_trait::async_trait]
1852impl ToolExecutor for SubstrateShadowExecutor {
1853    async fn execute(&self, tool: &str, params: &Value) -> Result<Value, String> {
1854        self.execute_with_action(tool, params, "", None).await
1855    }
1856
1857    async fn execute_with_action(
1858        &self,
1859        tool: &str,
1860        params: &Value,
1861        action_id: &str,
1862        timeout_ms: Option<u64>,
1863    ) -> Result<Value, String> {
1864        self.execute_with_action_in_session(tool, params, action_id, timeout_ms, None, 1)
1865            .await
1866    }
1867
1868    async fn execute_with_action_in_session(
1869        &self,
1870        tool: &str,
1871        params: &Value,
1872        action_id: &str,
1873        timeout_ms: Option<u64>,
1874        session_id: Option<&str>,
1875        attempt: u32,
1876    ) -> Result<Value, String> {
1877        if SUBSTRATE_OWNED_TOOLS.contains(&tool) {
1878            // Signal the engine to fall through to the bound substrate.
1879            // The exact `"unknown tool"` prefix is the contract the engine
1880            // dispatch matches on (`executor.rs`).
1881            return Err(format!("unknown tool: {tool}"));
1882        }
1883        self.inner
1884            .execute_with_action_in_session(
1885                tool, params, action_id, timeout_ms, session_id, attempt,
1886            )
1887            .await
1888    }
1889
1890    async fn execute_with_action_state_in_session(
1891        &self,
1892        tool: &str,
1893        params: &Value,
1894        action_id: &str,
1895        timeout_ms: Option<u64>,
1896        session_id: Option<&str>,
1897        attempt: u32,
1898        expected_effects: &HashMap<String, Value>,
1899        return_schema: Option<&Value>,
1900    ) -> Result<car_engine::ToolExecution, String> {
1901        if SUBSTRATE_OWNED_TOOLS.contains(&tool) {
1902            return Err(format!("unknown tool: {tool}"));
1903        }
1904        self.inner
1905            .execute_with_action_state_in_session(
1906                tool,
1907                params,
1908                action_id,
1909                timeout_ms,
1910                session_id,
1911                attempt,
1912                expected_effects,
1913                return_schema,
1914            )
1915            .await
1916    }
1917
1918    async fn execute_classified(
1919        &self,
1920        tool: &str,
1921        params: &Value,
1922        action_id: &str,
1923        timeout_ms: Option<u64>,
1924        session_id: Option<&str>,
1925        attempt: u32,
1926        expected_effects: &HashMap<String, Value>,
1927        return_schema: Option<&Value>,
1928    ) -> Result<car_engine::ToolExecution, ToolFailure> {
1929        if SUBSTRATE_OWNED_TOOLS.contains(&tool) {
1930            return Err(ToolFailure::ordinary(format!("unknown tool: {tool}")));
1931        }
1932        self.inner
1933            .execute_classified(
1934                tool,
1935                params,
1936                action_id,
1937                timeout_ms,
1938                session_id,
1939                attempt,
1940                expected_effects,
1941                return_schema,
1942            )
1943            .await
1944    }
1945}
1946
1947/// Voice event sink that forwards events to a specific WebSocket client
1948/// as `voice.event` JSON-RPC notifications.
1949///
1950/// Each `voice.transcribe_stream.start` call constructs one of these
1951/// bound to the originating client's [`WsChannel`], so a client only
1952/// receives events for sessions it started.
1953pub struct WsVoiceEventSink {
1954    pub channel: Arc<WsChannel>,
1955}
1956
1957impl car_voice::VoiceEventSink for WsVoiceEventSink {
1958    fn send(&self, session_id: &str, event_json: String) {
1959        use futures::SinkExt;
1960        let channel = self.channel.clone();
1961        let session_id = session_id.to_string();
1962        tokio::spawn(async move {
1963            let payload: Value = serde_json::from_str(&event_json)
1964                .unwrap_or_else(|_| Value::String(event_json.clone()));
1965            let notification = serde_json::json!({
1966                "jsonrpc": "2.0",
1967                "method": "voice.event",
1968                "params": {
1969                    "session_id": session_id,
1970                    "event": payload,
1971                },
1972            });
1973            let Ok(text) = serde_json::to_string(&notification) else {
1974                return;
1975            };
1976            let _ = channel
1977                .write
1978                .lock()
1979                .await
1980                .send(Message::Text(text.into()))
1981                .await;
1982        });
1983    }
1984
1985    fn send_binary(&self, frame: Vec<u8>) {
1986        use futures::SinkExt;
1987        let channel = self.channel.clone();
1988        tokio::spawn(async move {
1989            let _ = channel
1990                .write
1991                .lock()
1992                .await
1993                .send(Message::Binary(frame.into()))
1994                .await;
1995        });
1996    }
1997}
1998
1999/// Per-meeting fanout sink that ingests transcript text into a
2000/// session-scoped memgine using the `Arc<tokio::sync::Mutex<...>>`
2001/// wrapper, then forwards every event upstream untouched.
2002///
2003/// Lives here (not in `car-ffi-common`) because the engine handle uses
2004/// `tokio::sync::Mutex` per the "one-wrapper rule" — the FFI-common
2005/// `MeetingMemgineFanout` still uses `std::sync::Mutex` for the NAPI/
2006/// PyO3 bindings, which keep their sync wrappers. Each binding owns the
2007/// fanout that matches its lock primitive; the parsing/formatting logic
2008/// itself is shared via [`car_meeting::extract_transcript_for_ingest`].
2009///
2010/// `send` is called from the voice drain task and must be non-blocking,
2011/// so the lock acquisition is shipped to a `tokio::spawn`. Transcript
2012/// events are independent so reordering across spawned tasks is fine.
2013pub struct WsMemgineIngestSink {
2014    pub meeting_id: String,
2015    pub engine: Arc<Mutex<car_memgine::MemgineEngine>>,
2016    pub upstream: Arc<dyn car_voice::VoiceEventSink>,
2017}
2018
2019impl car_voice::VoiceEventSink for WsMemgineIngestSink {
2020    fn send(&self, voice_session_id: &str, event_json: String) {
2021        if let Ok(value) = serde_json::from_str::<Value>(&event_json) {
2022            if let Some((speaker, text)) = car_meeting::extract_transcript_for_ingest(
2023                &value,
2024                &self.meeting_id,
2025                voice_session_id,
2026            ) {
2027                let engine = self.engine.clone();
2028                tokio::spawn(async move {
2029                    let mut guard = engine.lock().await;
2030                    guard.ingest_conversation(&speaker, &text, chrono::Utc::now());
2031                });
2032            }
2033        }
2034        self.upstream.send(voice_session_id, event_json);
2035    }
2036}
2037
2038/// The last assistant turn a chat session produced, remembered so the *next*
2039/// user turn can be classified against it (the conversation-outcome signal).
2040/// The `trace_id`/`model_id` are carried straight from the `InferenceResult`
2041/// that produced the turn — never reconstructed from message order — so credit
2042/// always lands on the exact model/trace that generated the judged turn.
2043#[derive(Debug, Clone)]
2044pub struct LastChatTurn {
2045    /// The user request this assistant turn answered (for restatement similarity).
2046    pub user_text: String,
2047    pub assistant_text: String,
2048    pub trace_id: String,
2049    pub model_id: String,
2050}
2051
2052/// Per-client session.
2053pub struct ClientSession {
2054    pub client_id: String,
2055    pub runtime: Arc<Runtime>,
2056    pub channel: Arc<WsChannel>,
2057    pub host: Arc<crate::host::HostState>,
2058    /// Memgine handle. Wrapped in `tokio::sync::Mutex` so dispatcher
2059    /// handlers can hold the lock across `.await` points without
2060    /// risking poisoning. Migrated from `std::sync::Mutex` in the
2061    /// car-server-core extraction (U1) per the "one-wrapper rule".
2062    pub memgine: Arc<Mutex<car_memgine::MemgineEngine>>,
2063    /// Lazy browser session — first `browser.run` call launches Chromium,
2064    /// subsequent calls reuse it so element IDs resolve across invocations
2065    /// within the same WebSocket connection.
2066    pub browser: car_ffi_common::browser::BrowserSessionSlot,
2067    /// Per-connection auth state. Starts `false`; flips to `true`
2068    /// after a successful `session.auth` handshake. Always considered
2069    /// authenticated when `ServerState::auth_token` is unset (auth
2070    /// disabled). Closes Parslee-ai/car-releases#32.
2071    pub authenticated: Arc<std::sync::atomic::AtomicBool>,
2072    /// Negotiated daemon JSON-RPC protocol for this WebSocket connection.
2073    /// Starts at `0` (unnegotiated) and is set to
2074    /// `car_proto::PROTOCOL_VERSION` only after an exact-version
2075    /// `server.handshake`. The state is connection-scoped by construction, so
2076    /// a reconnect always has to negotiate again before using handshake-gated
2077    /// host/auth methods.
2078    pub negotiated_protocol_version: std::sync::atomic::AtomicU32,
2079    /// Capability names negotiated by this connection's authenticated
2080    /// `server.handshake`. Connection-scoped for the same reason as the wire
2081    /// version: a reconnect cannot inherit another socket's proof.
2082    pub negotiated_capabilities: Arc<std::sync::RwLock<std::collections::BTreeSet<String>>>,
2083    /// Bounded lifecycle state for inferences started on this exact socket.
2084    /// Keeping it per-session prevents an authenticated peer from probing IDs
2085    /// owned by another connection.
2086    pub inference_control: Arc<crate::inference_control::InferenceRegistry>,
2087    /// Host-management role (Parslee-ai/car#254). Starts `false`; flips
2088    /// to `true` only when the connection presents the per-launch host
2089    /// token via `session.auth { host_token }` (validated against
2090    /// `ServerState::host_token`). `authorize_run_access` requires this
2091    /// for cross-agent run-trace reads — being merely `host.subscribe`d
2092    /// is no longer sufficient, which is what closes the self-elevation
2093    /// hole. Cleared implicitly when the connection drops.
2094    pub is_host: std::sync::atomic::AtomicBool,
2095    /// Bound agent identity (#169). `Some(id)` once a lifecycle-agent
2096    /// child has called `session.auth { token, agent_id }` and the
2097    /// supervisor confirmed `agent_id` is supervised + token matches.
2098    /// Used by `agents.list` to surface which managed agents have
2099    /// actually attached vs. just being marked `Running` at the
2100    /// process level. Cleared at disconnect by `remove_session`.
2101    pub agent_id: Arc<tokio::sync::Mutex<Option<String>>>,
2102    /// Optional daemon-method allowlist bound by successful supervised-agent
2103    /// authentication. `None` is the backward-compatible unrestricted state;
2104    /// `Some(empty)` denies every application method. The dispatcher reads it
2105    /// once before any method-specific handler or notification interceptor.
2106    pub agent_method_allowlist: Arc<std::sync::RwLock<Option<std::collections::BTreeSet<String>>>>,
2107    /// Canonical schema digests for the narrow reverse-callback tools this
2108    /// client registered. Server-owned registry entries never land here.
2109    pub callback_tool_schema_digests:
2110        Arc<tokio::sync::RwLock<std::collections::HashMap<String, String>>>,
2111    /// Bound memory namespace (`session.auth { memory_namespace }`, #79/#80).
2112    ///
2113    /// The namespace identity has to survive the handshake, not just the
2114    /// lookup that binds `bound_memgine`. Without it the daemon knows *which
2115    /// graph* a session is using but not *what to call it*, so nothing can
2116    /// write that graph back to
2117    /// `~/.car/memory/memory-namespaces/<encoded-ns>.json` — which is
2118    /// why namespace memory was durable only until the next restart
2119    /// (car-releases#82). Paired with `bound_memgine` exactly as `agent_id` is.
2120    pub memory_namespace: tokio::sync::Mutex<Option<String>>,
2121    /// Bound persistent memgine (#170). `Some` after `session.auth`
2122    /// successfully attaches the connection to a daemon-owned
2123    /// per-agent memgine (paired with `agent_id`). Memory handlers
2124    /// route through [`ClientSession::effective_memgine`] which
2125    /// returns this when set, falling back to the ephemeral
2126    /// `memgine` field for browser/host/CLI connections.
2127    pub bound_memgine: tokio::sync::Mutex<Option<Arc<Mutex<car_memgine::MemgineEngine>>>>,
2128    /// The run currently bracketed on this connection (agent run
2129    /// tracing, U1). Set by `runs.start` **before** that handler
2130    /// responds, so the per-turn recorder (U2) always reads the
2131    /// `run_id` the bracket established — no race across the
2132    /// concurrently-spawned dispatch tasks (KTD3). Cleared by
2133    /// `runs.complete`. On disconnect with a still-set current run and
2134    /// no recorded terminal, the daemon marks it `Incomplete` (R5).
2135    pub current_run_id: tokio::sync::Mutex<Option<String>>,
2136    /// Serializes the authenticated run bracket on one WebSocket: start,
2137    /// proposal lifecycle, complete, and disconnect terminalization cannot
2138    /// interleave and produce ambiguous journal ordering.
2139    pub run_lifecycle_guard: Arc<tokio::sync::Mutex<()>>,
2140    /// Per-session permission-tier gate (survey §3.4.3/§5.2.5). Backs the
2141    /// `permission.*` JSON-RPC methods: the session holds the granted standing
2142    /// tier + risk classifier. Defaults to `SandboxEdit`, the tier
2143    /// `docs/websocket-protocol.md` publishes as the session default.
2144    ///
2145    /// `Arc`, not a bare lock, because this is also the gate
2146    /// [`crate::permission_gate::PermissionAdmissionGate`] enforces at
2147    /// proposal admission (Parslee-ai/car#890). ONE gate object behind both:
2148    /// a second instance would let the advisory `permission.evaluate` and the
2149    /// enforcement point disagree about the same action, which is precisely
2150    /// the bug class the admission gate was added to close.
2151    ///
2152    /// NOTE (kernel review C1): approval *records* do NOT live here. The
2153    /// gate's embedded ledger is deliberately unused in the daemon — every
2154    /// fingerprint-keyed HITL read/write goes through the SHARED
2155    /// [`ServerState::approval_ledger`] (journal-backed), so an approval
2156    /// recorded on one connection is visible to every other and survives
2157    /// restart. Tier state stays per-session; the approval store is
2158    /// daemon-wide.
2159    pub permission_gate: Arc<tokio::sync::RwLock<car_policy::PermissionGate>>,
2160    /// Connection-local fail-stop latch. A typed terminal tool callback sets
2161    /// it after its proposal has aborted and rolled back; the admission gate
2162    /// sharing this exact atomic rejects every later proposal on this socket.
2163    /// It is deliberately not durable and is discarded on reconnect.
2164    pub halted: Arc<AtomicBool>,
2165    /// Non-overlap guard for `evolution.run` on this session (kernel review
2166    /// S3): the dispatcher spawns requests concurrently even on one
2167    /// connection, and two interleaved cycles would double-dispatch the
2168    /// evolution mechanisms. A second `evolution.run` while one is in flight
2169    /// errs instead of overlapping.
2170    pub evolution_guard: crate::evolution::CycleGuard,
2171    /// Last assistant turn produced on this session's chat path, so the next
2172    /// user turn can be scored as a conversation outcome (Part B). `None` until
2173    /// the first chat reply; only set for declared-chat inference.
2174    pub last_chat_turn: tokio::sync::Mutex<Option<LastChatTurn>>,
2175    /// In-flight chat-infer count. Concurrent infers on one session have no
2176    /// well-defined turn adjacency (the dispatcher spawns a task per frame), so
2177    /// the outcome scorer only records a turn that began while the session was
2178    /// idle — otherwise it would score one infer's trace against an unrelated
2179    /// concurrent user turn (an active mislabel of routing stats). neo #1.
2180    pub chat_inflight: std::sync::atomic::AtomicUsize,
2181    /// Tenant identity bound at `session.auth { tenant_id }` (linus
2182    /// review C-4). Once bound, every tenant-scoped handler uses THIS
2183    /// identity: a per-request `tenant_id` param may restate it but a
2184    /// mismatch is rejected — an authenticated connection cannot hop
2185    /// into another tenant's namespace by editing request params.
2186    /// `None` (unbound) preserves the legacy per-request behavior.
2187    pub tenant: tokio::sync::Mutex<Option<String>>,
2188    /// Whether this connection has a live `tools.stream.event` forwarder
2189    /// task (C2). Set by `tools.stream.subscribe`; a concurrent second
2190    /// subscribe is a no-op instead of spawning a duplicate forwarder
2191    /// (which would double every notification). `Arc` so the forwarder
2192    /// task can RESET it on exit (write timeout on a stalled-but-
2193    /// recovering socket, broadcast closed) — a later re-subscribe then
2194    /// spawns a fresh forwarder rather than silently no-op'ing (Q2).
2195    pub tool_stream_subscribed: std::sync::Arc<std::sync::atomic::AtomicBool>,
2196}
2197
2198impl ClientSession {
2199    /// Returns the memgine handle the memory.* handlers should use:
2200    /// the bound per-agent memgine when this session attached via
2201    /// `session.auth { agent_id }` (#169 + #170), otherwise the
2202    /// ephemeral per-WS memgine. Cheap (one async lock + Arc clone).
2203    pub async fn effective_memgine(&self) -> Arc<Mutex<car_memgine::MemgineEngine>> {
2204        if let Some(eng) = self.bound_memgine.lock().await.as_ref() {
2205            return eng.clone();
2206        }
2207        self.memgine.clone()
2208    }
2209
2210    pub async fn bind_run_journal(&self, run_id: &str) -> Result<(), String> {
2211        self.runtime
2212            .event_log_handle()
2213            .lock()
2214            .await
2215            .bind_run(run_id, &self.client_id)
2216    }
2217
2218    pub async fn require_run_journal_binding(&self, run_id: &str) -> Result<(), String> {
2219        let log = self.runtime.event_log_handle();
2220        let log = log.lock().await;
2221        match log.active_run_binding() {
2222            Some((bound_run, bound_client, _))
2223                if bound_run == run_id && bound_client == self.client_id =>
2224            {
2225                Ok(())
2226            }
2227            Some((bound_run, bound_client, _)) => Err(format!(
2228                "active journal binding mismatch: expected run_id `{run_id}` / client_id `{}`, got `{bound_run}` / `{bound_client}`",
2229                self.client_id
2230            )),
2231            None => Err(format!(
2232                "active run `{run_id}` has no authenticated journal binding"
2233            )),
2234        }
2235    }
2236
2237    pub async fn clear_run_journal_binding(&self, run_id: &str) -> Result<(), String> {
2238        self.runtime
2239            .event_log_handle()
2240            .lock()
2241            .await
2242            .clear_run_binding(run_id, &self.client_id)
2243    }
2244
2245    pub async fn append_run_terminal_event(
2246        &self,
2247        ended: &car_proto::RunEnded,
2248    ) -> Result<(), String> {
2249        self.append_run_terminal_event_once(ended, &self.client_id)
2250            .await
2251            .map_err(|error| error.to_string())
2252    }
2253
2254    /// Append a terminal from an authenticated replacement socket while
2255    /// preserving the immutable client identity in the durable run trace.
2256    pub(crate) async fn append_resumed_run_terminal_event(
2257        &self,
2258        ended: &car_proto::RunEnded,
2259        durable_client_id: &str,
2260    ) -> Result<(), String> {
2261        self.append_run_terminal_event_once(ended, durable_client_id)
2262            .await
2263            .map_err(|error| error.to_string())
2264    }
2265
2266    pub async fn append_run_cancellation_requested_event(
2267        &self,
2268        requested: &car_proto::RunCancellationRequested,
2269    ) -> Result<(), String> {
2270        self.require_run_journal_binding(&requested.run_id).await?;
2271        let value = serde_json::to_value(requested).map_err(|error| error.to_string())?;
2272        let Value::Object(data) = value else {
2273            return Err("cancellation request did not serialize as an object".into());
2274        };
2275        self.runtime
2276            .event_log_handle()
2277            .lock()
2278            .await
2279            .append_critical_async(
2280                car_eventlog::EventKind::RunCancellationRequested,
2281                requested.action_id.as_deref(),
2282                None,
2283                data.into_iter().collect(),
2284                CRITICAL_JOURNAL_ACKNOWLEDGEMENT_TIMEOUT,
2285            )
2286            .await
2287            .map_err(|error| error.to_string())?;
2288        Ok(())
2289    }
2290
2291    pub async fn append_run_cancellation_result_event(
2292        &self,
2293        result: &car_proto::RunCancelResponse,
2294    ) -> Result<(), String> {
2295        self.require_run_journal_binding(&result.run_id).await?;
2296        let value = serde_json::to_value(result).map_err(|error| error.to_string())?;
2297        let Value::Object(data) = value else {
2298            return Err("cancellation result did not serialize as an object".into());
2299        };
2300        self.runtime
2301            .event_log_handle()
2302            .lock()
2303            .await
2304            .append_critical_async(
2305                car_eventlog::EventKind::RunCancellationResult,
2306                result.action_id.as_deref(),
2307                None,
2308                data.into_iter().collect(),
2309                CRITICAL_JOURNAL_ACKNOWLEDGEMENT_TIMEOUT,
2310            )
2311            .await
2312            .map_err(|error| error.to_string())?;
2313        Ok(())
2314    }
2315
2316    async fn append_run_terminal_event_once(
2317        &self,
2318        ended: &car_proto::RunEnded,
2319        durable_client_id: &str,
2320    ) -> Result<(), car_eventlog::CriticalAppendError> {
2321        let rejected = |reason| car_eventlog::CriticalAppendError::Rejected { reason };
2322        self.require_run_journal_binding(&ended.run_id)
2323            .await
2324            .map_err(rejected)?;
2325        if ended.client_id.as_deref() != Some(durable_client_id) {
2326            return Err(rejected(
2327                "RunEnded client_id does not match the durable run owner".into(),
2328            ));
2329        }
2330        let completion_digest = ended
2331            .completion_digest
2332            .as_deref()
2333            .ok_or_else(|| rejected("RunEnded is missing completion_digest".into()))?;
2334        let termination_kind = match &ended.termination {
2335            car_proto::RunTermination::Outcome { .. } => "outcome",
2336            car_proto::RunTermination::Incomplete => "incomplete",
2337            car_proto::RunTermination::Cancelled { .. } => "cancelled",
2338        };
2339        let log = self.runtime.event_log_handle();
2340        let mut log = log.lock().await;
2341        log.append_critical_async(
2342            car_eventlog::EventKind::RunCompleted,
2343            None,
2344            None,
2345            [
2346                (
2347                    "termination_kind".to_string(),
2348                    Value::from(termination_kind),
2349                ),
2350                (
2351                    "completion_digest".to_string(),
2352                    Value::from(completion_digest),
2353                ),
2354                (
2355                    "termination".to_string(),
2356                    serde_json::to_value(&ended.termination)
2357                        .map_err(|error| rejected(error.to_string()))?,
2358                ),
2359            ]
2360            .into(),
2361            CRITICAL_JOURNAL_ACKNOWLEDGEMENT_TIMEOUT,
2362        )
2363        .await?;
2364        Ok(())
2365    }
2366
2367    pub async fn append_run_started_event(
2368        &self,
2369        started: &car_proto::RunStarted,
2370    ) -> Result<(), String> {
2371        self.require_run_journal_binding(&started.run_id).await?;
2372        if started.client_id.as_deref() != Some(self.client_id.as_str()) {
2373            return Err("RunStarted client_id does not match the live ClientSession".into());
2374        }
2375        let mut data = HashMap::from([
2376            (
2377                "agent_id".to_string(),
2378                Value::from(started.agent_id.clone()),
2379            ),
2380            ("intent".to_string(), Value::from(started.intent.clone())),
2381            (
2382                "started_at".to_string(),
2383                serde_json::to_value(started.started_at).unwrap_or(Value::Null),
2384            ),
2385        ]);
2386        if let Some(description) = &started.outcome_description {
2387            data.insert(
2388                "outcome_description".to_string(),
2389                Value::from(description.clone()),
2390            );
2391        }
2392        self.runtime
2393            .event_log_handle()
2394            .lock()
2395            .await
2396            .append_critical_async(
2397                car_eventlog::EventKind::RunStarted,
2398                None,
2399                None,
2400                data,
2401                CRITICAL_JOURNAL_ACKNOWLEDGEMENT_TIMEOUT,
2402            )
2403            .await
2404            .map_err(|error| error.to_string())?;
2405        Ok(())
2406    }
2407}
2408
2409/// Builder for constructing a [`ServerState`] with embedder-supplied
2410/// dependencies. Embedders (e.g. `tokhn-daemon`) use this to inject
2411/// their own memgine handle and other shared infrastructure; the
2412/// Approval-gate policy for high-risk WS methods.
2413///
2414/// Every method in `methods` must be acknowledged via
2415/// `host.resolve_approval` before the dispatcher will route the
2416/// request to its handler. The dispatcher waits up to `timeout` for
2417/// a resolution; on timeout (or any non-`approve` resolution) the
2418/// request fails with JSON-RPC error `-32003`.
2419///
2420/// Default: gate enabled, the host-automation surface
2421/// (`automation.run_applescript`, `automation.run_powershell`,
2422/// `automation.shortcuts.run`, `messages.send`, `mail.send`, `vision.ocr`),
2423/// 60-second timeout.
2424/// `car-server --no-approvals` (or embedders calling
2425/// [`ServerStateConfig::with_approval_gate`] with `enabled=false`)
2426/// turns it off — only appropriate when no untrusted caller can
2427/// reach the WS port.
2428#[derive(Debug, Clone)]
2429pub struct ApprovalGate {
2430    /// Master switch. When `false`, every method dispatches without
2431    /// raising an approval — the pre-2026-05 behaviour.
2432    pub enabled: bool,
2433    /// Methods that require approval. Match is by exact method-name
2434    /// string against the JSON-RPC `method` field.
2435    pub methods: std::collections::HashSet<String>,
2436    /// How long to wait for the user to resolve the approval before
2437    /// timing out and surfacing an error to the caller.
2438    pub timeout: std::time::Duration,
2439}
2440
2441impl Default for ApprovalGate {
2442    fn default() -> Self {
2443        let methods = [
2444            "automation.run_applescript",
2445            "automation.run_powershell",
2446            "automation.shortcuts.run",
2447            "messages.send",
2448            "mail.send",
2449            "vision.ocr",
2450        ]
2451        .iter()
2452        .map(|s| s.to_string())
2453        .collect();
2454        Self {
2455            enabled: true,
2456            methods,
2457            timeout: std::time::Duration::from_secs(60),
2458        }
2459    }
2460}
2461
2462impl ApprovalGate {
2463    /// Disable the gate entirely. Equivalent to passing
2464    /// `car-server --no-approvals`. Only appropriate when no
2465    /// untrusted caller can reach the WS port.
2466    pub fn disabled() -> Self {
2467        Self {
2468            enabled: false,
2469            methods: std::collections::HashSet::new(),
2470            timeout: std::time::Duration::from_secs(60),
2471        }
2472    }
2473
2474    /// `true` if this method must be acknowledged before dispatch.
2475    pub fn requires_approval(&self, method: &str) -> bool {
2476        self.enabled && self.methods.contains(method)
2477    }
2478}
2479
2480/// standalone `car-server` binary uses [`ServerState::standalone`]
2481/// which calls `with_config` under the hood.
2482pub struct ServerStateConfig {
2483    pub journal_dir: PathBuf,
2484    /// Peer-message audit journal. `None` derives a path once from
2485    /// [`Self::journal_dir`] and the construction-time CAR state root. Tests
2486    /// and embedders can override it without mutating process-global `CAR_HOME`.
2487    pub peer_audit_journal: Option<PathBuf>,
2488    /// Optional pre-constructed memgine engine. When `None`, each
2489    /// `create_session` call builds a fresh engine; embedders that want
2490    /// to share a single engine across sessions can supply a clone of
2491    /// their `Arc<Mutex<MemgineEngine>>` here.
2492    pub shared_memgine: Option<Arc<Mutex<car_memgine::MemgineEngine>>>,
2493    /// Optional pre-constructed inference engine.
2494    pub inference: Option<Arc<car_inference::InferenceEngine>>,
2495    /// Optional embedder-supplied A2A runtime. Used by the in-core
2496    /// `A2aDispatcher` to execute peer-driven proposals. When `None`,
2497    /// the dispatcher uses a fresh `Runtime` with `register_agent_basics`
2498    /// — peer agents see CAR's built-in tools and nothing else,
2499    /// matching the behaviour of the standalone `start_a2a_listener`.
2500    pub a2a_runtime: Option<Arc<car_engine::Runtime>>,
2501    /// Optional embedder-supplied A2A task store. When `None`,
2502    /// defaults to `InMemoryTaskStore`. tokhn-style embedders that
2503    /// want a polling-friendly persistent store plug it in here.
2504    pub a2a_store: Option<Arc<dyn car_a2a::TaskStore>>,
2505    /// Optional embedder-supplied agent card factory. When `None`,
2506    /// the dispatcher serves a card built from the A2A runtime's
2507    /// tool schemas at construction time, advertising its public URL
2508    /// as `ws://127.0.0.1:9100/` (the WS surface the dispatcher itself
2509    /// is reachable on).
2510    pub a2a_card_source: Option<Arc<car_a2a::AgentCardSource>>,
2511    /// Approval-gate policy. When `None`, the dispatcher uses
2512    /// [`ApprovalGate::default`] (gate ON, the macOS-automation
2513    /// surface gated, 60s timeout). Pass
2514    /// [`ApprovalGate::disabled`] to opt out — only appropriate
2515    /// when no untrusted caller can reach the WS port.
2516    pub approval_gate: Option<ApprovalGate>,
2517    /// Journal path for the daemon-wide shared HITL approval ledger
2518    /// ([`ServerState::approval_ledger`]). `None` = the default
2519    /// `~/.car/approvals.jsonl`. Tests and embedders that must not touch the
2520    /// user's real home point this at their own path.
2521    pub approval_journal: Option<PathBuf>,
2522    /// Directory for the daemon-wide trajectory store
2523    /// ([`ServerState::trajectory_store`]). `None` = the default
2524    /// `~/.car/trajectories/`. Tests and embedders that must not touch the
2525    /// user's real home point this at their own path, same as
2526    /// `approval_journal`.
2527    pub trajectory_dir: Option<PathBuf>,
2528    /// Deterministic durability fault seam used by direct lifecycle tests.
2529    pub run_store_failures: Option<crate::run_store::RunStoreFailureInjector>,
2530    /// Deterministic blocking-read seam used by run-liveness tests.
2531    #[doc(hidden)]
2532    pub run_store_summary_read_gate: Option<crate::run_store::RunStoreSummaryReadGate>,
2533    /// Test-only scheduling seam around durable run-summary publication.
2534    #[doc(hidden)]
2535    pub run_store_summary_write_gate: Option<crate::run_store::RunStoreSummaryWriteGate>,
2536    /// Deterministic blocking append seam used by run-liveness tests.
2537    #[doc(hidden)]
2538    pub run_store_append_gate: Option<crate::run_store::RunStoreAppendGate>,
2539    #[doc(hidden)]
2540    pub run_completion_fence_gate: Option<RunCompletionFenceGate>,
2541    /// Deterministic blocking durable-lookup seam used by run-liveness tests.
2542    #[doc(hidden)]
2543    pub run_store_lookup_gate: Option<crate::run_store::RunStoreLookupGate>,
2544    /// Deterministic first-use directory-entry durability seam for RunStore.
2545    pub run_store_private_path_failures: Option<car_secrets::PrivatePathDurabilityFailureInjector>,
2546    /// Deterministic critical-journal fault seam used by direct lifecycle tests.
2547    pub journal_failures: Option<car_eventlog::JournalFailureInjector>,
2548    /// Private self-heal detection and bounded-attempt ledger. The standalone daemon sets this to
2549    /// `<CAR_HOME>/selfheal/detections.jsonl`; embedders/tests can isolate it.
2550    pub selfheal_ledger: Option<PathBuf>,
2551    /// Detection cadence. Eligible default-on auto-fix runs from the same
2552    /// non-overlapping tick and is independently controlled by config.toml.
2553    pub selfheal_interval_secs: u64,
2554    /// Deterministic evidence seam for integration tests and embedders.
2555    #[doc(hidden)]
2556    pub selfheal_evidence: Option<crate::selfheal::SelfhealEvidence>,
2557    /// Bounded source-checkout probe. Embedders fail closed unless they supply
2558    /// candidates; the standalone daemon derives this from process-local paths.
2559    pub selfheal_source_probe: crate::selfheal::SelfhealSourceProbe,
2560    /// Deterministic seam for the checkout-built replay-verb eligibility probe.
2561    #[doc(hidden)]
2562    pub selfheal_replay_verb_probe: crate::selfheal::SelfhealReplayVerbProbe,
2563    /// Test-only override for the startup reconciliation acknowledgement bound.
2564    #[doc(hidden)]
2565    pub startup_reconciliation_acknowledgement_timeout: std::time::Duration,
2566    /// Test-only override for the bounded in-process resume lease.
2567    #[doc(hidden)]
2568    pub run_resume_lease: std::time::Duration,
2569}
2570
2571impl ServerStateConfig {
2572    /// Minimal config suitable for the standalone car-server binary:
2573    /// only the journal dir is required; everything else is lazily
2574    /// constructed at first use.
2575    pub fn new(journal_dir: PathBuf) -> Self {
2576        Self {
2577            journal_dir,
2578            peer_audit_journal: None,
2579            shared_memgine: None,
2580            inference: None,
2581            a2a_runtime: None,
2582            a2a_store: None,
2583            a2a_card_source: None,
2584            approval_gate: None,
2585            approval_journal: None,
2586            trajectory_dir: None,
2587            run_store_failures: None,
2588            run_store_summary_read_gate: None,
2589            run_store_summary_write_gate: None,
2590            run_store_append_gate: None,
2591            run_completion_fence_gate: None,
2592            run_store_lookup_gate: None,
2593            run_store_private_path_failures: None,
2594            journal_failures: None,
2595            selfheal_ledger: None,
2596            selfheal_interval_secs: crate::selfheal::DEFAULT_SELFHEAL_INTERVAL_SECS,
2597            selfheal_evidence: None,
2598            selfheal_source_probe: crate::selfheal::SelfhealSourceProbe::empty(),
2599            selfheal_replay_verb_probe: crate::selfheal::SelfhealReplayVerbProbe::default(),
2600            startup_reconciliation_acknowledgement_timeout:
2601                STARTUP_RECONCILIATION_ACKNOWLEDGEMENT_TIMEOUT,
2602            run_resume_lease: RUN_RESUME_LEASE,
2603        }
2604    }
2605
2606    pub fn with_shared_memgine(mut self, engine: Arc<Mutex<car_memgine::MemgineEngine>>) -> Self {
2607        self.shared_memgine = Some(engine);
2608        self
2609    }
2610
2611    /// Override the peer-message audit journal path.
2612    pub fn with_peer_audit_journal(mut self, path: PathBuf) -> Self {
2613        self.peer_audit_journal = Some(path);
2614        self
2615    }
2616
2617    /// Override the shared approval-ledger journal path (default
2618    /// `~/.car/approvals.jsonl`).
2619    pub fn with_approval_journal(mut self, path: PathBuf) -> Self {
2620        self.approval_journal = Some(path);
2621        self
2622    }
2623
2624    pub fn with_trajectory_dir(mut self, dir: PathBuf) -> Self {
2625        self.trajectory_dir = Some(dir);
2626        self
2627    }
2628
2629    pub fn with_run_store_failures(
2630        mut self,
2631        failures: crate::run_store::RunStoreFailureInjector,
2632    ) -> Self {
2633        self.run_store_failures = Some(failures);
2634        self
2635    }
2636
2637    #[doc(hidden)]
2638    pub fn with_run_store_summary_read_gate(
2639        mut self,
2640        gate: crate::run_store::RunStoreSummaryReadGate,
2641    ) -> Self {
2642        self.run_store_summary_read_gate = Some(gate);
2643        self
2644    }
2645
2646    #[doc(hidden)]
2647    pub fn with_run_store_summary_write_gate(
2648        mut self,
2649        gate: crate::run_store::RunStoreSummaryWriteGate,
2650    ) -> Self {
2651        self.run_store_summary_write_gate = Some(gate);
2652        self
2653    }
2654
2655    #[doc(hidden)]
2656    pub fn with_run_store_append_gate(
2657        mut self,
2658        gate: crate::run_store::RunStoreAppendGate,
2659    ) -> Self {
2660        self.run_store_append_gate = Some(gate);
2661        self
2662    }
2663
2664    #[doc(hidden)]
2665    pub fn with_run_completion_fence_gate(mut self, gate: RunCompletionFenceGate) -> Self {
2666        self.run_completion_fence_gate = Some(gate);
2667        self
2668    }
2669
2670    #[doc(hidden)]
2671    pub fn with_run_resume_lease(mut self, lease: std::time::Duration) -> Self {
2672        self.run_resume_lease = lease;
2673        self
2674    }
2675
2676    #[doc(hidden)]
2677    pub fn with_run_store_lookup_gate(
2678        mut self,
2679        gate: crate::run_store::RunStoreLookupGate,
2680    ) -> Self {
2681        self.run_store_lookup_gate = Some(gate);
2682        self
2683    }
2684
2685    pub fn with_run_store_private_path_failures(
2686        mut self,
2687        failures: car_secrets::PrivatePathDurabilityFailureInjector,
2688    ) -> Self {
2689        self.run_store_private_path_failures = Some(failures);
2690        self
2691    }
2692
2693    pub fn with_journal_failures(mut self, failures: car_eventlog::JournalFailureInjector) -> Self {
2694        self.journal_failures = Some(failures);
2695        self
2696    }
2697
2698    pub fn with_selfheal_ledger(mut self, path: PathBuf) -> Self {
2699        self.selfheal_ledger = Some(path);
2700        self
2701    }
2702
2703    pub fn with_selfheal_interval_secs(mut self, interval_secs: u64) -> Self {
2704        self.selfheal_interval_secs = interval_secs.max(1);
2705        self
2706    }
2707
2708    #[doc(hidden)]
2709    pub fn with_selfheal_evidence(mut self, evidence: crate::selfheal::SelfhealEvidence) -> Self {
2710        self.selfheal_evidence = Some(evidence);
2711        self
2712    }
2713
2714    pub fn with_selfheal_source_probe(
2715        mut self,
2716        probe: crate::selfheal::SelfhealSourceProbe,
2717    ) -> Self {
2718        self.selfheal_source_probe = probe;
2719        self
2720    }
2721
2722    #[doc(hidden)]
2723    pub fn with_selfheal_replay_verb_probe(
2724        mut self,
2725        probe: crate::selfheal::SelfhealReplayVerbProbe,
2726    ) -> Self {
2727        self.selfheal_replay_verb_probe = probe;
2728        self
2729    }
2730
2731    #[doc(hidden)]
2732    pub fn with_startup_reconciliation_acknowledgement_timeout(
2733        mut self,
2734        timeout: std::time::Duration,
2735    ) -> Self {
2736        self.startup_reconciliation_acknowledgement_timeout = timeout;
2737        self
2738    }
2739
2740    pub fn with_inference(mut self, engine: Arc<car_inference::InferenceEngine>) -> Self {
2741        self.inference = Some(engine);
2742        self
2743    }
2744
2745    /// Plug in an embedder-supplied runtime for the A2A dispatcher.
2746    /// Use case: tokhn-daemon wants peers to see its OPA preflight
2747    /// tooling, not just CAR's `register_agent_basics` defaults.
2748    pub fn with_a2a_runtime(mut self, runtime: Arc<car_engine::Runtime>) -> Self {
2749        self.a2a_runtime = Some(runtime);
2750        self
2751    }
2752
2753    /// Plug in an embedder-supplied task store for the A2A
2754    /// dispatcher. Use case: tokhn's polling-friendly persistent
2755    /// store keyed by their session id.
2756    pub fn with_a2a_store(mut self, store: Arc<dyn car_a2a::TaskStore>) -> Self {
2757        self.a2a_store = Some(store);
2758        self
2759    }
2760
2761    /// Plug in an embedder-supplied agent card factory. The factory
2762    /// is invoked on every `agent/getAuthenticatedExtendedCard`
2763    /// dispatch, so embedders can reflect runtime tool changes.
2764    pub fn with_a2a_card_source(mut self, source: Arc<car_a2a::AgentCardSource>) -> Self {
2765        self.a2a_card_source = Some(source);
2766        self
2767    }
2768
2769    /// Override the approval-gate policy. Pass
2770    /// [`ApprovalGate::disabled`] to skip the gate entirely (only
2771    /// appropriate when no untrusted caller can reach the WS port);
2772    /// pass a customised [`ApprovalGate`] to add or remove methods
2773    /// or to change the timeout.
2774    pub fn with_approval_gate(mut self, gate: ApprovalGate) -> Self {
2775        self.approval_gate = Some(gate);
2776        self
2777    }
2778}
2779
2780/// The opted-in shared org-scope delivery subsystem + which org it serves. See
2781/// [`ServerState::org_sync`]. Held behind the same `std::sync::Mutex` as the
2782/// personal subsystem holder (init is serialized by the oplog's advisory lock);
2783/// the subsystem itself lives behind a `tokio::sync::Mutex`.
2784pub struct OrgSyncHolder {
2785    pub org: String,
2786    pub subsystem: Arc<tokio::sync::Mutex<crate::sync::SyncSubsystem>>,
2787}
2788
2789/// Which subsystem a write routes to. Pure decision, extracted so it is unit-
2790/// testable without a full daemon.
2791#[derive(Debug, PartialEq, Eq)]
2792enum SyncRoute {
2793    User,
2794    Org,
2795}
2796
2797/// Route by scope: a `Scope::Shared{org}` whose org matches the opted-in
2798/// `org_holder` goes to the org delivery subsystem; EVERYTHING else — `Personal`,
2799/// a different org, or no opted-in holder — goes to the personal subsystem. So
2800/// with `org_holder = None` (org-scope off) every scope routes `User`, byte-
2801/// identical to before.
2802fn route_for_scope(scope: &car_sync::Scope, org_holder: Option<&str>) -> SyncRoute {
2803    if let car_sync::Scope::Shared { org } = scope {
2804        if org_holder == Some(org.as_str()) {
2805            return SyncRoute::Org;
2806        }
2807    }
2808    SyncRoute::User
2809}
2810
2811/// Global server state shared across all connections.
2812pub struct ServerState {
2813    pub journal_dir: PathBuf,
2814    /// Resolved once when the state is constructed. Peer-message writes must
2815    /// use this path and never consult process-global `CAR_HOME` themselves.
2816    pub peer_audit_journal: PathBuf,
2817    pub sessions: Mutex<HashMap<String, Arc<ClientSession>>>,
2818    /// Serializes only session registry insertion/removal with the final
2819    /// liveness check + owner swap in `runs.resume`. This avoids holding the
2820    /// global session map while waiting on per-run durability and turns the
2821    /// replacement-present / old-owner-absent decision into one atomic fence.
2822    run_resume_liveness: Mutex<()>,
2823    run_resume_lease: std::time::Duration,
2824    run_completion_fence_gate: Option<RunCompletionFenceGate>,
2825    pub inference: std::sync::OnceLock<Arc<car_inference::InferenceEngine>>,
2826    pub host: Arc<crate::host::HostState>,
2827    /// When `Some`, `create_session` clones this handle into every new
2828    /// `ClientSession.memgine` — embedders that want a single shared
2829    /// memgine across all WS sessions set this. Standalone car-server
2830    /// leaves it `None`, which gives each session its own engine
2831    /// (preserving today's behavior).
2832    pub shared_memgine: Option<Arc<Mutex<car_memgine::MemgineEngine>>>,
2833    /// Feedback doctor inputs resolved through [`ServerStateConfig`].
2834    pub(crate) feedback_diagnostics: crate::feedback::FeedbackDiagnostics,
2835    /// Daemon-wide trajectory store, cloned into **every** session's
2836    /// `Runtime` so execution traces persist to one place.
2837    ///
2838    /// Shared rather than per-session on purpose. A trajectory is evidence
2839    /// about how a tool behaves, and that evidence does not belong to the
2840    /// connection that happened to produce it — per-session stores would
2841    /// scatter the history across connections and reset it on every
2842    /// reconnect, leaving the derived success rates built from a handful of
2843    /// samples. This is the same reasoning that makes the approval ledger
2844    /// daemon-wide.
2845    ///
2846    /// Until now nothing called `Runtime::with_trajectory_store`, so
2847    /// `persist_trajectory` returned early on every execution and the store
2848    /// was dead code: `ToolFeedback::from_trajectories` existed with no data
2849    /// to read. Wiring it here is what makes the feedback loop real.
2850    pub trajectory_store: Arc<car_memgine::TrajectoryStore>,
2851    /// Daemon-wide self-healing detector/fixer + CAR_HOME ledger. Shared
2852    /// across sessions so `selfheal.run` and the cadence cannot overlap and
2853    /// every caller reads the same dismissals/detections.
2854    pub selfheal: Arc<crate::selfheal::SelfhealService>,
2855    /// The self-healing *repair* loop: reads a configured tracker, runs a coder
2856    /// session, gates the result on a multi-model panel, opens a pull request.
2857    ///
2858    /// Distinct from [`Self::selfheal`], which detects and never writes. Shared
2859    /// across sessions for the same reason: `heal.run` and the cadence must not
2860    /// overlap, and both must read the same claim ledger.
2861    pub heal: Arc<crate::coder::heal_service::HealService>,
2862    /// Process-wide voice session registry. Each
2863    /// `voice.transcribe_stream.start` call registers its own per-client
2864    /// [`WsVoiceEventSink`] so events route back to the originating WS
2865    /// connection only.
2866    pub voice_sessions: Arc<car_voice::VoiceSessionRegistry>,
2867    /// Process-wide meeting registry. Meeting ids are global; each
2868    /// meeting binds to the originating client's WS for upstream
2869    /// events but persists transcripts to the resolved
2870    /// `.car/meetings/<id>/` regardless of which client started it.
2871    pub meetings: Arc<car_meeting::MeetingRegistry>,
2872    /// Process-wide A2UI surface store. Agent-produced surfaces are
2873    /// visible to every host UI subscriber, independent of the
2874    /// WebSocket session that applied the update.
2875    pub a2ui: car_a2ui::A2uiSurfaceStore,
2876    /// In-process UI-improvement agent. Invoked from
2877    /// `handle_a2ui_render_report` with each inbound report; returned
2878    /// `Decision::Patch` envelopes are applied via the standard
2879    /// `apply_a2ui_envelope` path so all subscribers see the patch.
2880    /// `Arc` so the agent's interior `DashMap` state survives across
2881    /// handler calls even when `ServerState` is cheap-cloned.
2882    pub ui_agent: Arc<car_ui_agent::UIImprovementAgent>,
2883    /// Per-surface oscillation detector for the UI-improvement
2884    /// loop. Sits between the agent's `Decision::Patch` and the
2885    /// apply path so A→B→A patch cycles get cooled down without
2886    /// the agent itself having to track history. neo's review:
2887    /// "controllers use workqueue backoff; reconcilers stay
2888    /// stateless."
2889    pub ui_agent_oscillation: Arc<crate::ui_agent_loop::OscillationDetector>,
2890    /// Per-surface iteration budget. Backstop against runaway
2891    /// loops the oscillation detector misses — caps total agent-
2892    /// driven patches per surface at `DEFAULT_MAX_ITERATIONS`.
2893    pub ui_agent_budget: Arc<crate::ui_agent_loop::IterationBudget>,
2894    /// Process-wide concurrency gate for inference RPC handlers. Sized
2895    /// from host RAM at startup, overridable via
2896    /// [`crate::admission::ENV_MAX_CONCURRENT`]. Without this, N
2897    /// concurrent users multiply KV-cache and activation memory and
2898    /// take the host out (#114-adjacent: filed alongside the daemon
2899    /// always-on rework). The semaphore lives on `ServerState` so it
2900    /// is shared across every WebSocket session in the same process.
2901    pub admission: Arc<crate::admission::InferenceAdmission>,
2902    /// Server-side A2A continuation auth keyed by A2UI surface id.
2903    /// Kept out of `A2uiSurface.owner` so host renderers never see
2904    /// bearer/API-key material.
2905    pub a2ui_route_auth: Mutex<HashMap<String, A2aRouteAuth>>,
2906    /// Lifecycle-managed agents — declarative manifest at
2907    /// `~/.car/agents.json` driving spawn/restart/stop. Closes
2908    /// Parslee-ai/car-releases#27. Lazy-initialized so embedders that
2909    /// don't want process supervision don't pay the disk-touch cost
2910    /// at server start.
2911    pub supervisor: std::sync::OnceLock<Arc<car_registry::supervisor::Supervisor>>,
2912    /// Declarative (in-daemon) agent registry — a parallel store to the
2913    /// supervisor for agents the coder→agent loop builds. Lazy-initialized on
2914    /// first use (`~/.car/declagents.json`).
2915    pub declagents: std::sync::OnceLock<Arc<car_registry::declarative::DeclRegistry>>,
2916    /// Routing learning state — per-agent success stats + agent→agent edge
2917    /// weights for capability-similarity routing (`~/.car/routing.json`).
2918    /// Lazy-initialized on first use, sibling to [`Self::declagents`].
2919    pub routing: std::sync::OnceLock<Arc<car_registry::routing::RoutingStore>>,
2920    /// Multi-device sync + execution-lease subsystem (B6). Lazily opened on
2921    /// first `sync.*`/`lease.*` contact, rooted at `<journal_dir>/sync/` — a
2922    /// daemon-held [`car_sync::SyncSession`] over an `FsRelay` (the single-user
2923    /// two-device loopback works out of the box) plus an in-process
2924    /// linearizable [`car_sync::InMemoryLeaseCoordinator`]. **One subsystem per
2925    /// daemon = one device in the sync fleet.** The `std::sync::Mutex<Option>`
2926    /// serializes the fallible first-open (the oplog journal holds an exclusive
2927    /// advisory lock, so a double-open would fail) without holding the lock
2928    /// across the subsystem's async work — the subsystem itself lives behind a
2929    /// `tokio::sync::Mutex` inside the `Arc`. A distributed relay + a
2930    /// cross-daemon linearizable `LeaseCoordinator` backend is the documented
2931    /// B6 follow-up; the traits are its contract.
2932    pub sync: std::sync::Mutex<Option<Arc<tokio::sync::Mutex<crate::sync::SyncSubsystem>>>>,
2933    /// The SHARED org-scope delivery subsystem — a SECOND device oplog whose relay
2934    /// scope is `org:{orgId}`, so `Scope::Shared{org}` ops converge across an org's
2935    /// members. `None` unless org-scope is opted in (the same gated
2936    /// `PARSLEE_SYNC_ORG_SCOPE` branch that resolves the org key builds it);
2937    /// populated by `open_sync_subsystem` alongside the personal subsystem. Only
2938    /// the `Scope::Shared` write sites + `pump` route to it — the personal path
2939    /// (`sync_subsystem()` + all readers) is untouched. Slice 8a.
2940    pub org_sync: std::sync::Mutex<Option<OrgSyncHolder>>,
2941    /// Manifest path this daemon is *observing* but does NOT own.
2942    /// Set by `car-server` when boot-time supervisor construction
2943    /// fails with [`car_registry::supervisor::SupervisorError::AlreadyRunning`]
2944    /// — another car-server process on the host holds the exclusive
2945    /// lock on this manifest. In that state, `supervisor()` returns a
2946    /// clear "observe-only" error so mutation handlers refuse
2947    /// (preventing the duplicate-spawn bug from
2948    /// Parslee-ai/car-releases#44), while read-only handlers
2949    /// (`agents.list`, `agents.health`) fall back to
2950    /// [`car_registry::supervisor::Supervisor::list_from_manifest`] /
2951    /// [`car_registry::supervisor::Supervisor::health_from_manifest`]
2952    /// so operators can still inspect what the primary daemon is
2953    /// supervising.
2954    pub observer_manifest_path: std::sync::OnceLock<PathBuf>,
2955    /// In-core A2A dispatcher — embedders that consume `car-server-core`
2956    /// get A2A reachability "for free" without standing up a separate
2957    /// HTTP listener. Closes Parslee-ai/car-releases#28. Lazy-init so
2958    /// the embedder can override the runtime / task store / agent card
2959    /// via [`ServerStateConfig::with_a2a_runtime`] etc. before the
2960    /// first dispatch.
2961    pub a2a_dispatcher: std::sync::OnceLock<Arc<car_a2a::A2aDispatcher>>,
2962    /// WS clients subscribed to A2UI envelope events. After every
2963    /// successful `a2ui.apply` / `a2ui.ingest`, the resulting
2964    /// `A2uiApplyResult` is broadcast to every subscriber as an
2965    /// `a2ui.event` JSON-RPC notification. Closes
2966    /// Parslee-ai/car-releases#29. Subscribers register via the
2967    /// `a2ui/subscribe` method and are auto-cleaned on WS disconnect.
2968    pub a2ui_subscribers: Mutex<HashMap<String, Arc<WsChannel>>>,
2969    /// Per-launch auth token. When `Some`, the WS dispatcher rejects
2970    /// non-auth methods on unauthenticated sessions until the client
2971    /// calls `session.auth` with the matching value. When `None`,
2972    /// auth is disabled and every connection works as before. Set
2973    /// at startup by `car-server` unless `--no-auth` is passed
2974    /// (default flipped 2026-05); embedders that want to enable
2975    /// auth call [`ServerState::install_auth_token`]. Closes
2976    /// Parslee-ai/car-releases#32.
2977    /// Shared MCP tool executor backing remote connectors. One per
2978    /// process: every WS session's runtime executor is a
2979    /// [`car_engine::McpToolExecutor::share_with_fallback`] view over
2980    /// this, so connector tools route here while non-connector tools
2981    /// fall back to the per-session [`WsToolExecutor`]. Connectors
2982    /// (their live sessions + routes) live on this shared instance, so
2983    /// a connector enabled in one WS session is reachable from all.
2984    pub mcp_executor: Arc<car_engine::McpToolExecutor>,
2985    /// Remote MCP connector manager (`~/.car/connectors.json`).
2986    /// Lazy-initialized over [`mcp_executor`](Self::mcp_executor) so
2987    /// embedders that never touch connectors pay no disk cost. See
2988    /// [`ServerState::connectors`].
2989    pub connectors: std::sync::OnceLock<Arc<car_connectors::ConnectorManager>>,
2990    /// One-shot guard so persisted connectors are loaded + dialed at
2991    /// most once per process (see [`ServerState::ensure_connectors_loaded`]).
2992    connectors_loaded: std::sync::atomic::AtomicBool,
2993    /// Runtime approval-transport channel supervisor (Units 1/2/3). Lazy-
2994    /// initialized at daemon boot by `spawn_channel_pollers`, which records the
2995    /// channels it spawned at startup. The host-gated `messaging.config.set`
2996    /// handler reaches it via `ServerState` to spawn a channel's watcher the
2997    /// instant the user enables it — no restart (U1). Also holds the per-channel
2998    /// liveness `messaging.status` reads (U2) and the send path writes (U3).
2999    /// `None` for embedders that never boot the channel pollers.
3000    pub channel_supervisor: std::sync::OnceLock<Arc<crate::channel_supervisor::ChannelSupervisor>>,
3001    /// Daemon-owned operations which must outlive the WebSocket request that
3002    /// started them. The tasks hold no [`ClientSession`] or [`WsChannel`]; only
3003    /// their oneshot response waiters are connection-scoped.
3004    ///
3005    /// Two kinds of work live here. Credential-backed auth operations (attempt
3006    /// completion and ordinary auth mutations), so a socket close can release
3007    /// its sink immediately without cancelling a coordinator/keychain operation
3008    /// halfway through its durable result. And `coder.start`, which registers a
3009    /// session and provisions a worktree *before* a multi-minute contract
3010    /// derivation: once that side effect exists on disk, the run's lifetime
3011    /// must not be owned by the board that asked for it.
3012    pub durable_tasks: Mutex<tokio::task::JoinSet<String>>,
3013    /// Per-daemon incarnation token persisted into redeeming auth leases.
3014    /// Awaiting-callback reservations intentionally have no daemon owner and
3015    /// survive a restart; a redeeming lease owned by a different incarnation
3016    /// is an orphan and is terminally failed by completion-status reconciliation.
3017    pub auth_completion_owner_id: String,
3018    pub auth_token: std::sync::OnceLock<String>,
3019    /// Per-launch **host** token — a credential distinct from
3020    /// `auth_token`, granting the host-management role (host-class
3021    /// reads like cross-agent run traces). Critically it is **never**
3022    /// served over `GET /auth-token`: a session becomes host-role only
3023    /// by presenting this via `session.auth { host_token }`, and the
3024    /// only way to obtain it is reading the `0600` `host-token` file,
3025    /// which a different local user cannot. This is what stops any
3026    /// authenticated local client from self-elevating to host and
3027    /// reading every agent's run traces (Parslee-ai/car#254). When
3028    /// `None`, host-role can't be granted (no host reads).
3029    pub host_token: std::sync::OnceLock<String>,
3030    /// Mobile Parslee Core runtime URL this daemon can hand to consumer host apps.
3031    /// Set by `car-server` at startup from the daemon bind address or an
3032    /// explicit public URL override. The token remains the per-launch auth
3033    /// token; this field only names the WebSocket endpoint.
3034    pub mobile_runtime_url: std::sync::OnceLock<String>,
3035    /// Explicit public mobile runtime URL to register after an authorized
3036    /// credential action succeeds. Absent for the default loopback URL, which
3037    /// is discoverable locally but must not be advertised to Parslee cloud.
3038    pub mobile_registration_url: std::sync::OnceLock<String>,
3039    /// Parslee cloud identity activated by an explicit credential-backed
3040    /// request after `car auth login` has been completed.
3041    pub parslee_session: std::sync::OnceLock<crate::parslee_auth::ParsleeSession>,
3042    /// `agent_id -> client_id` map of currently-attached lifecycle
3043    /// agents (#169). Populated by the `session.auth` handler when a
3044    /// supervised child presents its `agent_id` + per-agent token;
3045    /// drained on disconnect by `remove_session`. Single-claim:
3046    /// a second connection presenting the same `agent_id` is
3047    /// rejected so the daemon-side per-agent state stays unambiguous.
3048    pub attached_agents: Mutex<HashMap<String, String>>,
3049    /// `agent_id -> persistent memgine` map (#170). Lazy-loaded on
3050    /// first connection per id from `~/.car/memory/agents/<id>.jsonl`,
3051    /// retained across daemon restart, surviving any single
3052    /// disconnect/reconnect of the supervised child. Connections
3053    /// that auth without an `agent_id` (browser, host, ad-hoc CLI)
3054    /// keep the per-WS ephemeral memgine on `ClientSession.memgine`
3055    /// — no behaviour change.
3056    pub agent_memgines: Mutex<HashMap<String, Arc<Mutex<car_memgine::MemgineEngine>>>>,
3057    /// Daemon-owned memgines keyed by a host-declared **memory namespace**
3058    /// (`session.auth { memory_namespace }`), lazily loaded from
3059    /// `~/.car/memory/memory-namespaces/<encoded-ns>.json` — the filename is a
3060    /// percent-encoding of the namespace, injective so two namespaces can never
3061    /// share a snapshot file (#891).
3062    ///
3063    /// A separate axis from `agent_memgines`, deliberately: one agent may work
3064    /// across several namespaces, and two hosts may share a namespace without
3065    /// sharing an identity (Parslee-ai/car-releases#79). A host that binds none
3066    /// keeps today's behaviour — the daemon's shared graph — so the MCP-to-WS
3067    /// shared knowledge base is untouched.
3068    pub namespace_memgines: Mutex<HashMap<String, Arc<Mutex<car_memgine::MemgineEngine>>>>,
3069    /// Live coder sessions keyed by `session_id` (built-in coding agent).
3070    /// Process-wide so a session outlives the WS connection that started
3071    /// it; terminal sessions stay listed for `coder.get`/`coder.list`
3072    /// history until daemon restart (snapshots persist under
3073    /// `~/.car/coder/`).
3074    pub coder_sessions: Mutex<crate::coder::rpc::CoderSessionMap>,
3075    /// Monotonic base for [`Self::coder_disk_gc_at`], taken at construction.
3076    pub coder_disk_gc_base: std::time::Instant,
3077    /// Seconds since [`Self::coder_disk_gc_base`] at the last coder state-dir
3078    /// sweep. Rate-limits the disk GC amortized onto `coder.start` (car#1339)
3079    /// so a burst of starts re-reads the directory once, not once each.
3080    ///
3081    /// Monotonic rather than Unix seconds because this is an INTERVAL: a wall
3082    /// clock that steps backwards would stamp a future value and suppress every
3083    /// later sweep for the daemon's lifetime, which is the bug this closes,
3084    /// reintroduced through the clock.
3085    ///
3086    /// On the state rather than a process-global static so each `ServerState` —
3087    /// every test builds its own — carries its own stamp, and one test's sweep
3088    /// cannot suppress another's.
3089    pub coder_disk_gc_at: AtomicU64,
3090    /// Live `coder.event` subscribers keyed by `(session_id, client_id)`
3091    /// — explicit fanout, same shape as `run_subscribers`. Lock order:
3092    /// the session's event buffer → this map; never the reverse.
3093    pub coder_subscribers: Mutex<HashMap<(String, String), Arc<WsChannel>>>,
3094    /// `coder.watch` board subscribers keyed by `client_id` — one
3095    /// subscription per connection covering EVERY session, so a board learns
3096    /// about runs started by any other client (`car code`, CarHost, milo)
3097    /// without polling. Fanned to as `coder.session_changed`. Lock order:
3098    /// a session's event buffer → `coder_subscribers` → this map; never the
3099    /// reverse.
3100    ///
3101    /// The value carries a **registration generation** alongside the channel.
3102    /// A board re-registers periodically (its `coder.watch` is idempotent and
3103    /// returns a fresh snapshot), and the fanout sheds a wedged watcher *after*
3104    /// releasing this lock — so a shed that removed by `client_id` alone would
3105    /// delete a registration made in that window and silently unwatch a healthy
3106    /// board. The shed compares the generation and removes only the entry it
3107    /// actually timed out on.
3108    pub coder_watchers: Mutex<HashMap<String, (u64, Arc<WsChannel>)>>,
3109    /// The one long-lived task that fans `coder.session_changed` out to
3110    /// [`coder_watchers`](Self::coder_watchers), lazily started on the first
3111    /// notification. Deliberately a single drain rather than a task per event:
3112    /// a running session emits on every tool call, and spawn-per-event against
3113    /// a half-open board socket accumulated blocked tasks without bound, each
3114    /// pinning the socket's write half past teardown (car#209's shape on a new
3115    /// path). Daemon-owned, so it is correctly absent from any connection's
3116    /// `conn_tasks`; it exits when this `ServerState` drops.
3117    pub(crate) coder_watch_notify: std::sync::OnceLock<mpsc::UnboundedSender<String>>,
3118    /// Live `coder.discuss.*` conversations keyed by `discussion_id`. In-memory
3119    /// only and deliberately NOT persisted across daemon restarts: a discussion
3120    /// is a thinking surface, not a run, and resuming one would mean rebuilding
3121    /// an assistant runtime and a model thread from a snapshot that never
3122    /// existed.
3123    pub coder_discussions: Mutex<crate::coder::discuss::DiscussionMap>,
3124    /// Open-discussion slots, one permit per allowed discussion. A permit is
3125    /// taken BEFORE `coder.discuss.start` does any of its expensive async work
3126    /// and lives inside the [`DiscussionEntry`](crate::coder::discuss::DiscussionEntry)
3127    /// it admitted, so it comes back when the discussion is dropped. Counting
3128    /// the map instead was a TOCTOU check — the count was read, the lock
3129    /// released, and two awaits (substrate bind + runtime build) ran before the
3130    /// insert, so N pipelined starts all passed a cap of N=8.
3131    pub(crate) coder_discussion_slots: Arc<tokio::sync::Semaphore>,
3132    /// In-flight `agents.chat` sessions keyed by `session_id`. See
3133    /// [`ChatSession`] for shape. Populated by `agents.chat`,
3134    /// cleared on terminal `agent.chat.event` or
3135    /// `agents.chat.cancel`. Disconnect cleanup happens in
3136    /// `remove_session` — any in-flight session bound to either the
3137    /// disconnecting host or agent client is dropped so subsequent
3138    /// stray notifications from a respawned agent fall on the floor
3139    /// rather than racing into a stale stream.
3140    pub chat_sessions: Mutex<HashMap<String, ChatSession>>,
3141    /// Per-recipient channel guards for peer messaging (`agents.message`).
3142    ///
3143    /// Keyed by recipient agent id. Bounds the *channel* rather than the
3144    /// sender's authority: size, identical repeats, per-sender rate, and queue
3145    /// depth. Two agents answering each other form a loop that no policy rule
3146    /// catches, because neither is misbehaving — this is what terminates it.
3147    ///
3148    /// Entries are dropped in `remove_session` when the agent detaches, so a
3149    /// restarted agent starts with a clean budget rather than inheriting the
3150    /// rate history of the process that used to own its name.
3151    pub peer_guards: Mutex<HashMap<String, car_peers::DeliveryGuard>>,
3152    /// Earned standing per sending principal — a peer key inbound, a local
3153    /// agent id outbound.
3154    ///
3155    /// **Deliberately not reaped on detach**, unlike [`Self::peer_guards`] one
3156    /// line above. That hook exists so a respawned agent does not inherit a
3157    /// dead process's rate and dedupe budget, which is *channel* state.
3158    /// Standing is the opposite kind: clearing it on disconnect would let a
3159    /// degraded agent wipe its record by reattaching, which is a cheaper
3160    /// erasure than the renaming this design already refuses to allow.
3161    /// Bounded by TTL and map cap instead, pruned on write.
3162    pub peer_standing: Mutex<HashMap<String, crate::peers::PeerStanding>>,
3163    /// Peer messages held for operator approval, oldest first.
3164    ///
3165    /// A message is held when the sender's `read_only` posture is
3166    /// `require_approval`. Bounded at [`car_peers::HOLD_CAP`]; past that the
3167    /// oldest is dropped, so a stream of held messages cannot grow without
3168    /// limit. Kept flat rather than per-recipient because approval is an
3169    /// operator action over the whole queue, not a per-agent one.
3170    ///
3171    /// Deliberately in memory only: a held message is a live decision awaiting a
3172    /// human, and one that outlived a daemon restart would be delivered into a
3173    /// world that had moved on.
3174    pub held_peer_messages: Mutex<std::collections::VecDeque<crate::peers::HeldPeerMessage>>,
3175    /// Live view of CAR daemons discovered on the local network.
3176    ///
3177    /// `None` when mDNS could not start (no multicast, a locked-down sandbox) —
3178    /// LAN discovery is then simply absent rather than the daemon failing to
3179    /// boot. A network with no peers and a network CAR cannot browse look the
3180    /// same to a caller, which is why the status surface reports which it is.
3181    pub lan_discovery: std::sync::Mutex<Option<car_a2a::lan::LanDirectory>>,
3182    /// This daemon's ed25519 peer identity, and the keys it accepts.
3183    ///
3184    /// The identity signs outbound cross-host requests; the trust set decides
3185    /// which inbound signers are CAR. Both are `None` until the A2A surface
3186    /// starts, because neither is meaningful without a network surface to
3187    /// authenticate.
3188    pub peer_identity: std::sync::Mutex<Option<Arc<car_a2a::peer_auth::PeerIdentity>>>,
3189    pub peer_trust: car_a2a::peer_auth::PeerTrust,
3190    /// In-process collectors for `agent.chat` streams that have **no** host UI
3191    /// to forward to — currently the A2A conversational bridge, which reverse-
3192    /// calls `agent.chat` on a host session and aggregates the streamed deltas
3193    /// into a single reply. Keyed by `session_id`. When a collector exists for a
3194    /// session, `try_forward_agent_chat_event` feeds chunks here instead of
3195    /// forwarding to a host channel. The collecting task owns the entry's
3196    /// lifetime (inserts before the reverse-call, removes when done/timed out).
3197    pub chat_collectors: Mutex<HashMap<String, ChatCollector>>,
3198    /// Standing deterministic chat goals keyed by `session_id`. This is a
3199    /// host-facing status/control registry, separate from ephemeral
3200    /// `chat_sessions` routing so a goal can be inspected after a turn finishes.
3201    pub chat_goals: Mutex<HashMap<String, ChatGoalState>>,
3202    /// Agent runs keyed by `run_id` (agent run tracing, U1). Process-
3203    /// wide (not per-session) so a run's record outlives the WS
3204    /// connection that produced it — the durable, connection-
3205    /// independent boundary `client_id` cannot be (R1). Populated by
3206    /// `runs.start`, made terminal by `runs.complete`, and swept to
3207    /// `Incomplete` on a mid-run disconnect past the grace window
3208    /// (R5). U2/U3 build the per-turn recorder and disk store on top
3209    /// of this registry.
3210    pub runs: Mutex<HashMap<String, RunMeta>>,
3211    /// Per-run serialization for trace append/fsync and terminal persistence.
3212    /// Filesystem work never holds the global [`runs`](Self::runs) lock.
3213    run_durability_locks: Mutex<HashMap<String, std::sync::Weak<Mutex<()>>>>,
3214    /// Live `runs.trace.event` subscribers keyed by `(run_id,
3215    /// host_client_id)` (agent run tracing, U4). Each value is a
3216    /// [`crate::host::RunTraceSubscriber`] — the producer side of a
3217    /// bounded channel whose dedicated drain task writes frames to that
3218    /// connection's WS. Two CarHost windows on one run register two
3219    /// distinct entries (explicit fanout — the built-in notification
3220    /// registry is single-subscriber-per-method).
3221    ///
3222    /// **Lock contract (invariant #1):** `runs.subscribe` snapshots the
3223    /// run's turns AND inserts the subscriber while holding the
3224    /// [`runs`](Self::runs) lock; the recorder (`record_run_turns`),
3225    /// `start_run`, `complete_run`, and `mark_run_incomplete` append to
3226    /// `runs` and push to `run_subscribers` while holding the SAME
3227    /// `runs` lock — so snapshot/register and append/notify are
3228    /// serialized. No turn appended in the snapshot/register window is
3229    /// dropped (gap) or double-delivered (dup). The lock order is always
3230    /// `runs` → `run_subscribers`; never the reverse.
3231    pub run_subscribers: Mutex<HashMap<(String, String), crate::host::RunTraceSubscriber>>,
3232    /// Every browser the drawer can reach (`browser.view.*`), keyed by
3233    /// conversation/agent-session — plus the one standing user session every
3234    /// conversation without an agent-attached browser shares. Owns the
3235    /// per-view snapshot/cursor/subscriber fanout; see
3236    /// [`crate::browser_view`].
3237    pub browser_views: Arc<crate::browser_view::BrowserViewRegistry>,
3238    /// Disk-backed run-trace store (agent run tracing, U3). Source of
3239    /// truth for REPLAY (U5) — persists each run's `RunStarted`, turns,
3240    /// and terminal record as JSONL under `~/.car/runs/{agent_id}/` so a
3241    /// run survives daemon restarts (R4), bounded by retention/GC (R6)
3242    /// and protected at rest with `0600`/`0700` perms + backup exclusion
3243    /// (R14). The in-memory [`runs`](Self::runs) buffer stays the source
3244    /// for the LIVE stream (U4); this store mirrors what was recorded.
3245    /// Derived from `journal_dir` at construction (sibling `runs/`).
3246    pub run_store: crate::run_store::RunStore,
3247    journal_failures: Option<car_eventlog::JournalFailureInjector>,
3248    /// Bound MCP HTTP-streamable URL (e.g.
3249    /// `"http://127.0.0.1:9102/mcp"`) — `car-server` installs this
3250    /// after binding the listener. Used by the
3251    /// `agents.invoke_external` handler to default
3252    /// `InvokeOptions.mcp_endpoint` so external agents
3253    /// (Claude Code today) load the daemon's CAR namespace via
3254    /// `--mcp-config` automatically. `None` when MCP isn't bound
3255    /// (e.g. `--mcp-bind disabled`).
3256    pub mcp_url: std::sync::OnceLock<String>,
3257    /// Approval gate for high-risk WS methods (audit 2026-05). The
3258    /// gate intercepts `automation.run_applescript`,
3259    /// `automation.shortcuts.run`, `messages.send`, `mail.send`, and
3260    /// `vision.ocr` before they dispatch, raises a
3261    /// `host.create_approval` for the user to act on, and waits
3262    /// (with a timeout) for `host.resolve_approval`. Approve →
3263    /// dispatch continues; deny / timeout → JSON-RPC error code
3264    /// `-32003`. The set of gated methods and the wait timeout are
3265    /// embedder-overridable via
3266    /// [`ServerStateConfig::with_approval_gate`].
3267    pub approval_gate: ApprovalGate,
3268    /// Out-of-process supervision (proposal item 5). Holds the subscribed
3269    /// supervisors and the intents parked on their verdicts; the matching
3270    /// [`crate::supervision::SupervisionGate`] is registered as an
3271    /// `AdmissionGate` on every session runtime alongside
3272    /// `StaticVerificationGate`.
3273    ///
3274    /// Daemon-wide rather than per-session, deliberately: a supervisor connects
3275    /// on its OWN WebSocket and supervises proposals submitted on OTHER
3276    /// sessions. A per-session registry would only ever see the supervisor's
3277    /// own (empty) traffic.
3278    ///
3279    /// Costs nothing until someone subscribes — the gate returns `Allow`
3280    /// without building an intent while the subscriber set is empty.
3281    pub supervision: Arc<crate::supervision::SupervisionRegistry>,
3282    /// The daemon-wide shared HITL approval ledger (kernel review C1) —
3283    /// journal-backed at `~/.car/approvals.jsonl` (override via
3284    /// [`ServerStateConfig::with_approval_journal`]). ONE ledger for all
3285    /// sessions: `permission.approve`/`reject` on any connection records
3286    /// here, and every ledger consumer (`permission.evaluate`/`pending`,
3287    /// `evolution.run`, `cascade.run`, `skill.enforce_deployment`/
3288    /// `ingest_governed`) reads here — so an approval granted by a host
3289    /// connection is visible to the agent connection that surfaced it and
3290    /// survives daemon restart. Per-session gates keep only tier/classifier
3291    /// state (see [`ClientSession::permission_gate`]). Falls back to an
3292    /// in-memory ledger (loudly warned, approvals NOT restart-durable) only
3293    /// when the journal is unopenable.
3294    pub approval_ledger: Arc<tokio::sync::RwLock<car_policy::ApprovalLedger>>,
3295    /// The in-process harness evaluator, when the binary installed one — the
3296    /// thing that lets `evolution.run` grade a harness candidate ITSELF
3297    /// instead of waiting for an operator to run `car-bench-harness` twice and
3298    /// hand both files back in.
3299    ///
3300    /// `None` on a build that did not install one (every embedder, and any
3301    /// binary other than `car-server`): `evolution.run` with `harness_measure`
3302    /// then ERRS rather than quietly falling back to HITL, because an opt-in
3303    /// that silently does nothing would report an unattended cycle that never
3304    /// measured anything.
3305    ///
3306    /// A `std::sync::RwLock` on purpose — no caller holds it across an await;
3307    /// they clone the `Arc` out and drop the guard.
3308    pub harness_measurer: std::sync::RwLock<Option<Arc<dyn crate::evolution::HarnessMeasurer>>>,
3309    /// A2A-runtime / store / card factory carried over from the
3310    /// embedder's [`ServerStateConfig`]. Consumed lazily on first
3311    /// `a2a_dispatcher()` call so embedders can construct
3312    /// `ServerState` without paying the runtime spin-up cost when
3313    /// they don't actually use the A2A surface.
3314    pub(crate) a2a_runtime: std::sync::Mutex<Option<Arc<car_engine::Runtime>>>,
3315    pub(crate) a2a_store: std::sync::Mutex<Option<Arc<dyn car_a2a::TaskStore>>>,
3316    pub(crate) a2a_card_source: std::sync::Mutex<Option<Arc<car_a2a::AgentCardSource>>>,
3317}
3318
3319/// Default location of the shared approval-ledger journal: `approvals.jsonl`
3320/// directly under the CAR state root, so it follows `CAR_HOME` rather than
3321/// pinning itself to `~/.car` (which is still where it lands when `CAR_HOME` is
3322/// unset — HOME, or USERPROFILE on Windows). Creates the root best-effort;
3323/// `None` when neither `CAR_HOME` nor a home directory is resolvable.
3324fn default_approval_journal_path() -> Option<PathBuf> {
3325    let dir = car_home::root()?;
3326    let _ = std::fs::create_dir_all(&dir);
3327    Some(dir.join("approvals.jsonl"))
3328}
3329
3330/// Resolve the peer audit journal once, while constructing [`ServerState`].
3331///
3332/// The standalone daemon's default journal directory is `<CAR_HOME>/journals`,
3333/// so preserve the established `<CAR_HOME>/peer-messages.jsonl` location in
3334/// that configuration. A custom journal directory — including every hermetic
3335/// test tempdir — owns its peer audit journal directly. No writer re-reads
3336/// `CAR_HOME`, so process-global state cannot redirect a configured test state.
3337fn default_peer_audit_journal_path(journal_dir: &Path) -> PathBuf {
3338    if let Some(car_dir) = car_home::root() {
3339        if journal_dir == car_dir.join("journals") {
3340            return car_dir.join("peer-messages.jsonl");
3341        }
3342    }
3343    journal_dir.join("peer-messages.jsonl")
3344}
3345
3346/// The daemon's state root: `CAR_HOME` when set, otherwise `~/.car` (HOME, or
3347/// USERPROFILE on Windows).
3348///
3349/// One convention, used by [`default_approval_journal_path`], the `messaging.*`
3350/// handlers and now the project policy loader — a second one would mean an
3351/// operator's rules living somewhere their pairing config does not. Going
3352/// through `car_home` is what keeps that true once an operator relocates the
3353/// root: everything moves together or nothing does.
3354fn car_home_dir() -> Option<PathBuf> {
3355    car_home::root()
3356}
3357
3358/// Register the declarative rules in `<car_dir>/policies/*.toml` on `runtime`.
3359///
3360/// **A malformed file is fatal, on purpose. Do not downgrade this to a
3361/// warning.** The loader's own words (`car_policy::load_policy_dir`): "a
3362/// security rule that fails to parse must surface, never be silently skipped."
3363/// A rule file exists to *stop* something; a typo that turns "never message
3364/// anyone outside the allowlist" into a no-op fails open, and it fails open
3365/// silently, at exactly the moment the operator believes they are protected. A
3366/// daemon that refuses to open the session is loud, immediate, and trivially
3367/// fixed; a daemon that logs a warning nobody reads is neither. The error text
3368/// names the offending file (`PolicyLoadError`'s `Display`) so the fix is
3369/// mechanical.
3370///
3371/// A **missing** directory is not a failure — most projects have no rules, and
3372/// `load_policy_dir` already returns an empty set for it.
3373///
3374/// Split out of [`ServerState::create_session`] so the fatal-vs-silent
3375/// behaviour is testable against a temp directory instead of the real `$HOME`.
3376pub(crate) async fn apply_project_policies(
3377    runtime: &Runtime,
3378    car_dir: &std::path::Path,
3379) -> Result<(), String> {
3380    match runtime.load_project_policies(car_dir).await {
3381        Ok(_) => Ok(()),
3382        Err(e) => Err(format!(
3383            "refusing to start with unreadable project policy rules: {e}. \
3384             Fix or remove the file — a policy rule that fails to parse is a \
3385             security control that would silently not exist."
3386        )),
3387    }
3388}
3389
3390impl ServerState {
3391    async fn run_durability_lock(&self, run_id: &str) -> Arc<Mutex<()>> {
3392        let mut locks = self.run_durability_locks.lock().await;
3393        if let Some(lock) = locks.get(run_id).and_then(std::sync::Weak::upgrade) {
3394            return lock;
3395        }
3396        locks.retain(|_, lock| lock.strong_count() > 0);
3397        let lock = Arc::new(Mutex::new(()));
3398        locks.insert(run_id.to_string(), Arc::downgrade(&lock));
3399        lock
3400    }
3401
3402    async fn quarantine_run_trace_locked(
3403        &self,
3404        runs: &mut HashMap<String, RunMeta>,
3405        run_id: &str,
3406        detail: String,
3407    ) -> String {
3408        let message = run_trace_corruption_message(run_id, detail);
3409        if let Some(meta) = runs.get_mut(run_id) {
3410            if meta.trace_corruption.as_deref() != Some(message.as_str()) {
3411                meta.durability_generation = meta.durability_generation.wrapping_add(1);
3412            }
3413            meta.trace_corruption = Some(message.clone());
3414        }
3415        let mut subs = self.run_subscribers.lock().await;
3416        subs.retain(|(subscribed_run, _), _| subscribed_run != run_id);
3417        message
3418    }
3419
3420    /// Publish corruption found by a strict replay read into live lifecycle
3421    /// state. Disk scanning and marker persistence happen before this call;
3422    /// only the per-run durability gate and short registry/subscriber mutation
3423    /// execute here, so a corrupt trace cannot remain writable or subscribed.
3424    pub(crate) async fn quarantine_run_trace_from_read(
3425        &self,
3426        run_id: &str,
3427        detail: String,
3428    ) -> String {
3429        let durability_lock = self.run_durability_lock(run_id).await;
3430        let _durability_guard = durability_lock.lock().await;
3431        let mut runs = self.runs.lock().await;
3432        self.quarantine_run_trace_locked(&mut runs, run_id, detail)
3433            .await
3434    }
3435
3436    /// Constructor for the standalone `car-server` binary. Each WS
3437    /// connection gets its own per-session memgine — matches the
3438    /// pre-extraction default and is correct for a single-process
3439    /// daemon serving one user at a time.
3440    ///
3441    /// **Embedders must not call this.** It silently leaves
3442    /// `shared_memgine = None`, which re-introduces the dual-memgine
3443    /// bug U7 was created to prevent (one engine in the embedder, a
3444    /// fresh one inside every WS session). Embedders use
3445    /// [`ServerState::embedded`] instead, which makes the shared
3446    /// engine handle a required argument so it cannot be forgotten.
3447    pub fn standalone(journal_dir: PathBuf) -> Self {
3448        Self::with_config(ServerStateConfig::new(journal_dir))
3449    }
3450
3451    /// The daemon-held multi-device-sync + execution-lease subsystem (B6),
3452    /// lazily opened on first `sync.*`/`lease.*` contact and rooted at
3453    /// `<journal_dir>/sync/`. Serialized first-open (see the [`Self::sync`]
3454    /// field docs): the oplog journal's exclusive advisory lock means only one
3455    /// open may succeed, so the fallible init runs under the field's
3456    /// `std::sync::Mutex` while the subsystem itself is shared behind a
3457    /// `tokio::sync::Mutex`.
3458    pub fn sync_subsystem(
3459        &self,
3460    ) -> Result<Arc<tokio::sync::Mutex<crate::sync::SyncSubsystem>>, String> {
3461        let mut guard = self
3462            .sync
3463            .lock()
3464            .map_err(|_| "sync subsystem init lock poisoned".to_string())?;
3465        if let Some(existing) = guard.as_ref() {
3466            return Ok(existing.clone());
3467        }
3468        let root = self.journal_dir.join("sync");
3469        let subsystem = self.open_sync_subsystem(&root)?;
3470        let arc = Arc::new(tokio::sync::Mutex::new(subsystem));
3471        *guard = Some(arc.clone());
3472        Ok(arc)
3473    }
3474
3475    /// Route a write to the right subsystem BY SCOPE: an opted-in
3476    /// `Scope::Shared{org}` goes to the shared org delivery subsystem (relay scope
3477    /// `org:{orgId}`, so it converges cross-member); everything else — `Personal`,
3478    /// or an org with no opted-in delivery — goes to the personal subsystem, which
3479    /// is byte-identical to before when org-scope is off. Used ONLY at the write
3480    /// sites; every reader keeps calling [`Self::sync_subsystem`]. (8a)
3481    pub fn subsystem_for_scope(
3482        &self,
3483        scope: &car_sync::Scope,
3484    ) -> Result<Arc<tokio::sync::Mutex<crate::sync::SyncSubsystem>>, String> {
3485        // Init the personal subsystem first — this also builds + stores the org
3486        // subsystem when opted in (via `open_sync_subsystem`).
3487        let user = self.sync_subsystem()?;
3488        let guard = self
3489            .org_sync
3490            .lock()
3491            .map_err(|_| "org sync lock poisoned".to_string())?;
3492        match route_for_scope(scope, guard.as_ref().map(|h| h.org.as_str())) {
3493            SyncRoute::Org => Ok(guard
3494                .as_ref()
3495                .expect("route_for_scope returns Org only when a holder exists")
3496                .subsystem
3497                .clone()),
3498            SyncRoute::User => Ok(user),
3499        }
3500    }
3501
3502    /// The opted-in shared org delivery subsystem, if any — for pumping both scopes.
3503    pub fn org_subsystem(&self) -> Option<Arc<tokio::sync::Mutex<crate::sync::SyncSubsystem>>> {
3504        match self.org_sync.lock() {
3505            Ok(g) => g.as_ref().map(|h| h.subsystem.clone()),
3506            // Surface the poison loudly — do NOT silently degrade to personal-only
3507            // sync (linus B). Poison only occurs on a panic while locked.
3508            Err(_) => {
3509                tracing::error!(
3510                    "org_sync lock poisoned — skipping the org pump (org-scope delivery degraded)"
3511                );
3512                None
3513            }
3514        }
3515    }
3516
3517    /// Build the sync subsystem for `root`, selecting the backend from config.
3518    ///
3519    /// Default is the local shared-directory `FsRelay`. When
3520    /// `PARSLEE_SYNC_BACKEND=parslee` (env or keychain), it builds the
3521    /// Parslee-backed device: a `ParsleeSyncTransport` at
3522    /// `PARSLEE_CAR_SYNC_BASE_URL` (default `{api_base}/sync`), authed with the
3523    /// live refresh-aware bearer, scoped `user:{account_id}`, E2E-encrypted under
3524    /// a passphrase-derived key (`PARSLEE_SYNC_PASSPHRASE`, the zero-knowledge
3525    /// cross-device key — same passphrase → same key on every device, never sent
3526    /// to the server). Missing a Parslee login falls back to local (logged);
3527    /// missing the passphrase is a hard error (refusing to sync unencrypted).
3528    fn open_sync_subsystem(
3529        &self,
3530        root: &std::path::Path,
3531    ) -> Result<crate::sync::SyncSubsystem, String> {
3532        let backend = car_secrets::resolve_env_or_keychain("PARSLEE_SYNC_BACKEND")
3533            .unwrap_or_default()
3534            .to_lowercase();
3535        if backend != "parslee" {
3536            return crate::sync::SyncSubsystem::open(root);
3537        }
3538
3539        let Some(account_id) = self
3540            .parslee_session
3541            .get()
3542            .map(|s| s.identity.account_id.clone())
3543            .filter(|a| !a.is_empty())
3544        else {
3545            tracing::warn!(
3546                "sync backend=parslee but no Parslee login is present; using local sync until login"
3547            );
3548            return crate::sync::SyncSubsystem::open(root);
3549        };
3550
3551        let Some(passphrase) = car_secrets::resolve_env_or_keychain("PARSLEE_SYNC_PASSPHRASE")
3552            .filter(|p| !p.is_empty())
3553        else {
3554            return Err(
3555                "sync backend=parslee requires PARSLEE_SYNC_PASSPHRASE (the zero-knowledge \
3556                 cross-device sync key) — set it in the keychain; refusing to sync unencrypted"
3557                    .to_string(),
3558            );
3559        };
3560
3561        let base_url = car_secrets::resolve_env_or_keychain("PARSLEE_CAR_SYNC_BASE_URL")
3562            .filter(|u| !u.is_empty())
3563            .unwrap_or_else(|| {
3564                let api_base = car_auth::api_base(None);
3565                format!("{}/sync", api_base.trim_end_matches('/'))
3566            });
3567
3568        let bearer: car_parslee::sync_transport::BearerFn = Arc::new(car_auth::access_token);
3569        let transport = Arc::new(
3570            car_parslee::sync_transport::ParsleeSyncTransport::new(base_url, bearer)
3571                .map_err(|e| format!("sync: build Parslee transport: {e}"))?,
3572        );
3573        // Argon2id-stretch the passphrase into the master ONCE at open (memory-hard
3574        // work factor for a password-equivalent secret), then feed BOTH the
3575        // per-audience provider and the org identity from it — so Argon2id runs
3576        // exactly once, not once per derivation.
3577        let master = car_sync::StretchedMaster::from_passphrase(passphrase.as_bytes(), &account_id);
3578        let base: Arc<dyn car_sync::SyncKeyProvider> =
3579            Arc::new(car_sync::DerivedKeyProvider::from_master(&master));
3580
3581        // Org-scope shared-brain activation — REAL provisioning, but DEFAULT-OFF.
3582        // `PARSLEE_SYNC_ORG_SCOPE=<orgId>:<granter_hex>[,<granter_hex>...]` names the
3583        // org AND the admin-designated trusted granter key(s), held LOCALLY (never
3584        // backend-fetched — see `parse_org_scope_config`). Unset → `base`
3585        // (byte-identical to the personal-only path). Malformed → hard error above.
3586        // The configured org MUST match the signed-in `active_organization` or we
3587        // refuse loudly. Turning it ON for a real tenant is slice 12, still gated on
3588        // the cryptographer audit of the login_secret→identity entropy dependency.
3589        let mut pending_org: Option<OrgSyncHolder> = None;
3590        let provider: Arc<dyn car_sync::SyncKeyProvider> = match Self::parse_org_scope_config(
3591            car_secrets::resolve_env_or_keychain("PARSLEE_SYNC_ORG_SCOPE"),
3592        )? {
3593            None => base,
3594            Some((org_id, granters)) => {
3595                // Bind the configured org to the signed-in active organization — never
3596                // encrypt org ops for an org this account is not actually a member of.
3597                let active_org = self
3598                    .parslee_session
3599                    .get()
3600                    .and_then(|s| s.identity.active_organization.clone())
3601                    .filter(|o| !o.is_empty());
3602                if active_org.as_deref() != Some(org_id.as_str()) {
3603                    return Err(format!(
3604                        "PARSLEE_SYNC_ORG_SCOPE org {org_id:?} does not match the signed-in active \
3605                         organization {active_org:?} — refusing to activate org scope"
3606                    ));
3607                }
3608
3609                // Bring the directory trait into scope for `publish_pubkey`.
3610                use car_sync::OrgKeyDirectory as _;
3611                let my_secret = car_sync::derive_x25519_identity(&master, &account_id);
3612                let mut directory = car_sync::NetworkOrgKeyDirectory::new(
3613                    transport.clone(),
3614                    format!("org:{org_id}"),
3615                );
3616
3617                // Announce our X25519 identity pubkey so a granter can wrap K_org for
3618                // us (idempotent LWW). Non-fatal: a failure just means future grants
3619                // may not reach us; any already-resolved roots still work. This routes
3620                // to the bearer-authenticated backend, but whether the backend
3621                // ENFORCES "only this account may publish its own pubkey" (and "only a
3622                // K_org holder may publish a wrap") is the confidentiality-critical,
3623                // AUDIT-OWNED check — this is not a guarantee against a hostile backend.
3624                let my_pub = car_sync::x25519_public(&my_secret);
3625                let my_pub_hex: String = my_pub
3626                    .as_bytes()
3627                    .iter()
3628                    .map(|b| format!("{b:02x}"))
3629                    .collect();
3630                if let Err(e) = directory.publish_pubkey(&account_id, &my_pub_hex) {
3631                    tracing::warn!(org = %org_id, error = %e, "org-scope: failed to publish member pubkey — future grants may not reach this member");
3632                }
3633
3634                // Resolve EVERY epoch of K_org we can unwrap (slice 10) → the
3635                // multi-epoch map the cipher selects by kid. `org → { epoch → K_org }`.
3636                // Fail-closed: Err (directory unreachable) and empty (ungranted) both
3637                // leave org scope on DenyCipher, NEVER demoted to the personal key.
3638                let mut roots = std::collections::HashMap::new();
3639                match car_sync::resolve_all_org_roots(
3640                    &directory,
3641                    &org_id,
3642                    &my_secret,
3643                    &account_id,
3644                    &granters,
3645                ) {
3646                    Ok(map) if !map.is_empty() => {
3647                        tracing::info!(org = %org_id, epochs = ?map.keys().collect::<Vec<_>>(), "org-scope: resolved K_org generations");
3648                        roots.insert(org_id.clone(), map);
3649                    }
3650                    Ok(_) => tracing::warn!(
3651                        org = %org_id,
3652                        "org-scope: no trusted grant for this member — org scope fails closed (DenyCipher)"
3653                    ),
3654                    Err(e) => tracing::warn!(
3655                        org = %org_id, error = %e,
3656                        "org-scope: org-key directory unreachable — org scope fails closed (DenyCipher), NOT demoted to the personal key"
3657                    ),
3658                }
3659                // ALWAYS OrgAwareKeyProvider once opted in — NEVER the bare
3660                // DerivedKeyProvider, which has no Scope::Shared case and would
3661                // silently encrypt org ops under the PERSONAL key (a fail-open).
3662                let provider: Arc<dyn car_sync::SyncKeyProvider> =
3663                    Arc::new(car_sync::OrgAwareKeyProvider::new(base, roots));
3664
3665                // Open the SHARED org delivery subsystem: a SECOND device oplog
3666                // under `root/org-{orgId}/` whose relay scope is `org:{orgId}`, with
3667                // the SAME OrgAwareKeyProvider (it keys Scope::Shared{org} on K_org).
3668                // Scope::Shared{org} writes route here (subsystem_for_scope) and
3669                // converge across the org's members. `org_id` is a canonical slug
3670                // (validated on parse), so it is filesystem-safe. (8a)
3671                let org_root = root.join(format!("org-{org_id}"));
3672                let org_subsystem = crate::sync::SyncSubsystem::open_remote(
3673                    &org_root,
3674                    transport.clone(),
3675                    format!("org:{org_id}"),
3676                    provider.clone(),
3677                )?;
3678                // Do NOT commit to `org_sync` yet — hold the org subsystem in a
3679                // local and only install it AFTER the personal open below succeeds.
3680                // Else a failed personal open would orphan the org journal's
3681                // exclusive lock and permanently wedge sync retry (linus A).
3682                pending_org = Some(OrgSyncHolder {
3683                    org: org_id.clone(),
3684                    subsystem: Arc::new(tokio::sync::Mutex::new(org_subsystem)),
3685                });
3686                tracing::info!(org = %org_id, "org-scope: opened shared org delivery subsystem (relay org:{org_id})");
3687                provider
3688            }
3689        };
3690        let user = crate::sync::SyncSubsystem::open_remote(
3691            root,
3692            transport,
3693            format!("user:{account_id}"),
3694            provider,
3695        )?;
3696        // Personal open succeeded — NOW commit the org holder. On the failure path
3697        // above, `pending_org` drops here, releasing the org journal lock so a
3698        // retry can re-open cleanly.
3699        if let Some(holder) = pending_org {
3700            *self
3701                .org_sync
3702                .lock()
3703                .map_err(|_| "org sync init lock poisoned".to_string())? = Some(holder);
3704        }
3705        Ok(user)
3706    }
3707
3708    /// Parse `PARSLEE_SYNC_ORG_SCOPE` (`<orgId>:<granter_hex>[,<granter_hex>...]`).
3709    /// `Ok(None)` when unset/empty → the personal-only path. `Err` on ANY malformed
3710    /// value — a typo must be a loud failure, never a silent fall-through to a
3711    /// wrong or unauthenticated state. Returns the org id + the (≥1) admin-designated
3712    /// trusted granter verifying keys.
3713    ///
3714    /// The granter set is held LOCALLY (operator-provisioned), NEVER fetched from
3715    /// the backend: whoever fills the `trusted` slice holds the authority, and if
3716    /// the same backend that serves the directory could name the granters it could
3717    /// name ITSELF a granter and substitute `K_org`. A backend-sourced / signed
3718    /// granter manifest is an audit-owned trust decision (slice 12), not this.
3719    /// Pure over the raw value so it is unit-testable without the environment.
3720    fn parse_org_scope_config(
3721        raw: Option<String>,
3722    ) -> Result<Option<(String, Vec<car_sync::OrgVerifyingKey>)>, String> {
3723        let Some(raw) = raw.filter(|s| !s.is_empty()) else {
3724            return Ok(None);
3725        };
3726        let (org_id, granters_csv) = raw.split_once(':').ok_or_else(|| {
3727            "PARSLEE_SYNC_ORG_SCOPE must be <orgId>:<granter_ed25519_pubkey_hex>[,...]".to_string()
3728        })?;
3729        // Honor the "loud failure" rule: a non-canonical org id (space, Unicode,
3730        // the platform's raw tenant form) must error here, not silently resolve
3731        // nothing — it would never match a real wrap's canonical `org` anyway.
3732        car_sync::require_canonical_org(org_id)
3733            .map_err(|e| format!("PARSLEE_SYNC_ORG_SCOPE: bad orgId: {e}"))?;
3734        let granters = granters_csv
3735            .split(',')
3736            .map(str::trim)
3737            .filter(|h| !h.is_empty())
3738            .map(|hex| {
3739                car_sync::parse_ed25519_verifying(hex)
3740                    .map_err(|e| format!("PARSLEE_SYNC_ORG_SCOPE: bad granter key {hex:?}: {e}"))
3741            })
3742            .collect::<Result<Vec<_>, _>>()?;
3743        if granters.is_empty() {
3744            return Err(
3745                "PARSLEE_SYNC_ORG_SCOPE: at least one granter ed25519 pubkey is required".into(),
3746            );
3747        }
3748        Ok(Some((org_id.to_string(), granters)))
3749    }
3750
3751    /// Persist the daemon-wide standing chat-goal registry. The snapshot is
3752    /// taken under the async mutex, then serialized and atomically rewritten on
3753    /// the blocking pool so chat/event dispatch does not block a Tokio worker.
3754    pub async fn persist_chat_goals(&self) -> Result<(), String> {
3755        let path = chat_goals_path_from_journal_dir(&self.journal_dir);
3756        let goals = self.chat_goals.lock().await.clone();
3757        tokio::task::spawn_blocking(move || {
3758            write_chat_goals_atomic(&path, &goals)
3759                .map_err(|e| format!("write {}: {e}", path.display()))
3760        })
3761        .await
3762        .map_err(|e| format!("persist chat goals join: {e}"))?
3763    }
3764
3765    /// Constructor for embedders (e.g. `tokhn-daemon`). The shared
3766    /// memgine handle is **required**: every WS session created by
3767    /// this state will reuse the same engine, preventing the
3768    /// dual-memgine bug.
3769    ///
3770    /// For embedders that also want to inject a pre-warmed inference
3771    /// engine or other advanced wiring, build a [`ServerStateConfig`]
3772    /// directly and call [`ServerState::with_config`].
3773    pub fn embedded(
3774        journal_dir: PathBuf,
3775        shared_memgine: Arc<Mutex<car_memgine::MemgineEngine>>,
3776    ) -> Self {
3777        Self::with_config(ServerStateConfig::new(journal_dir).with_shared_memgine(shared_memgine))
3778    }
3779
3780    /// Build a `ServerState` from a [`ServerStateConfig`] — the path
3781    /// embedders use when they need to inject a shared memgine *and*
3782    /// a pre-warmed inference engine, or any other advanced wiring
3783    /// the convenience constructors don't cover.
3784    pub fn with_config(cfg: ServerStateConfig) -> Self {
3785        Self::try_with_config(cfg)
3786            .unwrap_or_else(|error| panic!("server state startup failed: {error}"))
3787    }
3788
3789    /// Build a state with test-owned feedback diagnostic roots and no ambient
3790    /// registry probes. This bypasses the production diagnostic default rather
3791    /// than constructing it and then overriding it, so state construction never
3792    /// resolves `HOME`/`USERPROFILE` for feedback. Public only because Cargo
3793    /// integration tests link `car-server-core` without `cfg(test)`.
3794    #[doc(hidden)]
3795    pub fn with_config_and_isolated_feedback_diagnostics(
3796        cfg: ServerStateConfig,
3797        models_dir: PathBuf,
3798        huggingface_hub_root: PathBuf,
3799    ) -> Self {
3800        Self::try_with_config_and_feedback_diagnostics(
3801            cfg,
3802            crate::feedback::FeedbackDiagnostics::isolated(models_dir, huggingface_hub_root),
3803        )
3804        .unwrap_or_else(|error| panic!("server state startup failed: {error}"))
3805    }
3806
3807    /// Fallible startup surface used by the daemon before it begins listening.
3808    /// Durable lifecycle outbox replay is acknowledgement-bounded; any
3809    /// durability-unknown result stops construction and leaves its RunStore
3810    /// preimage/quarantine intact for an exact later retry.
3811    pub fn try_with_config(cfg: ServerStateConfig) -> Result<Self, String> {
3812        Self::try_with_config_and_feedback_diagnostics(
3813            cfg,
3814            crate::feedback::FeedbackDiagnostics::production(),
3815        )
3816    }
3817
3818    fn try_with_config_and_feedback_diagnostics(
3819        cfg: ServerStateConfig,
3820        feedback_diagnostics: crate::feedback::FeedbackDiagnostics,
3821    ) -> Result<Self, String> {
3822        let startup_acknowledgement_timeout = cfg.startup_reconciliation_acknowledgement_timeout;
3823        if startup_acknowledgement_timeout.is_zero()
3824            || startup_acknowledgement_timeout > car_eventlog::MAX_CRITICAL_ACKNOWLEDGEMENT_TIMEOUT
3825        {
3826            return Err(format!(
3827                "startup reconciliation acknowledgement timeout must be between 1ns and {}ms, got {}ms",
3828                car_eventlog::MAX_CRITICAL_ACKNOWLEDGEMENT_TIMEOUT.as_millis(),
3829                startup_acknowledgement_timeout.as_millis()
3830            ));
3831        }
3832        let inference = std::sync::OnceLock::new();
3833        if let Some(eng) = cfg.inference {
3834            // OnceLock::set returns Err if already set — fresh OnceLock
3835            // means it's empty, so this is infallible here.
3836            let _ = inference.set(eng);
3837        }
3838        let voice_sessions = Arc::new(car_voice::VoiceSessionRegistry::new());
3839        // Reap sessions whose clients dropped without calling
3840        // voice.transcribe_stream.stop (WS disconnect, process exit,
3841        // etc.). Listener handles otherwise leak for the daemon's
3842        // lifetime. `with_config` is sync but always called from the
3843        // `#[tokio::main]` entry point, so `Handle::try_current()`
3844        // inside `start_sweeper` finds the runtime.
3845        voice_sessions.start_sweeper();
3846        // UI-improvement agent is pure decision logic — no I/O, no
3847        // persistence handle. Memgine ingest of strategy outcomes is
3848        // the caller's responsibility (handler.rs after a successful
3849        // Decision::Patch). Keeps the agent crate Mutex-flavor
3850        // agnostic so it can compose with std/tokio mutex callers.
3851        let ui_agent = Arc::new(car_ui_agent::UIImprovementAgent::with_default_strategies());
3852        let ui_agent_oscillation = Arc::new(crate::ui_agent_loop::OscillationDetector::new());
3853        let ui_agent_budget = Arc::new(crate::ui_agent_loop::IterationBudget::new());
3854        // Disk-backed run-trace store (U3). Rooted at the `runs/` sibling
3855        // of the journal dir (`~/.car/runs`), with retention read from
3856        // `~/.car/config.toml`. Boot-time GC enforces the retention cap
3857        // (R6) — best-effort; a failed GC must never block startup, so the
3858        // count is dropped. Never evicts an in-progress run.
3859        let mut run_store = crate::run_store::RunStore::from_journal_dir(&cfg.journal_dir);
3860        if let Some(failures) = cfg.run_store_failures.clone() {
3861            run_store = run_store.with_failure_injector(failures);
3862        }
3863        if let Some(gate) = cfg.run_store_summary_read_gate.clone() {
3864            run_store = run_store.with_summary_read_gate(gate);
3865        }
3866        if let Some(gate) = cfg.run_store_summary_write_gate.clone() {
3867            run_store = run_store.with_summary_write_gate(gate);
3868        }
3869        if let Some(gate) = cfg.run_store_append_gate.clone() {
3870            run_store = run_store.with_append_gate(gate);
3871        }
3872        if let Some(gate) = cfg.run_store_lookup_gate.clone() {
3873            run_store = run_store.with_lookup_gate(gate);
3874        }
3875        if let Some(failures) = cfg.run_store_private_path_failures.clone() {
3876            run_store = run_store.with_private_path_failure_injector(failures);
3877        }
3878        run_store
3879            .reconcile_proposal_retry_rollbacks()
3880            .map_err(|error| format!("proposal retry rollback reconciliation failed: {error}"))?;
3881        let chat_goals = load_chat_goals_from_disk(&cfg.journal_dir);
3882        let selfheal_ledger = cfg.selfheal_ledger.clone().unwrap_or_else(|| {
3883            cfg.journal_dir
3884                .parent()
3885                .unwrap_or(&cfg.journal_dir)
3886                .join("selfheal")
3887                .join("detections.jsonl")
3888        });
3889        // Config lives beside the daemon's other state. A missing or empty
3890        // `heal.toml` is a disabled loop, which is the default: there are no
3891        // compiled-in targets, because a loop that opened pull requests in a
3892        // repository nobody named would be a surprise in somebody else's
3893        // tracker.
3894        let heal_state_dir =
3895            crate::coder::rpc::coder_state_dir().unwrap_or_else(|_| cfg.journal_dir.join("coder"));
3896        let heal_config_dir = car_home::root_or_relative();
3897        let heal = Arc::new(crate::coder::heal_service::HealService::with_config_dir(
3898            crate::coder::heal_config::HealConfig::load(&heal_config_dir),
3899            heal_state_dir,
3900            heal_config_dir,
3901        ));
3902        let selfheal = Arc::new(crate::selfheal::SelfhealService::open(
3903            selfheal_ledger,
3904            cfg.selfheal_interval_secs,
3905            cfg.selfheal_evidence.clone(),
3906            cfg.selfheal_source_probe.clone(),
3907            cfg.selfheal_replay_verb_probe.clone(),
3908        )?);
3909        // Drop terminal history outside the configured retention window before
3910        // lifecycle replay. This keeps the reconciliation set bounded even
3911        // after years of newsroom runs; in-progress traces are never evicted.
3912        let _evicted_before_replay = run_store.gc();
3913        let mut startup_journals =
3914            StartupJournalCache::new(&cfg.journal_dir, cfg.journal_failures.as_ref());
3915        // Establish the durable RunStarted bracket before replaying a pending
3916        // proposal terminal. Then adopt the crash orphan and run lifecycle
3917        // reconciliation again to add RunCompleted. Both passes share the
3918        // per-client journal cache, are exact-row idempotent, and preserve:
3919        // RunStarted -> ProposalCompleted -> RunCompleted.
3920        reconcile_durable_run_journals(
3921            &mut startup_journals,
3922            &run_store,
3923            startup_acknowledgement_timeout,
3924        )?;
3925        reconcile_pending_proposal_journals(
3926            &mut startup_journals,
3927            &run_store,
3928            startup_acknowledgement_timeout,
3929        )?;
3930        reconcile_completed_proposal_guards(&run_store);
3931        // The in-memory map is empty at boot, so any remaining on-disk
3932        // InProgress run is a crashed prior process; mark it Incomplete so it
3933        // is terminal and age-GC-eligible (FIX 4).
3934        let _adopted = run_store.adopt_orphans();
3935        reconcile_durable_run_journals(
3936            &mut startup_journals,
3937            &run_store,
3938            startup_acknowledgement_timeout,
3939        )?;
3940        let evicted = run_store.gc();
3941        if evicted > 0 {
3942            // Said out loud, for the same reason the coder GC below says it: an
3943            // operator whose own config just ate part of their store has to be
3944            // able to see it happened. This was silent, which is what made a
3945            // zero cap a data-loss incident rather than a log line (car#1338) —
3946            // and `max_age_days = 1` where someone meant `10` still deletes nine
3947            // days of traces, which the age of the policy has nothing to do with.
3948            tracing::info!(
3949                evicted,
3950                max_per_agent = ?run_store.retention().count_cap(),
3951                max_age_days = ?run_store.retention().age_cap_days(),
3952                "pruned run traces (~/.car/config.toml [runs] retention)"
3953            );
3954        }
3955        // Adopt crash/restart-orphaned coder sessions the same way: the
3956        // in-memory `coder_sessions` map is empty at boot, so any non-terminal
3957        // snapshot under the coder state dir is a prior process's orphan. Mark
3958        // genuinely-stranded ones Failed ("daemon restarted mid-session") and
3959        // preserve a `needs_approval` orphan whose worktree still exists so the
3960        // user can inspect/approve it by hand. Best-effort — never blocks boot.
3961        if let Ok(coder_dir) = crate::coder::rpc::coder_state_dir() {
3962            let _coder_adopted = crate::coder::session::adopt_orphaned_sessions(&coder_dir);
3963            // Then prune, the same way `run_store.gc()` above prunes run
3964            // traces. Adoption runs first on purpose: it is what turns a
3965            // stranded orphan terminal, and only terminal sessions are
3966            // collectable. An orphan adopted just now carries a fresh
3967            // `updated_at`, so this boot never collects what this boot
3968            // rewrote — it ages out on a later one.
3969            let retention = crate::coder::config::CoderConfig::load().session_retention();
3970            let collected = crate::coder::session::gc_sessions(
3971                &coder_dir,
3972                &retention,
3973                crate::coder::session::SweepScope::Boot,
3974            );
3975            if collected > 0 {
3976                // Said out loud, unlike `run_store.gc()` above. That policy has
3977                // existed since the run store did; this one is NEW and applies
3978                // retroactively, so the first boot after an upgrade can delete
3979                // years of snapshots under caps the operator never chose. An
3980                // operator who did not want that has to be able to see it
3981                // happened.
3982                tracing::info!(
3983                    collected,
3984                    max_sessions = retention.max_sessions,
3985                    max_age_days = retention.max_age_days,
3986                    "pruned coder session snapshots (~/.car/coder.toml retention)"
3987                );
3988            }
3989        }
3990        // The daemon-wide shared approval ledger (kernel review C1): one
3991        // journal-backed store for every session's HITL records, so approvals
3992        // cross connections and survive restart. In-memory fallback is loud —
3993        // a silent downgrade here would fake durability.
3994        let peer_audit_journal = cfg
3995            .peer_audit_journal
3996            .clone()
3997            .unwrap_or_else(|| default_peer_audit_journal_path(&cfg.journal_dir));
3998        let approval_ledger = {
3999            let path = cfg.approval_journal.or_else(default_approval_journal_path);
4000            match path {
4001                Some(p) => {
4002                    // The journal's parent must exist before the first
4003                    // append-mode open (ApprovalLedger::record does not
4004                    // create directories).
4005                    if let Some(parent) = p.parent() {
4006                        let _ = std::fs::create_dir_all(parent);
4007                    }
4008                    match car_policy::ApprovalLedger::with_journal(&p) {
4009                        Ok(l) => {
4010                            if l.skipped_on_load() > 0 {
4011                                tracing::warn!(
4012                                    path = %p.display(),
4013                                    skipped = l.skipped_on_load(),
4014                                    "approval-ledger journal had unparseable lines; partial ledger loaded"
4015                                );
4016                            }
4017                            l
4018                        }
4019                        Err(e) => {
4020                            tracing::warn!(
4021                                path = %p.display(), error = %e,
4022                                "approval-ledger journal unopenable; falling back to IN-MEMORY \
4023                                 ledger — approvals will NOT survive restart"
4024                            );
4025                            car_policy::ApprovalLedger::new()
4026                        }
4027                    }
4028                }
4029                None => {
4030                    tracing::warn!(
4031                        "no CAR_HOME/HOME/USERPROFILE to place approvals.jsonl under the CAR \
4032                         state root; approval ledger is IN-MEMORY — approvals will NOT survive \
4033                         restart"
4034                    );
4035                    car_policy::ApprovalLedger::new()
4036                }
4037            }
4038        };
4039        // Built before the struct literal so the browser-view registry can be
4040        // pointed at the SAME `HostState` it will broadcast sign-in attention
4041        // on (`host.event`) — see `crate::browser_attention`.
4042        let host = Arc::new(crate::host::HostState::new());
4043        let browser_views = Arc::new(crate::browser_view::BrowserViewRegistry::default());
4044        browser_views.set_signin_attention(Arc::new(
4045            crate::browser_attention::HostSignInAttention::new(Arc::clone(&host)),
4046        ));
4047        Ok(Self {
4048            journal_dir: cfg.journal_dir,
4049            peer_audit_journal,
4050            sessions: Mutex::new(HashMap::new()),
4051            inference,
4052            host,
4053            shared_memgine: cfg.shared_memgine,
4054            feedback_diagnostics,
4055            trajectory_store: Arc::new(car_memgine::TrajectoryStore::new(
4056                &cfg.trajectory_dir
4057                    .unwrap_or_else(car_memgine::TrajectoryStore::default_path),
4058            )),
4059            selfheal,
4060            heal,
4061            voice_sessions,
4062            meetings: Arc::new(car_meeting::MeetingRegistry::new()),
4063            a2ui: car_a2ui::A2uiSurfaceStore::new(),
4064            ui_agent,
4065            ui_agent_oscillation,
4066            ui_agent_budget,
4067            admission: Arc::new(crate::admission::InferenceAdmission::new()),
4068            a2ui_route_auth: Mutex::new(HashMap::new()),
4069            supervisor: std::sync::OnceLock::new(),
4070            declagents: std::sync::OnceLock::new(),
4071            routing: std::sync::OnceLock::new(),
4072            sync: std::sync::Mutex::new(None),
4073            org_sync: std::sync::Mutex::new(None),
4074            observer_manifest_path: std::sync::OnceLock::new(),
4075            a2a_dispatcher: std::sync::OnceLock::new(),
4076            a2a_runtime: std::sync::Mutex::new(cfg.a2a_runtime),
4077            a2a_store: std::sync::Mutex::new(cfg.a2a_store),
4078            a2a_card_source: std::sync::Mutex::new(cfg.a2a_card_source),
4079            a2ui_subscribers: Mutex::new(HashMap::new()),
4080            mcp_executor: Arc::new(car_engine::McpToolExecutor::new()),
4081            connectors: std::sync::OnceLock::new(),
4082            connectors_loaded: std::sync::atomic::AtomicBool::new(false),
4083            channel_supervisor: std::sync::OnceLock::new(),
4084            durable_tasks: Mutex::new(tokio::task::JoinSet::new()),
4085            auth_completion_owner_id: uuid::Uuid::new_v4().simple().to_string(),
4086            auth_token: std::sync::OnceLock::new(),
4087            host_token: std::sync::OnceLock::new(),
4088            mobile_runtime_url: std::sync::OnceLock::new(),
4089            mobile_registration_url: std::sync::OnceLock::new(),
4090            parslee_session: std::sync::OnceLock::new(),
4091            attached_agents: Mutex::new(HashMap::new()),
4092            agent_memgines: Mutex::new(HashMap::new()),
4093            namespace_memgines: Mutex::new(HashMap::new()),
4094            coder_sessions: Mutex::new(HashMap::new()),
4095            coder_disk_gc_base: std::time::Instant::now(),
4096            // `0` IS "just swept": boot ran the same sweep a few lines above,
4097            // and zero seconds have elapsed since the base taken on the line
4098            // above this one. The first `coder.start` is therefore a no-op
4099            // rather than an immediate re-scan.
4100            coder_disk_gc_at: AtomicU64::new(0),
4101            coder_subscribers: Mutex::new(HashMap::new()),
4102            coder_watchers: Mutex::new(HashMap::new()),
4103            coder_watch_notify: std::sync::OnceLock::new(),
4104            coder_discussions: Mutex::new(HashMap::new()),
4105            coder_discussion_slots: Arc::new(tokio::sync::Semaphore::new(
4106                crate::coder::discuss::MAX_OPEN_DISCUSSIONS,
4107            )),
4108            chat_sessions: Mutex::new(HashMap::new()),
4109            peer_guards: Mutex::new(HashMap::new()),
4110            peer_standing: Mutex::new(HashMap::new()),
4111            held_peer_messages: Mutex::new(std::collections::VecDeque::new()),
4112            lan_discovery: std::sync::Mutex::new(None),
4113            peer_identity: std::sync::Mutex::new(None),
4114            peer_trust: car_a2a::peer_auth::PeerTrust::new(),
4115            chat_collectors: Mutex::new(HashMap::new()),
4116            chat_goals: Mutex::new(chat_goals),
4117            runs: Mutex::new(HashMap::new()),
4118            run_resume_liveness: Mutex::new(()),
4119            run_resume_lease: cfg.run_resume_lease,
4120            run_completion_fence_gate: cfg.run_completion_fence_gate,
4121            run_durability_locks: Mutex::new(HashMap::new()),
4122            run_subscribers: Mutex::new(HashMap::new()),
4123            browser_views,
4124            run_store,
4125            journal_failures: cfg.journal_failures,
4126            mcp_url: std::sync::OnceLock::new(),
4127            approval_gate: cfg.approval_gate.unwrap_or_default(),
4128            supervision: Arc::new(crate::supervision::SupervisionRegistry::default()),
4129            approval_ledger: Arc::new(tokio::sync::RwLock::new(approval_ledger)),
4130            // Installed by the daemon binary at startup (see
4131            // `car-server/src/main.rs`), never here: the implementation lives
4132            // in `car-bench`, which depends on this crate.
4133            harness_measurer: std::sync::RwLock::new(None),
4134        })
4135    }
4136
4137    /// Install the in-process harness evaluator `evolution.run`'s
4138    /// `harness_measure` path grades candidates with. Called once at daemon
4139    /// startup; a later call replaces it.
4140    pub fn set_harness_measurer(&self, m: Arc<dyn crate::evolution::HarnessMeasurer>) {
4141        match self.harness_measurer.write() {
4142            Ok(mut guard) => *guard = Some(m),
4143            // Poisoned only if a panic happened while the lock was held.
4144            // Refusing loudly beats leaving `evolution.run` to believe an
4145            // evaluator is installed when none is.
4146            Err(e) => tracing::error!(
4147                error = %e,
4148                "harness_measurer lock poisoned; the in-process harness evaluator was NOT installed"
4149            ),
4150        }
4151    }
4152
4153    /// The installed harness evaluator, if any. Clones the `Arc` out and drops
4154    /// the guard so callers never hold a `std::sync` lock across an await.
4155    pub fn harness_measurer(&self) -> Option<Arc<dyn crate::evolution::HarnessMeasurer>> {
4156        match self.harness_measurer.read() {
4157            Ok(guard) => guard.clone(),
4158            Err(e) => {
4159                tracing::error!(error = %e, "harness_measurer lock poisoned; treating as absent");
4160                None
4161            }
4162        }
4163    }
4164
4165    /// Start one daemon-owned operation and return a connection-scoped result
4166    /// receiver. Dropping the receiver never cancels the operation.
4167    ///
4168    /// The caller awaits the receiver exactly as it would have awaited the
4169    /// operation, so response shape and timing are unchanged for a connection
4170    /// that stays alive. What changes is the failure mode: when the connection
4171    /// goes away, the per-connection `JoinSet` aborts only this waiter, and the
4172    /// operation itself runs to completion on the daemon.
4173    pub async fn spawn_durable_operation<F, E>(
4174        &self,
4175        operation_name: impl Into<String>,
4176        operation: F,
4177    ) -> tokio::sync::oneshot::Receiver<Result<serde_json::Value, E>>
4178    where
4179        F: std::future::Future<Output = Result<serde_json::Value, E>> + Send + 'static,
4180        E: Send + 'static,
4181    {
4182        let (result_tx, result_rx) = tokio::sync::oneshot::channel();
4183        let operation_name = operation_name.into();
4184        let mut tasks = self.durable_tasks.lock().await;
4185        while let Some(result) = tasks.try_join_next() {
4186            if let Err(error) = result {
4187                tracing::warn!(error = %error, "daemon-owned task failed to join");
4188            }
4189        }
4190        tasks.spawn(async move {
4191            let result = operation.await;
4192            if result_tx.send(result).is_err() {
4193                tracing::debug!(
4194                    operation = %operation_name,
4195                    "daemon-owned operation finished after its response waiter closed"
4196                );
4197            }
4198            operation_name
4199        });
4200        result_rx
4201    }
4202
4203    /// Start one fire-and-reconcile daemon-owned operation.
4204    pub async fn spawn_durable_task<F>(&self, operation_name: impl Into<String>, operation: F)
4205    where
4206        F: std::future::Future<Output = ()> + Send + 'static,
4207    {
4208        let operation_name = operation_name.into();
4209        let mut tasks = self.durable_tasks.lock().await;
4210        while let Some(result) = tasks.try_join_next() {
4211            if let Err(error) = result {
4212                tracing::warn!(error = %error, "daemon-owned task failed to join");
4213            }
4214        }
4215        tasks.spawn(async move {
4216            operation.await;
4217            operation_name
4218        });
4219    }
4220
4221    /// Enable the per-launch auth handshake. After this call, every
4222    /// new WS connection must call `session.auth` with `token` as
4223    /// the first frame; otherwise the connection is closed. Called
4224    /// by `car-server` at startup unless `--no-auth` is set
4225    /// (default flipped 2026-05); embedders supply their own token
4226    /// if they want the same posture. Returns `Err(token)` when
4227    /// auth was already installed.
4228    pub fn install_auth_token(&self, token: String) -> Result<(), String> {
4229        self.auth_token.set(token)
4230    }
4231
4232    /// Install the per-launch host token (Parslee-ai/car#254). A
4233    /// session that later presents this via `session.auth { host_token }`
4234    /// is granted the host-management role (`ClientSession::is_host`),
4235    /// which `authorize_run_access` requires for cross-agent run-trace
4236    /// reads. Set by `car-server` at startup (mints + writes the `0600`
4237    /// `host-token` file) unless `--no-auth` is set. Returns `Err(token)`
4238    /// when a host token was already installed.
4239    pub fn install_host_token(&self, token: String) -> Result<(), String> {
4240        self.host_token.set(token)
4241    }
4242
4243    pub fn install_mobile_runtime_url(&self, url: String) -> Result<(), String> {
4244        self.mobile_runtime_url.set(url)
4245    }
4246
4247    pub fn install_mobile_registration_url(&self, url: String) -> Result<(), String> {
4248        self.mobile_registration_url.set(url)
4249    }
4250
4251    pub fn install_parslee_session(
4252        &self,
4253        session: crate::parslee_auth::ParsleeSession,
4254    ) -> Result<(), crate::parslee_auth::ParsleeSession> {
4255        self.parslee_session.set(session)
4256    }
4257
4258    /// Install the runtime channel supervisor (Units 1/2/3). Called by
4259    /// `car-server` at boot right after `spawn_channel_pollers` builds it, so
4260    /// the host-gated `messaging.config.set` handler can reach it via
4261    /// `ServerState` to spawn a channel's watcher on enable. Idempotent on the
4262    /// first call; a second install is rejected (returns the supervisor back).
4263    pub fn install_channel_supervisor(
4264        &self,
4265        supervisor: Arc<crate::channel_supervisor::ChannelSupervisor>,
4266    ) -> Result<(), Arc<crate::channel_supervisor::ChannelSupervisor>> {
4267        self.channel_supervisor.set(supervisor)
4268    }
4269
4270    /// Install the bound MCP URL after car-server's listener is up.
4271    /// Idempotent on the first call; subsequent calls are accepted
4272    /// silently (matches the supervisor / a2a_dispatcher install
4273    /// idiom). Returns `Err(())` when an MCP URL was already
4274    /// installed — embedders should treat this as "another
4275    /// component beat us to it" and use whichever value is now set.
4276    pub fn install_mcp_url(&self, url: String) -> Result<(), String> {
4277        self.mcp_url.set(url)
4278    }
4279
4280    /// Lazy-initialize and return the remote MCP connector manager,
4281    /// backed by `~/.car/connectors.json` over the shared
4282    /// [`mcp_executor`](Self::mcp_executor). Construction is sync and
4283    /// touches no network — call [`ensure_connectors_loaded`](crate::session::ServerState::ensure_connectors_loaded) to dial
4284    /// persisted connectors.
4285    pub fn connectors(&self) -> Arc<car_connectors::ConnectorManager> {
4286        self.connectors
4287            .get_or_init(|| {
4288                let mgr = car_connectors::ConnectorManager::new(self.mcp_executor.clone())
4289                    .unwrap_or_else(|_| {
4290                        // HOME unresolved (rare): fall back to a CWD-relative
4291                        // manifest so connectors still work this session.
4292                        car_connectors::ConnectorManager::with_path(
4293                            self.mcp_executor.clone(),
4294                            std::path::PathBuf::from("connectors.json"),
4295                        )
4296                    });
4297                Arc::new(mgr)
4298            })
4299            .clone()
4300    }
4301
4302    /// Load persisted connectors and dial them, **once** per process.
4303    /// Idempotent: the first caller does the work; later callers return
4304    /// immediately. `car-server` calls this at boot so connector tools
4305    /// are live before clients connect; the `connectors.*` handlers call
4306    /// it too, so an embedder that skips the boot call still gets a
4307    /// lazy load on first use. Enabled tools discovered here are
4308    /// registered into any already-open sessions.
4309    pub async fn ensure_connectors_loaded(&self) {
4310        use std::sync::atomic::Ordering;
4311        if self.connectors_loaded.swap(true, Ordering::SeqCst) {
4312            return;
4313        }
4314        let mgr = self.connectors();
4315        match mgr.load_and_connect().await {
4316            Ok(entries) => self.register_connector_entries(&entries).await,
4317            Err(e) => tracing::warn!("connector load failed: {e}"),
4318        }
4319    }
4320
4321    /// Register connector [`car_engine::ToolEntry`]s into every currently-open
4322    /// session's runtime registry, so a tool enabled mid-session
4323    /// becomes visible without a reconnect. New sessions pick up the
4324    /// enabled set in [`create_session`](crate::session::ServerState::create_session).
4325    pub async fn register_connector_entries(&self, entries: &[car_engine::ToolEntry]) {
4326        if entries.is_empty() {
4327            return;
4328        }
4329        let sessions: Vec<Arc<ClientSession>> =
4330            self.sessions.lock().await.values().cloned().collect();
4331        for session in sessions {
4332            for entry in entries {
4333                session.runtime.register_tool_entry(entry.clone()).await;
4334            }
4335        }
4336    }
4337
4338    /// Unregister connector tools (by canonical name) from every open
4339    /// session's runtime, so a disabled or removed connector's tools
4340    /// stop being visible to the model and accepted by the validator.
4341    pub async fn unregister_connector_tools(&self, canonical_names: &[String]) {
4342        if canonical_names.is_empty() {
4343            return;
4344        }
4345        let sessions: Vec<Arc<ClientSession>> =
4346            self.sessions.lock().await.values().cloned().collect();
4347        for session in sessions {
4348            for name in canonical_names {
4349                session.runtime.unregister_tool(name).await;
4350            }
4351        }
4352    }
4353
4354    /// Whether any currently connected WS client holds the host-management
4355    /// role — authenticated via `session.auth { host_token }`
4356    /// (`ClientSession::is_host`), NOT the same thing as `host.subscribe`
4357    /// membership (see the doc comment on the auth check this mirrors,
4358    /// around `handle_session_auth`). In practice: is a CarHost / Command
4359    /// Deck app connected right now, so its browser drawer is a real
4360    /// visible surface?
4361    ///
4362    /// Scans the (small) live session set fresh on every call — no caching,
4363    /// so this is always current. The deciding signal for Task 7's
4364    /// headless/headed browser-launch default and its sign-in host-gone
4365    /// fallback (`assistant::browser_tools::HostConnectivity`).
4366    pub async fn any_host_connected(&self) -> bool {
4367        self.sessions
4368            .lock()
4369            .await
4370            .values()
4371            .any(|s| s.is_host.load(std::sync::atomic::Ordering::Acquire))
4372    }
4373
4374    /// Lazy-initialize and return the agent supervisor. The first
4375    /// call constructs a [`car_registry::supervisor::Supervisor`] backed by
4376    /// `~/.car/agents.json` + `~/.car/logs/`. Embedders that need a
4377    /// non-default location should call
4378    /// [`ServerState::install_supervisor`] before any handler runs.
4379    ///
4380    /// In observer mode (set via [`install_observer_manifest`](crate::session::ServerState::install_observer_manifest)),
4381    /// returns a clear error mentioning the manifest path the
4382    /// primary daemon owns. This prevents the second daemon from
4383    /// re-attempting `user_default()` (which would also fail with
4384    /// `AlreadyRunning`) on every WS call, and gives mutation
4385    /// handlers a stable refusal path. Read-only handlers
4386    /// (`agents.list`, `agents.health`) should call
4387    /// [`Self::observer_manifest_path`] first and fall back to
4388    /// [`car_registry::supervisor::Supervisor::list_from_manifest`] /
4389    /// `health_from_manifest` when set. Closes
4390    /// Parslee-ai/car-releases#44.
4391    /// The declarative-agent registry, lazy-initialized on first use.
4392    pub fn declagents(&self) -> Result<Arc<car_registry::declarative::DeclRegistry>, String> {
4393        if let Some(r) = self.declagents.get() {
4394            return Ok(r.clone());
4395        }
4396        let r = Arc::new(car_registry::declarative::DeclRegistry::user_default()?);
4397        let _ = self.declagents.set(r);
4398        Ok(self.declagents.get().expect("set or pre-existing").clone())
4399    }
4400
4401    pub fn routing(&self) -> Result<Arc<car_registry::routing::RoutingStore>, String> {
4402        if let Some(r) = self.routing.get() {
4403            return Ok(r.clone());
4404        }
4405        let r = Arc::new(car_registry::routing::RoutingStore::user_default()?);
4406        let _ = self.routing.set(r);
4407        Ok(self.routing.get().expect("set or pre-existing").clone())
4408    }
4409
4410    pub fn supervisor(&self) -> Result<Arc<car_registry::supervisor::Supervisor>, String> {
4411        if let Some(s) = self.supervisor.get() {
4412            return Ok(s.clone());
4413        }
4414        if let Some(p) = self.observer_manifest_path.get() {
4415            return Err(format!(
4416                "this car-server is observe-only — another car-server process \
4417                 holds the supervisor lock for {}. Mutations refuse here; route \
4418                 them to the primary daemon, or stop the other car-server first.",
4419                p.display()
4420            ));
4421        }
4422        let s = car_registry::supervisor::Supervisor::user_default()
4423            .map(Arc::new)
4424            .map_err(|e| e.to_string())?;
4425        // OnceLock::set returns the original arg back on collision —
4426        // a concurrent caller racing through user_default. Take
4427        // whichever wins.
4428        let _ = self.supervisor.set(s);
4429        Ok(self.supervisor.get().expect("set or pre-existing").clone())
4430    }
4431
4432    /// Replace the lazy default with a caller-supplied supervisor.
4433    /// Returns `Err(())` when a supervisor was already installed.
4434    /// Used by the standalone `car-server` binary to call
4435    /// `start_all()` on a known-good handle without paying the
4436    /// lazy-init lookup cost.
4437    pub fn install_supervisor(
4438        &self,
4439        supervisor: Arc<car_registry::supervisor::Supervisor>,
4440    ) -> Result<(), Arc<car_registry::supervisor::Supervisor>> {
4441        self.supervisor.set(supervisor)
4442    }
4443
4444    /// Non-acquiring read of the currently-installed supervisor.
4445    /// Unlike [`supervisor`](Self::supervisor), this does NOT lazy-
4446    /// init via `user_default()` — it returns `None` instead of
4447    /// constructing a fresh `Supervisor` and acquiring the
4448    /// `<manifest>.lock` as a side effect. Use this from read-only
4449    /// metadata paths (`host.subscribe` identity, status surfaces)
4450    /// where causing lock acquisition on observation would be a
4451    /// Heisenberg subscribe — the act of asking "do you own the
4452    /// lock?" must not be the act of taking it.
4453    pub fn supervisor_if_installed(&self) -> Option<Arc<car_registry::supervisor::Supervisor>> {
4454        self.supervisor.get().cloned()
4455    }
4456
4457    /// Mark this daemon as *observing* a manifest owned by another
4458    /// car-server process. After this call, `supervisor()` returns
4459    /// an "observe-only" error and read-only handlers
4460    /// (`agents.list`, `agents.health`) fall back to the static
4461    /// `Supervisor::list_from_manifest` / `health_from_manifest`
4462    /// paths. Idempotent — subsequent calls with the same path are
4463    /// no-ops; a different path returns `Err(())`. Closes
4464    /// Parslee-ai/car-releases#44.
4465    pub fn install_observer_manifest(&self, path: PathBuf) -> Result<(), PathBuf> {
4466        self.observer_manifest_path.set(path)
4467    }
4468
4469    /// Path of the manifest this daemon is observing but not
4470    /// supervising. `None` when this daemon owns the supervisor
4471    /// (the normal case) or when no manifest is configured at all
4472    /// (no `HOME`, embedder didn't install one).
4473    pub fn observer_manifest_path(&self) -> Option<&PathBuf> {
4474        self.observer_manifest_path.get()
4475    }
4476
4477    /// Lazy-initialize and return the in-core A2A dispatcher. The
4478    /// first call constructs an [`car_a2a::A2aDispatcher`] from
4479    /// either the embedder's overrides (set via
4480    /// [`ServerStateConfig::with_a2a_runtime`] / `with_a2a_store` /
4481    /// `with_a2a_card_source`) or sensible defaults: a fresh
4482    /// `Runtime` with `register_agent_basics` registered, an
4483    /// `InMemoryTaskStore`, and a card built from the runtime's
4484    /// tool schemas advertising `ws://127.0.0.1:9100/` as the
4485    /// public URL. Closes Parslee-ai/car-releases#28.
4486    pub async fn a2a_dispatcher(&self) -> Arc<car_a2a::A2aDispatcher> {
4487        if let Some(d) = self.a2a_dispatcher.get() {
4488            return d.clone();
4489        }
4490
4491        // Embedder overrides take precedence; fall back to defaults
4492        // for each slot independently (so an embedder that only
4493        // wants a custom card can leave the runtime + store at
4494        // defaults). `Mutex::take()` consumes the slot so the
4495        // defaults aren't reconstructed on a racing init that loses
4496        // the OnceLock::set call below.
4497        let runtime = self
4498            .a2a_runtime
4499            .lock()
4500            .expect("a2a_runtime mutex poisoned")
4501            .take();
4502        let runtime = match runtime {
4503            Some(r) => r,
4504            None => {
4505                let r = Arc::new(car_engine::Runtime::new());
4506                r.register_agent_basics().await;
4507                r
4508            }
4509        };
4510
4511        let store = self
4512            .a2a_store
4513            .lock()
4514            .expect("a2a_store mutex poisoned")
4515            .take()
4516            .unwrap_or_else(|| Arc::new(car_a2a::InMemoryTaskStore::new()));
4517
4518        let card_source = self
4519            .a2a_card_source
4520            .lock()
4521            .expect("a2a_card_source mutex poisoned")
4522            .take();
4523        let card_source = match card_source {
4524            Some(c) => c,
4525            None => {
4526                let card = car_a2a::build_default_agent_card(
4527                    &runtime,
4528                    car_a2a::AgentCardConfig::minimal(
4529                        "Common Agent Runtime",
4530                        "Embedded CAR daemon — A2A v1.0 reachable over WebSocket JSON-RPC.",
4531                        "ws://127.0.0.1:9100/",
4532                        car_a2a::AgentProvider {
4533                            organization: "Parslee".into(),
4534                            url: Some("https://github.com/Parslee-ai/car".into()),
4535                        },
4536                    ),
4537                )
4538                .await;
4539                Arc::new(move || card.clone()) as Arc<car_a2a::AgentCardSource>
4540            }
4541        };
4542
4543        let dispatcher = Arc::new(car_a2a::A2aDispatcher::new(runtime, store, card_source));
4544        // OnceLock::set returns Err on race — accept whichever
4545        // dispatcher won and clone-return that one.
4546        let _ = self.a2a_dispatcher.set(dispatcher);
4547        self.a2a_dispatcher
4548            .get()
4549            .expect("a2a_dispatcher set or pre-existing")
4550            .clone()
4551    }
4552
4553    /// Record the start of a run and return its [`RunMeta`]. The
4554    /// `run_id` is minted by the caller (the `runs.start` handler)
4555    /// so it can set `session.current_run_id` to the same value
4556    /// **before** responding (KTD3). Idempotent collision on an
4557    /// already-present `run_id` is treated as a fresh insert (uuids
4558    /// don't collide in practice; if one did, the latest start wins).
4559    ///
4560    /// U3: also writes the `RunStarted` line to the disk store + creates
4561    /// the run file, so the run is on disk from the first record (REPLAY
4562    /// survives a restart even before any turn lands). Disk failures are
4563    /// logged, never fatal — the in-memory registry is the live path.
4564    /// Fan one `runs.trace.event` out to every live subscriber of
4565    /// `run_id` (agent run tracing, U4). Called by the lifecycle methods
4566    /// **while they hold the `runs` lock** so append-and-notify is
4567    /// serialized with subscribe's snapshot-and-register (invariant #1).
4568    ///
4569    /// Each push is a non-blocking `try_send` onto the subscriber's
4570    /// bounded channel — the WS socket is written only by that
4571    /// subscriber's dedicated drain task, never here (invariant #2). A
4572    /// full channel drops the event (the slow-subscriber case); the
4573    /// client detects the cursor gap and re-subscribes (R8). Takes the
4574    /// already-acquired subscribers guard so the caller controls the lock
4575    /// scope and the `runs` → `run_subscribers` order.
4576    fn fanout_locked(
4577        subscribers: &HashMap<(String, String), crate::host::RunTraceSubscriber>,
4578        run_id: &str,
4579        agent_id: &str,
4580        record: car_proto::RunRecord,
4581        cursor: usize,
4582        status: car_proto::RunLiveStatus,
4583    ) {
4584        for ((sub_run, _client), sub) in subscribers.iter() {
4585            if sub_run != run_id {
4586                continue;
4587            }
4588            let event = car_proto::RunTraceEvent {
4589                run_id: run_id.to_string(),
4590                agent_id: agent_id.to_string(),
4591                record: record.clone(),
4592                cursor,
4593                status,
4594            };
4595            // Best-effort: a wedged subscriber's full channel drops the
4596            // event rather than stalling the producer. Logged at debug —
4597            // the client backfills via re-subscribe.
4598            if !sub.push(event) {
4599                tracing::debug!(
4600                    run_id,
4601                    "run-trace: dropped event for slow subscriber (channel full)"
4602                );
4603            }
4604        }
4605    }
4606
4607    /// Atomically reserve a run id/idempotency key across every live session.
4608    /// A per-run durability lock serializes the disk absence check with the
4609    /// final process-wide map insert. The global `runs` lock is held only for
4610    /// the two bounded map checks, never for filesystem enumeration/replay.
4611    pub async fn reserve_run(&self, meta: RunMeta) -> Result<RunReservation, String> {
4612        fn existing_reservation(
4613            existing: &RunMeta,
4614            requested: &RunMeta,
4615        ) -> Result<RunReservation, String> {
4616            if existing.client_id != requested.client_id {
4617                return Err(format!(
4618                    "{} run `{}` belongs to client_id `{}`",
4619                    car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX,
4620                    requested.run_id,
4621                    existing.client_id
4622                ));
4623            }
4624            if existing.agent_id != requested.agent_id {
4625                return Err(format!(
4626                    "{} run `{}` was retried under a different agent",
4627                    car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX,
4628                    requested.run_id
4629                ));
4630            }
4631            let changed_fields = [
4632                (existing.intent != requested.intent).then_some("intent"),
4633                (existing.outcome_description != requested.outcome_description)
4634                    .then_some("outcome_description"),
4635            ]
4636            .into_iter()
4637            .flatten()
4638            .collect::<Vec<_>>();
4639            if !changed_fields.is_empty() {
4640                return Err(format!(
4641                    "{} idempotency conflict for run `{}`: retry changed occurrence-defining {}",
4642                    car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX,
4643                    requested.run_id,
4644                    changed_fields.join(" and ")
4645                ));
4646            }
4647            Ok(RunReservation::Existing(existing.clone()))
4648        }
4649
4650        let durability_lock = self.run_durability_lock(&meta.run_id).await;
4651        let _durability_guard = durability_lock.lock().await;
4652        {
4653            let runs = self.runs.lock().await;
4654            if let Some(existing) = runs.get(&meta.run_id) {
4655                return existing_reservation(existing, &meta);
4656            }
4657        }
4658        let store = self.run_store.clone();
4659        let durable_run_id = meta.run_id.clone();
4660        let durable = tokio::task::spawn_blocking(move || store.run_started(&durable_run_id))
4661            .await
4662            .map_err(|error| format!("RunStarted reservation read task failed: {error}"))?
4663            .map_err(|error| format!("RunStarted reservation read failed: {error}"))?;
4664        if let Some(started) = durable {
4665            return Err(format!(
4666                "{} persisted run `{}` belongs to client_id `{}` and cannot be adopted",
4667                car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX,
4668                meta.run_id,
4669                started.client_id.as_deref().unwrap_or("historical/unknown")
4670            ));
4671        }
4672        let mut runs = self.runs.lock().await;
4673        if let Some(existing) = runs.get(&meta.run_id) {
4674            return existing_reservation(existing, &meta);
4675        }
4676        runs.insert(meta.run_id.clone(), meta);
4677        Ok(RunReservation::New)
4678    }
4679
4680    /// Release a newly reserved candidate when `runs.start` rejects before it
4681    /// binds or persists that candidate. The exact pristine metadata and the
4682    /// absent RunStore boundary are both re-checked under the per-run durability
4683    /// lock. This deliberately does not touch `current_run_id` or the journal
4684    /// binding, which may still belong to an older live run on this session.
4685    pub(crate) async fn release_unpersisted_run_reservation(
4686        &self,
4687        session: &ClientSession,
4688        expected: &RunMeta,
4689    ) -> Result<(), String> {
4690        if expected.client_id != session.client_id
4691            || expected.start_committed
4692            || expected.termination.is_some()
4693            || expected.ended_at.is_some()
4694            || !expected.turns.is_empty()
4695            || expected.pending_terminal.is_some()
4696        {
4697            return Err(format!(
4698                "run `{}` is not a pristine reservation owned by this session",
4699                expected.run_id
4700            ));
4701        }
4702        let still_expected = |existing: &RunMeta| {
4703            existing.run_id == expected.run_id
4704                && existing.client_id == expected.client_id
4705                && existing.agent_id == expected.agent_id
4706                && existing.intent == expected.intent
4707                && existing.outcome_description == expected.outcome_description
4708                && existing.started_at == expected.started_at
4709                && existing.durability_generation == expected.durability_generation
4710                && !existing.start_committed
4711                && existing.termination.is_none()
4712                && existing.ended_at.is_none()
4713                && existing.turns.is_empty()
4714                && existing.pending_terminal.is_none()
4715        };
4716        let durability_lock = self.run_durability_lock(&expected.run_id).await;
4717        let _durability_guard = durability_lock.lock().await;
4718        {
4719            let runs = self.runs.lock().await;
4720            let existing = runs
4721                .get(&expected.run_id)
4722                .ok_or_else(|| format!("reserved run `{}` is absent", expected.run_id))?;
4723            if !still_expected(existing) {
4724                return Err(format!(
4725                    "reserved run `{}` changed before pre-start release",
4726                    expected.run_id
4727                ));
4728            }
4729        }
4730        let store = self.run_store.clone();
4731        let durable_run_id = expected.run_id.clone();
4732        if tokio::task::spawn_blocking(move || store.run_started(&durable_run_id))
4733            .await
4734            .map_err(|error| format!("RunStarted release read task failed: {error}"))?
4735            .map_err(|error| format!("RunStarted release read failed: {error}"))?
4736            .is_some()
4737        {
4738            return Err(format!(
4739                "reserved run `{}` reached durable storage and cannot be released",
4740                expected.run_id
4741            ));
4742        }
4743        let mut runs = self.runs.lock().await;
4744        let existing = runs
4745            .get(&expected.run_id)
4746            .ok_or_else(|| format!("reserved run `{}` is absent", expected.run_id))?;
4747        if !still_expected(existing) {
4748            return Err(format!(
4749                "reserved run `{}` changed during pre-start release",
4750                expected.run_id
4751            ));
4752        }
4753        runs.remove(&expected.run_id);
4754        Ok(())
4755    }
4756
4757    pub async fn persist_run_start(&self, run_id: &str) -> Result<car_proto::RunStarted, String> {
4758        let durability_lock = self.run_durability_lock(run_id).await;
4759        let _durability_guard = durability_lock.lock().await;
4760        let started = {
4761            let runs = self.runs.lock().await;
4762            let meta = runs
4763                .get(run_id)
4764                .ok_or_else(|| format!("unknown reserved run_id `{run_id}`"))?;
4765            car_proto::RunStarted {
4766                run_id: meta.run_id.clone(),
4767                client_id: Some(meta.client_id.clone()),
4768                agent_id: meta.agent_id.clone(),
4769                intent: meta.intent.clone(),
4770                outcome_description: meta.outcome_description.clone(),
4771                started_at: meta.started_at,
4772            }
4773        };
4774        let store = self.run_store.clone();
4775        let durable_started = started.clone();
4776        tokio::task::spawn_blocking(move || store.write_started(&durable_started))
4777            .await
4778            .map_err(|error| format!("RunStarted durability task failed: {error}"))?
4779            .map_err(|error| format!("RunStarted durability failed: {error}"))?;
4780        Ok(started)
4781    }
4782
4783    pub async fn commit_run_start(&self, run_id: &str) -> Result<(), String> {
4784        let mut runs = self.runs.lock().await;
4785        let meta = runs
4786            .get_mut(run_id)
4787            .ok_or_else(|| format!("unknown reserved run_id `{run_id}`"))?;
4788        if meta.start_committed {
4789            return Ok(());
4790        }
4791        meta.start_committed = true;
4792        meta.durability_generation = meta.durability_generation.wrapping_add(1);
4793        let started = car_proto::RunStarted {
4794            run_id: meta.run_id.clone(),
4795            client_id: Some(meta.client_id.clone()),
4796            agent_id: meta.agent_id.clone(),
4797            intent: meta.intent.clone(),
4798            outcome_description: meta.outcome_description.clone(),
4799            started_at: meta.started_at,
4800        };
4801        let subs = self.run_subscribers.lock().await;
4802        Self::fanout_locked(
4803            &subs,
4804            &started.run_id,
4805            &started.agent_id,
4806            car_proto::RunRecord::Started(started.clone()),
4807            0,
4808            car_proto::RunLiveStatus::InProgress,
4809        );
4810        Ok(())
4811    }
4812
4813    /// Resolve a `runs.start` transaction whose acknowledgement failed before
4814    /// the owning socket disconnected. Returns `true` when no durable start
4815    /// boundary existed and the reservation was released. Returns `false`
4816    /// after reconciling an exact durable `RunStarted`, which the disconnect
4817    /// sweep must then close as one historical `Incomplete` occurrence.
4818    pub(crate) async fn reconcile_or_release_unacknowledged_start(
4819        &self,
4820        session: &ClientSession,
4821        run_id: &str,
4822    ) -> Result<bool, String> {
4823        let durability_lock = self.run_durability_lock(run_id).await;
4824        let _durability_guard = durability_lock.lock().await;
4825        let expected = {
4826            let runs = self.runs.lock().await;
4827            let meta = runs
4828                .get(run_id)
4829                .ok_or_else(|| format!("unknown reserved run_id `{run_id}`"))?;
4830            if meta.client_id != session.client_id {
4831                return Err(format!(
4832                    "{} run `{run_id}` belongs to client_id `{}`",
4833                    car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX,
4834                    meta.client_id
4835                ));
4836            }
4837            if meta.start_committed {
4838                return Ok(false);
4839            }
4840            if meta.is_terminal() || meta.pending_terminal.is_some() {
4841                return Err(format!(
4842                    "uncommitted run `{run_id}` unexpectedly has terminal state"
4843                ));
4844            }
4845            car_proto::RunStarted {
4846                run_id: meta.run_id.clone(),
4847                client_id: Some(meta.client_id.clone()),
4848                agent_id: meta.agent_id.clone(),
4849                intent: meta.intent.clone(),
4850                outcome_description: meta.outcome_description.clone(),
4851                started_at: meta.started_at,
4852            }
4853        };
4854
4855        let store = self.run_store.clone();
4856        let durable_run_id = run_id.to_string();
4857        let durable = tokio::task::spawn_blocking(move || store.run_started(&durable_run_id))
4858            .await
4859            .map_err(|error| format!("RunStarted recovery read task failed: {error}"))?
4860            .map_err(|error| format!("RunStarted recovery read failed: {error}"))?;
4861        match durable {
4862            Some(durable) => {
4863                if durable != expected {
4864                    return Err(format!(
4865                        "durable RunStarted identity for `{run_id}` does not match its live reservation"
4866                    ));
4867                }
4868                // Re-run both exact idempotent durability barriers. The first
4869                // retry turns a prior write-without-fsync into an acknowledged
4870                // RunStore boundary; the second either appends RunStarted or
4871                // fsyncs the exact journal row already pending in memory.
4872                let store = self.run_store.clone();
4873                let durable_expected = expected.clone();
4874                tokio::task::spawn_blocking(move || store.write_started(&durable_expected))
4875                    .await
4876                    .map_err(|error| {
4877                        format!("RunStarted recovery durability task failed: {error}")
4878                    })?
4879                    .map_err(|error| format!("RunStarted recovery durability failed: {error}"))?;
4880                session.append_run_started_event(&expected).await?;
4881                self.commit_run_start(run_id).await?;
4882                Ok(false)
4883            }
4884            None => {
4885                // `runs.start` orders RunStore before journal append, so an
4886                // absent RunStore boundary proves no lifecycle row was ever
4887                // admitted to the journal. Remove only the zero-byte file an
4888                // open-before-write failure may have left behind.
4889                let store = self.run_store.clone();
4890                let durable_expected = expected.clone();
4891                tokio::task::spawn_blocking(move || {
4892                    store.rollback_empty_run_start(&durable_expected)
4893                })
4894                .await
4895                .map_err(|error| format!("empty RunStarted rollback task failed: {error}"))?
4896                .map_err(|error| format!("empty RunStarted rollback failed: {error}"))?;
4897                {
4898                    let mut runs = self.runs.lock().await;
4899                    let meta = runs.get(run_id).ok_or_else(|| {
4900                        format!("reserved run `{run_id}` disappeared during rollback")
4901                    })?;
4902                    let still_expected = meta.client_id == session.client_id
4903                        && !meta.start_committed
4904                        && !meta.is_terminal()
4905                        && meta.pending_terminal.is_none()
4906                        && meta.agent_id == expected.agent_id
4907                        && meta.intent == expected.intent
4908                        && meta.outcome_description == expected.outcome_description
4909                        && meta.started_at == expected.started_at;
4910                    if !still_expected {
4911                        return Err(format!(
4912                            "reserved run `{run_id}` changed during empty-start rollback"
4913                        ));
4914                    }
4915                    runs.remove(run_id);
4916                }
4917                session.clear_run_journal_binding(run_id).await?;
4918                let mut current = session.current_run_id.lock().await;
4919                if current.as_deref() == Some(run_id) {
4920                    *current = None;
4921                }
4922                Ok(true)
4923            }
4924        }
4925    }
4926
4927    /// Compatibility/internal path for CAR-owned in-process callers that do
4928    /// not expose `runs.start`. The public RPC uses reserve/persist/journal/
4929    /// commit explicitly so its acknowledgement includes both durable stores.
4930    pub async fn start_run(&self, meta: RunMeta) -> Result<(), String> {
4931        let run_id = meta.run_id.clone();
4932        self.reserve_run(meta).await?;
4933        self.persist_run_start(&run_id).await?;
4934        self.commit_run_start(&run_id).await
4935    }
4936
4937    /// Make a run terminal with a harness-reported outcome
4938    /// (`runs.complete`). Returns `Err` if the `run_id` is unknown or
4939    /// already terminal — the handler maps that to a JSON-RPC error so
4940    /// a double-complete or stale id is visible, not silently swallowed.
4941    ///
4942    /// U3: also appends the terminal `RunEnded` line to the run's JSONL
4943    /// file so REPLAY (U5) reports the right status (Completed) after a
4944    /// restart. The disk write happens after the lock is released; disk
4945    /// failures are logged, never fatal.
4946    pub async fn prepare_run_completion(
4947        &self,
4948        run_id: &str,
4949        termination: car_proto::RunTermination,
4950    ) -> Result<car_proto::RunEnded, String> {
4951        self.prepare_run_completion_for_active_owner(run_id, termination, None)
4952            .await
4953    }
4954
4955    pub(crate) async fn prepare_run_completion_for_active_owner(
4956        &self,
4957        run_id: &str,
4958        termination: car_proto::RunTermination,
4959        expected_active_client_id: Option<&str>,
4960    ) -> Result<car_proto::RunEnded, String> {
4961        let completion_digest = run_completion_digest(&termination)?;
4962        if expected_active_client_id.is_some() {
4963            if let Some(gate) = &self.run_completion_fence_gate {
4964                gate.wait_if_armed().await;
4965            }
4966        }
4967        let durability_lock = self.run_durability_lock(run_id).await;
4968        let _durability_guard = durability_lock.lock().await;
4969        let ended = {
4970            let mut runs = self.runs.lock().await;
4971            let meta = runs
4972                .get_mut(run_id)
4973                .ok_or_else(|| format!("unknown run_id `{run_id}`"))?;
4974            if let Some(error) = &meta.trace_corruption {
4975                return Err(error.clone());
4976            }
4977            if !meta.start_committed {
4978                return Err(format!("run `{run_id}` start is not durably committed"));
4979            }
4980            if expected_active_client_id.is_some_and(|expected| meta.active_client_id != expected) {
4981                return Err(format!(
4982                    "run `{run_id}` active owner changed before terminalization"
4983                ));
4984            }
4985            if let Some(requested) = &meta.cancellation_pending {
4986                let matches_cancel = matches!(
4987                    &termination,
4988                    car_proto::RunTermination::Cancelled { cancellation }
4989                        if cancellation.run_id == requested.run_id
4990                            && cancellation.idempotency_key == requested.idempotency_key
4991                            && cancellation.reason_digest == requested.reason_digest
4992                            && cancellation.principal == requested.principal
4993                            && cancellation.action_id == requested.action_id
4994                            && cancellation.request_id == requested.request_id
4995                );
4996                if !matches_cancel {
4997                    return Err(format!(
4998                        "run `{run_id}` has a durable cancellation request and is quarantined"
4999                    ));
5000                }
5001            }
5002            if let Some(existing) = &meta.termination {
5003                if !same_run_termination(existing, &termination) {
5004                    return Err(format!("run `{run_id}` already has a different terminal"));
5005                }
5006                meta.resume_lease = None;
5007                car_proto::RunEnded {
5008                    run_id: run_id.to_string(),
5009                    client_id: Some(meta.client_id.clone()),
5010                    agent_id: meta.agent_id.clone(),
5011                    termination,
5012                    completion_digest: Some(completion_digest),
5013                    ended_at: meta
5014                        .ended_at
5015                        .ok_or_else(|| "terminal run is missing ended_at".to_string())?,
5016                }
5017            } else if let Some(pending) = &meta.pending_terminal {
5018                if !same_run_termination(&pending.termination, &termination)
5019                    || pending.completion_digest.as_deref() != Some(completion_digest.as_str())
5020                {
5021                    return Err(format!(
5022                        "run `{run_id}` has a different pending terminal transaction"
5023                    ));
5024                }
5025                meta.resume_lease = None;
5026                pending.clone()
5027            } else {
5028                let ended = car_proto::RunEnded {
5029                    run_id: run_id.to_string(),
5030                    client_id: Some(meta.client_id.clone()),
5031                    agent_id: meta.agent_id.clone(),
5032                    termination,
5033                    completion_digest: Some(completion_digest),
5034                    ended_at: chrono::Utc::now(),
5035                };
5036                meta.pending_terminal = Some(ended.clone());
5037                meta.resume_lease = None;
5038                meta.durability_generation = meta.durability_generation.wrapping_add(1);
5039                ended
5040            }
5041        };
5042        let store = self.run_store.clone();
5043        let durable_ended = ended.clone();
5044        let persisted = tokio::task::spawn_blocking(move || store.write_ended(&durable_ended))
5045            .await
5046            .map_err(|error| format!("RunEnded durability task failed: {error}"))?;
5047        if let Err(error) = persisted {
5048            if crate::run_store::is_trace_corruption_error(&error) {
5049                let mut runs = self.runs.lock().await;
5050                let message = self
5051                    .quarantine_run_trace_locked(&mut runs, run_id, error.to_string())
5052                    .await;
5053                return Err(message);
5054            }
5055            return Err(format!("RunEnded durability failed: {error}"));
5056        }
5057        Ok(ended)
5058    }
5059
5060    pub async fn commit_run_completion(&self, ended: &car_proto::RunEnded) -> Result<(), String> {
5061        {
5062            let mut runs = self.runs.lock().await;
5063            let meta = runs
5064                .get_mut(&ended.run_id)
5065                .ok_or_else(|| format!("unknown run_id `{}`", ended.run_id))?;
5066            if let Some(existing) = &meta.termination {
5067                if same_run_termination(existing, &ended.termination)
5068                    && meta.ended_at == Some(ended.ended_at)
5069                {
5070                    return Ok(());
5071                }
5072                return Err(format!(
5073                    "run `{}` already has a different terminal",
5074                    ended.run_id
5075                ));
5076            }
5077            if !meta
5078                .pending_terminal
5079                .as_ref()
5080                .is_some_and(|pending| same_run_ended(pending, ended))
5081            {
5082                return Err(format!(
5083                    "run `{}` terminal commit does not match its prepared transaction",
5084                    ended.run_id
5085                ));
5086            }
5087            meta.termination = Some(ended.termination.clone());
5088            meta.ended_at = Some(ended.ended_at);
5089            meta.pending_terminal = None;
5090            meta.resume_lease = None;
5091            meta.cancellation_pending = None;
5092            meta.cancellation_receipt = None;
5093            meta.durability_generation = meta.durability_generation.wrapping_add(1);
5094            let cursor = meta.turn_cursor();
5095            let status = meta.live_status();
5096            let agent_id = meta.agent_id.clone();
5097            let subs = self.run_subscribers.lock().await;
5098            Self::fanout_locked(
5099                &subs,
5100                &ended.run_id,
5101                &agent_id,
5102                car_proto::RunRecord::Ended(ended.clone()),
5103                cursor,
5104                status,
5105            );
5106        }
5107        // Heap hygiene: the run is now terminal and fully flushed to disk,
5108        // and the terminal `runs.trace.event` has already fanned out to any
5109        // live subscriber. Drop the in-memory per-turn buffer (each entry
5110        // holds a full prompt + CLI output, potentially MBs) so completed
5111        // runs don't pin the heaviest payloads for the daemon's lifetime.
5112        // The lightweight `RunMeta` header (agent_id, status, started_at,
5113        // termination) stays resident so `run_meta`/status lookups still
5114        // work. A late `runs.subscribe` to this terminal run re-sources its
5115        // snapshot from disk (see `subscribe_run`). Only terminal runs are
5116        // cleared; an in-progress run keeps its turns (the live snapshot
5117        // source). `Vec::new()` frees the buffer's capacity, not just its
5118        // length.
5119        self.clear_terminal_run_turns(&ended.run_id).await;
5120        Ok(())
5121    }
5122
5123    pub async fn complete_run(
5124        &self,
5125        run_id: &str,
5126        termination: car_proto::RunTermination,
5127    ) -> Result<car_proto::RunEnded, String> {
5128        let ended = self.prepare_run_completion(run_id, termination).await?;
5129        self.commit_run_completion(&ended).await?;
5130        Ok(ended)
5131    }
5132
5133    /// Commit the body-free request to RunStore and the authenticated opening
5134    /// client's journal under the same per-run durability lock used by
5135    /// terminal completion. Memory changes only after both receipts exist.
5136    pub async fn persist_run_cancellation_requested(
5137        &self,
5138        owner_session: &ClientSession,
5139        requested: &car_proto::RunCancellationRequested,
5140    ) -> Result<(), String> {
5141        let run_id = requested.run_id.as_str();
5142        let durability_lock = self.run_durability_lock(run_id).await;
5143        let _durability_guard = durability_lock.lock().await;
5144        let agent_id = {
5145            let runs = self.runs.lock().await;
5146            let meta = runs
5147                .get(run_id)
5148                .ok_or_else(|| format!("unknown run_id `{run_id}`"))?;
5149            if meta.is_terminal() || meta.pending_terminal.is_some() {
5150                return Err(format!("run `{run_id}` is already terminal"));
5151            }
5152            if meta.client_id != owner_session.client_id || !meta.start_committed {
5153                return Err("run cancellation client/run binding mismatch".into());
5154            }
5155            if let Some(existing) = &meta.cancellation_pending {
5156                if existing != requested {
5157                    return Err(format!(
5158                        "run `{run_id}` already has a different cancellation request"
5159                    ));
5160                }
5161                return Ok(());
5162            }
5163            meta.agent_id.clone()
5164        };
5165        let store = self.run_store.clone();
5166        let durable_agent = agent_id.clone();
5167        let durable_requested = requested.clone();
5168        tokio::task::spawn_blocking(move || {
5169            store.write_cancellation_requested(&durable_agent, &durable_requested)
5170        })
5171        .await
5172        .map_err(|error| format!("cancellation request durability task failed: {error}"))?
5173        .map_err(|error| format!("cancellation request durability failed: {error}"))?;
5174        owner_session
5175            .append_run_cancellation_requested_event(requested)
5176            .await?;
5177        let mut runs = self.runs.lock().await;
5178        let meta = runs
5179            .get_mut(run_id)
5180            .ok_or_else(|| format!("unknown run_id `{run_id}`"))?;
5181        meta.cancellation_pending = Some(requested.clone());
5182        meta.durability_generation = meta.durability_generation.wrapping_add(1);
5183        let cursor = meta.turn_cursor();
5184        let agent_id = meta.agent_id.clone();
5185        let subs = self.run_subscribers.lock().await;
5186        Self::fanout_locked(
5187            &subs,
5188            run_id,
5189            &agent_id,
5190            car_proto::RunRecord::CancellationRequested(requested.clone()),
5191            cursor,
5192            car_proto::RunLiveStatus::CancellationPending,
5193        );
5194        Ok(())
5195    }
5196
5197    pub async fn persist_run_cancellation_result(
5198        &self,
5199        owner_session: &ClientSession,
5200        result: &car_proto::RunCancelResponse,
5201    ) -> Result<(), String> {
5202        let run_id = result.run_id.as_str();
5203        let durability_lock = self.run_durability_lock(run_id).await;
5204        let _durability_guard = durability_lock.lock().await;
5205        let agent_id = {
5206            let runs = self.runs.lock().await;
5207            let meta = runs
5208                .get(run_id)
5209                .ok_or_else(|| format!("unknown run_id `{run_id}`"))?;
5210            if meta.is_terminal() {
5211                return Err(format!("run `{run_id}` is already terminal"));
5212            }
5213            let requested = meta
5214                .cancellation_pending
5215                .as_ref()
5216                .ok_or_else(|| format!("run `{run_id}` has no cancellation request"))?;
5217            if requested.idempotency_key != result.idempotency_key
5218                || requested.reason_digest != result.reason_digest
5219                || requested.principal != result.principal
5220            {
5221                return Err(format!("run `{run_id}` cancellation receipt mismatch"));
5222            }
5223            if let Some(existing) = &meta.cancellation_receipt {
5224                if existing != result {
5225                    return Err(format!(
5226                        "run `{run_id}` has a different cancellation receipt"
5227                    ));
5228                }
5229                return Ok(());
5230            }
5231            meta.agent_id.clone()
5232        };
5233        let store = self.run_store.clone();
5234        let durable_agent = agent_id.clone();
5235        let durable_result = result.clone();
5236        tokio::task::spawn_blocking(move || {
5237            store.write_cancellation_result(&durable_agent, &durable_result)
5238        })
5239        .await
5240        .map_err(|error| format!("cancellation result durability task failed: {error}"))?
5241        .map_err(|error| format!("cancellation result durability failed: {error}"))?;
5242        owner_session
5243            .append_run_cancellation_result_event(result)
5244            .await?;
5245        let mut runs = self.runs.lock().await;
5246        let meta = runs
5247            .get_mut(run_id)
5248            .ok_or_else(|| format!("unknown run_id `{run_id}`"))?;
5249        meta.cancellation_receipt = Some(result.clone());
5250        meta.durability_generation = meta.durability_generation.wrapping_add(1);
5251        let cursor = meta.turn_cursor();
5252        let agent_id = meta.agent_id.clone();
5253        let subs = self.run_subscribers.lock().await;
5254        Self::fanout_locked(
5255            &subs,
5256            run_id,
5257            &agent_id,
5258            car_proto::RunRecord::CancellationResult(result.clone()),
5259            cursor,
5260            car_proto::RunLiveStatus::CancellationPending,
5261        );
5262        Ok(())
5263    }
5264
5265    /// Finish an unconfirmed cancellation after the opening socket has gone
5266    /// away. The durable `RunStarted` client id is the journal authority: a
5267    /// reconnect may request recovery, but it may not redirect the receipt to
5268    /// its own journal. RunStore, that authenticated journal, live state, and
5269    /// subscribers advance as one retryable transaction before success is
5270    /// returned.
5271    pub async fn persist_recovered_run_cancellation_result(
5272        &self,
5273        agent_id: &str,
5274        result: &car_proto::RunCancelResponse,
5275    ) -> Result<(), String> {
5276        if result.status != car_proto::RunCancellationStatus::TerminationUnconfirmed
5277            || result.terminal_digest.is_some()
5278        {
5279            return Err("recovered cancellation receipt must be termination_unconfirmed".into());
5280        }
5281        let run_id = result.run_id.as_str();
5282        let durability_lock = self.run_durability_lock(run_id).await;
5283        let _durability_guard = durability_lock.lock().await;
5284        let store = self.run_store.clone();
5285        let durable_agent = agent_id.to_string();
5286        let durable_run = run_id.to_string();
5287        let records = tokio::task::spawn_blocking(move || {
5288            store.get_run_trace_for_checked(&durable_agent, &durable_run)
5289        })
5290        .await
5291        .map_err(|error| format!("recovered cancellation trace task failed: {error}"))?
5292        .map_err(|error| format!("recovered cancellation trace read failed: {error}"))?
5293        .ok_or_else(|| "recovered cancellation durable owner is unavailable".to_string())?;
5294        let started = records
5295            .iter()
5296            .find_map(|record| match record {
5297                car_proto::RunRecord::Started(started) => Some(started),
5298                _ => None,
5299            })
5300            .ok_or_else(|| "recovered cancellation is missing durable RunStarted".to_string())?;
5301        if started.agent_id != agent_id {
5302            return Err("recovered cancellation durable agent mismatch".into());
5303        }
5304        let client_id = started
5305            .client_id
5306            .as_deref()
5307            .filter(|client_id| {
5308                !client_id.is_empty()
5309                    && !client_id.contains('/')
5310                    && !client_id.contains('\\')
5311                    && *client_id != "."
5312                    && *client_id != ".."
5313            })
5314            .ok_or_else(|| {
5315                "recovered cancellation has no authenticated durable journal owner".to_string()
5316            })?
5317            .to_string();
5318        if records
5319            .iter()
5320            .any(|record| matches!(record, car_proto::RunRecord::Ended(_)))
5321        {
5322            return Err(format!("run `{run_id}` is already terminal"));
5323        }
5324        let requested = records
5325            .iter()
5326            .find_map(|record| match record {
5327                car_proto::RunRecord::CancellationRequested(requested) => Some(requested),
5328                _ => None,
5329            })
5330            .ok_or_else(|| format!("run `{run_id}` has no durable cancellation request"))?;
5331        if requested.receipt_version != result.receipt_version
5332            || requested.run_id != result.run_id
5333            || requested.idempotency_key != result.idempotency_key
5334            || requested.reason_digest != result.reason_digest
5335            || requested.principal != result.principal
5336            || requested.action_id != result.action_id
5337            || requested.request_id != result.request_id
5338        {
5339            return Err(format!(
5340                "run `{run_id}` recovered cancellation receipt mismatch"
5341            ));
5342        }
5343        if let Some(existing) = records.iter().find_map(|record| match record {
5344            car_proto::RunRecord::CancellationResult(existing) => Some(existing),
5345            _ => None,
5346        }) {
5347            if existing != result {
5348                return Err(format!(
5349                    "run `{run_id}` has a different durable cancellation receipt"
5350                ));
5351            }
5352        }
5353        {
5354            let runs = self.runs.lock().await;
5355            if let Some(meta) = runs.get(run_id) {
5356                if meta.agent_id != agent_id
5357                    || meta.client_id != client_id
5358                    || !meta.start_committed
5359                    || meta.is_terminal()
5360                    || meta.cancellation_pending.as_ref() != Some(requested)
5361                {
5362                    return Err(format!(
5363                        "run `{run_id}` live state does not authenticate recovered cancellation"
5364                    ));
5365                }
5366                if meta
5367                    .cancellation_receipt
5368                    .as_ref()
5369                    .is_some_and(|existing| existing != result)
5370                {
5371                    return Err(format!(
5372                        "run `{run_id}` has a different live cancellation receipt"
5373                    ));
5374                }
5375            }
5376        }
5377
5378        let store = self.run_store.clone();
5379        let durable_agent = agent_id.to_string();
5380        let durable_result = result.clone();
5381        tokio::task::spawn_blocking(move || {
5382            store.write_cancellation_result(&durable_agent, &durable_result)
5383        })
5384        .await
5385        .map_err(|error| format!("recovered cancellation durability task failed: {error}"))?
5386        .map_err(|error| format!("recovered cancellation durability failed: {error}"))?;
5387
5388        let journal_dir = self.journal_dir.clone();
5389        let journal_failures = self.journal_failures.clone();
5390        let journal_client = client_id.clone();
5391        let journal_result = result.clone();
5392        let journal_was_new = tokio::task::spawn_blocking(move || {
5393            append_recovered_cancellation_result_journal(
5394                journal_dir,
5395                journal_failures,
5396                journal_client,
5397                journal_result,
5398            )
5399        })
5400        .await
5401        .map_err(|error| format!("recovered cancellation journal task failed: {error}"))??;
5402
5403        let mut runs = self.runs.lock().await;
5404        let (cursor, fanout) = match runs.get_mut(run_id) {
5405            Some(meta) => {
5406                let fanout = meta.cancellation_receipt.is_none();
5407                meta.cancellation_receipt = Some(result.clone());
5408                if fanout {
5409                    meta.durability_generation = meta.durability_generation.wrapping_add(1);
5410                }
5411                (meta.turn_cursor(), fanout)
5412            }
5413            None => (
5414                records
5415                    .iter()
5416                    .filter(|record| matches!(record, car_proto::RunRecord::Turn(_)))
5417                    .count(),
5418                journal_was_new,
5419            ),
5420        };
5421        if fanout {
5422            let subscribers = self.run_subscribers.lock().await;
5423            Self::fanout_locked(
5424                &subscribers,
5425                run_id,
5426                agent_id,
5427                car_proto::RunRecord::CancellationResult(result.clone()),
5428                cursor,
5429                car_proto::RunLiveStatus::CancellationPending,
5430            );
5431        }
5432        Ok(())
5433    }
5434
5435    /// Mark a run `Incomplete` (R5) — used by disconnect cleanup when a
5436    /// harness drops without `runs.complete`. No-op if the run is
5437    /// already terminal (the common healthy-close case where
5438    /// `runs.complete` won the race). Returns `true` if it actually
5439    /// wrote the `Incomplete` marker.
5440    ///
5441    /// U3: on the transition to `Incomplete`, appends the terminal
5442    /// `RunEnded { Incomplete }` line to disk so an orphaned run reports
5443    /// `Incomplete` (not `InProgress`) on REPLAY after a restart. Disk
5444    /// failures are logged, never fatal.
5445    pub async fn prepare_run_incomplete(&self, run_id: &str) -> Option<car_proto::RunEnded> {
5446        self.prepare_run_incomplete_for_active_owner(run_id, None)
5447            .await
5448    }
5449
5450    async fn prepare_run_incomplete_for_active_owner(
5451        &self,
5452        run_id: &str,
5453        expected_active_client_id: Option<&str>,
5454    ) -> Option<car_proto::RunEnded> {
5455        // If a `runs.complete` transaction already prepared its exact outcome,
5456        // disconnect cleanup must finish that transaction, never overwrite it
5457        // with Incomplete.
5458        let termination = self
5459            .runs
5460            .lock()
5461            .await
5462            .get(run_id)
5463            .and_then(|meta| meta.pending_terminal.as_ref())
5464            .map(|ended| ended.termination.clone())
5465            .unwrap_or(car_proto::RunTermination::Incomplete);
5466        match self
5467            .prepare_run_completion_for_active_owner(run_id, termination, expected_active_client_id)
5468            .await
5469        {
5470            Ok(ended) => Some(ended),
5471            Err(error) => {
5472                tracing::error!(run_id, %error, "failed to prepare durable incomplete terminal");
5473                None
5474            }
5475        }
5476    }
5477
5478    pub async fn mark_run_incomplete(&self, run_id: &str) -> Option<car_proto::RunEnded> {
5479        let ended = self.prepare_run_incomplete(run_id).await?;
5480        if let Err(error) = self.commit_run_completion(&ended).await {
5481            tracing::error!(run_id, %error, "failed to commit incomplete terminal");
5482            return None;
5483        }
5484        Some(ended)
5485    }
5486
5487    /// Free the in-memory per-turn buffer of a TERMINAL run, keeping the
5488    /// lightweight `RunMeta` header resident (agent run tracing, heap
5489    /// hygiene follow-up). Called at the end of `complete_run` /
5490    /// `mark_run_incomplete`, AFTER the disk flush and terminal fanout, so
5491    /// the disk store remains the durable source of truth and any late
5492    /// `runs.subscribe` re-sources the snapshot from disk
5493    /// ([`Self::subscribe_run`]).
5494    ///
5495    /// Guards on `is_terminal()` so a same-id reuse that somehow re-opened
5496    /// the run (it shouldn't — ids are uuids) never has live turns dropped.
5497    /// `Vec::new()` releases the buffer's capacity, not just its length —
5498    /// the per-turn payloads (full prompt + CLI output) are the heavy part.
5499    async fn clear_terminal_run_turns(&self, run_id: &str) {
5500        let mut runs = self.runs.lock().await;
5501        if let Some(meta) = runs.get_mut(run_id) {
5502            if meta.is_terminal() {
5503                meta.turns = Vec::new();
5504            }
5505        }
5506    }
5507
5508    /// Non-acquiring read of a run's current metadata (clone). Used by
5509    /// tests and by the U2 recorder to learn a run's owning `agent_id`.
5510    ///
5511    /// NOTE: this clones the ENTIRE `RunMeta`, including its `turns` buffer
5512    /// (full prompts + CLI output — up to the per-run caps). Hot callers
5513    /// that need only the run's header facts must use [`Self::run_header`]
5514    /// instead; cloning the whole buffer per batch RPC is a ~512 MB
5515    /// worst-case copy at this PR's caps (ADV-2).
5516    pub async fn run_meta(&self, run_id: &str) -> Option<RunMeta> {
5517        self.runs.lock().await.get(run_id).cloned()
5518    }
5519
5520    /// Lightweight, non-cloning read of a run's header facts under the
5521    /// `runs` lock: `(agent_id, terminal, turn_count, trace_corruption)`.
5522    /// The owning agent id and optional error are the only heap allocations;
5523    /// This is what `handle_runs_record_turns` needs — owner for the write
5524    /// authz check, terminality for the distinguishable `run_terminal` drop,
5525    /// and the current turn count for the (fast-path) ceiling pre-check —
5526    /// without the `run_meta` deep clone of every recorded turn's payload.
5527    pub async fn run_header(&self, run_id: &str) -> Option<(String, bool, usize, Option<String>)> {
5528        self.runs.lock().await.get(run_id).map(|m| {
5529            (
5530                m.agent_id.clone(),
5531                m.is_terminal(),
5532                m.turns.len(),
5533                m.trace_corruption.clone(),
5534            )
5535        })
5536    }
5537
5538    /// Lightweight authenticated producer binding for run lifecycle handlers:
5539    /// `(client_id, terminal)`. Never falls back to disk because an on-disk row
5540    /// has no live WebSocket producer to authorize new writes.
5541    pub async fn run_lifecycle_binding(&self, run_id: &str) -> Option<(String, bool)> {
5542        self.runs.lock().await.get(run_id).map(|meta| {
5543            (
5544                meta.active_client_id.clone(),
5545                meta.is_terminal() || meta.trace_corruption.is_some(),
5546            )
5547        })
5548    }
5549
5550    /// `(client_id, terminal, start_committed, completion_pending)` for the
5551    /// authenticated producer path. Proposal writes require the exact owner,
5552    /// a committed start, and no prepared terminal transaction.
5553    pub async fn run_lifecycle_state(&self, run_id: &str) -> Option<(String, bool, bool, bool)> {
5554        self.run_lifecycle_state_with_corruption(run_id).await.map(
5555            |(client_id, terminal, committed, pending, _)| {
5556                (client_id, terminal, committed, pending)
5557            },
5558        )
5559    }
5560
5561    pub async fn run_lifecycle_state_with_corruption(
5562        &self,
5563        run_id: &str,
5564    ) -> Option<(String, bool, bool, bool, Option<String>)> {
5565        self.runs.lock().await.get(run_id).map(|meta| {
5566            (
5567                meta.active_client_id.clone(),
5568                meta.is_terminal(),
5569                meta.start_committed,
5570                meta.pending_terminal.is_some() || meta.cancellation_pending.is_some(),
5571                meta.trace_corruption.clone(),
5572            )
5573        })
5574    }
5575
5576    /// Atomically fence an orphaned live run to a replacement socket that has
5577    /// already authenticated as the run's agent. The immutable durable owner
5578    /// never changes: proposal and terminal artifacts continue to validate
5579    /// against the original `RunStarted` identity.
5580    pub(crate) async fn resume_run(
5581        &self,
5582        run_id: &str,
5583        agent_id: &str,
5584        replacement_client_id: &str,
5585    ) -> Result<RunResumeBinding, String> {
5586        // Serialize the ownership swap with every durable append/finalization
5587        // for this run. A task that already crossed this fence commits before
5588        // resume; one that has only captured the stale socket identity loses
5589        // after resume and cannot write any bytes.
5590        let durability_lock = self.run_durability_lock(run_id).await;
5591        let _durability_guard = durability_lock.lock().await;
5592        // Session creation/removal takes this same short fence around the map
5593        // mutation. Keep both guards through the run-owner compare-and-swap so
5594        // neither a late contender nor a disconnect can invalidate liveness
5595        // between validation and commit.
5596        let _liveness_guard = self.run_resume_liveness.lock().await;
5597        let live_clients = self
5598            .sessions
5599            .lock()
5600            .await
5601            .keys()
5602            .cloned()
5603            .collect::<std::collections::HashSet<_>>();
5604        if !live_clients.contains(replacement_client_id) {
5605            return Err("runs.resume replacement socket is no longer active".into());
5606        }
5607        let mut runs = self.runs.lock().await;
5608        let meta = runs
5609            .get_mut(run_id)
5610            .ok_or_else(|| "run not found or not authorized".to_string())?;
5611        if meta.agent_id != agent_id {
5612            return Err("run not found or not authorized".into());
5613        }
5614        if meta.is_terminal() {
5615            return Err(format!(
5616                "{} run `{run_id}` is already terminal",
5617                car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX
5618            ));
5619        }
5620        if !meta.start_committed {
5621            return Err(format!(
5622                "{} run `{run_id}` start is not durably committed",
5623                car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX
5624            ));
5625        }
5626        if meta.pending_terminal.is_some()
5627            || meta.cancellation_pending.is_some()
5628            || meta.trace_corruption.is_some()
5629        {
5630            return Err(format!(
5631                "{} run `{run_id}` is quarantined and cannot be resumed",
5632                car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX
5633            ));
5634        }
5635
5636        if meta.active_client_id == replacement_client_id {
5637            let resumed_from_client_id =
5638                meta.resume_predecessor_client_id.clone().ok_or_else(|| {
5639                    format!(
5640                        "{} run `{run_id}` is already active on this socket",
5641                        car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX
5642                    )
5643                })?;
5644            return Ok(RunResumeBinding {
5645                run_id: meta.run_id.clone(),
5646                agent_id: meta.agent_id.clone(),
5647                active_client_id: meta.active_client_id.clone(),
5648                resumed_from_client_id,
5649            });
5650        }
5651        if live_clients.contains(&meta.active_client_id) {
5652            return Err(format!(
5653                "{} run `{run_id}` still has a live owner",
5654                car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX
5655            ));
5656        }
5657        let lease_is_current = meta.resume_lease.as_ref().is_some_and(|lease| {
5658            lease.disconnected_client_id == meta.active_client_id
5659                && tokio::time::Instant::now() < lease.expires_at
5660        });
5661        if !lease_is_current {
5662            return Err(format!(
5663                "{} run `{run_id}` resume lease is unavailable or expired",
5664                car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX
5665            ));
5666        }
5667
5668        let resumed_from_client_id = std::mem::replace(
5669            &mut meta.active_client_id,
5670            replacement_client_id.to_string(),
5671        );
5672        meta.resume_predecessor_client_id = Some(resumed_from_client_id.clone());
5673        meta.resume_lease = None;
5674        meta.durability_generation = meta.durability_generation.wrapping_add(1);
5675        Ok(RunResumeBinding {
5676            run_id: meta.run_id.clone(),
5677            agent_id: meta.agent_id.clone(),
5678            active_client_id: meta.active_client_id.clone(),
5679            resumed_from_client_id,
5680        })
5681    }
5682
5683    /// Returns the immutable durable owner and the currently fenced live owner.
5684    pub(crate) async fn run_owner_binding(&self, run_id: &str) -> Option<(String, String)> {
5685        self.runs
5686            .lock()
5687            .await
5688            .get(run_id)
5689            .map(|meta| (meta.client_id.clone(), meta.active_client_id.clone()))
5690    }
5691
5692    /// Append per-turn trace records to a run's in-memory buffer (agent
5693    /// run tracing, U2). The recorder calls this after every
5694    /// `proposal.submit` on a session with a current run, passing the
5695    /// `RunRecord::Turn`s the recorder produced for that proposal.
5696    ///
5697    /// Returns a [`RecordRunTurnsOutcome`]:
5698    /// - `Appended { new_total }` — the batch landed; `new_total` is the
5699    ///   run's new total turn count (the caller computes its next
5700    ///   `start_index` from it).
5701    /// - `RefusedCeiling` — the batch would take the run past
5702    ///   [`RECORD_TURNS_RUN_CEILING`] and was refused WHOLE (the under-lock
5703    ///   runaway backstop, ADV-1). Nothing was appended.
5704    /// - `UnknownOrTerminal` — the `run_id` is unknown (the bracket was
5705    ///   never opened) or already terminal (a turn arriving after
5706    ///   `runs.complete` is dropped — the run is closed). Nothing was
5707    ///   appended.
5708    ///
5709    /// U3 flushes the in-memory buffer to disk; U4 broadcasts it. Both read
5710    /// via [`Self::run_turns`].
5711    ///
5712    /// U3: the same turn records are appended to the run's JSONL file so
5713    /// REPLAY (U5) sees them after a restart. Persistence is the commit point:
5714    /// only after the exact JSONL batch is durable do memory and subscribers
5715    /// advance. The operation runs in a daemon-owned task, so dropping a
5716    /// request waiter cannot strand a persisted batch between disk and memory.
5717    pub async fn record_run_turns(
5718        self: &Arc<Self>,
5719        run_id: &str,
5720        records: Vec<car_proto::RunRecord>,
5721    ) -> RecordRunTurnsOutcome {
5722        self.record_run_turns_for_owner(run_id, None, records).await
5723    }
5724
5725    /// Append client-narrated turns only while `active_client_id` still owns
5726    /// the run. The socket id is daemon-minted and captured by the handler
5727    /// before it detaches the work into a daemon-owned task; a later resume
5728    /// therefore fences delayed work from the predecessor without trusting a
5729    /// caller-supplied credential.
5730    #[doc(hidden)]
5731    pub async fn record_run_turns_for_active_owner(
5732        self: &Arc<Self>,
5733        run_id: &str,
5734        active_client_id: &str,
5735        records: Vec<car_proto::RunRecord>,
5736    ) -> RecordRunTurnsOutcome {
5737        self.record_run_turns_for_owner(run_id, Some(active_client_id.to_string()), records)
5738            .await
5739    }
5740
5741    async fn record_run_turns_for_owner(
5742        self: &Arc<Self>,
5743        run_id: &str,
5744        expected_active_client_id: Option<String>,
5745        records: Vec<car_proto::RunRecord>,
5746    ) -> RecordRunTurnsOutcome {
5747        let state = Arc::clone(self);
5748        let run_id = run_id.to_string();
5749        match tokio::spawn(async move {
5750            state
5751                .record_run_turns_owned(&run_id, expected_active_client_id.as_deref(), records)
5752                .await
5753        })
5754        .await
5755        {
5756            Ok(outcome) => outcome,
5757            Err(error) => RecordRunTurnsOutcome::PersistenceFailed(format!(
5758                "durable turn append task failed: {error}"
5759            )),
5760        }
5761    }
5762
5763    /// Make a proposal's authenticated final trace durable exactly once and
5764    /// reflect a newly-appended batch into the live registry/subscription
5765    /// stream before proposal finalization can acknowledge success.
5766    pub async fn ensure_proposal_run_turns(
5767        self: &Arc<Self>,
5768        pending: &crate::run_store::PendingProposalFinalization,
5769    ) -> Result<(), String> {
5770        let state = Arc::clone(self);
5771        let pending = pending.clone();
5772        tokio::spawn(async move { state.ensure_proposal_run_turns_owned(&pending).await })
5773            .await
5774            .map_err(|error| format!("proposal trace durability task failed: {error}"))?
5775    }
5776
5777    async fn ensure_proposal_run_turns_owned(
5778        &self,
5779        pending: &crate::run_store::PendingProposalFinalization,
5780    ) -> Result<(), String> {
5781        let durability_lock = self.run_durability_lock(&pending.run_id).await;
5782        let _durability_guard = durability_lock.lock().await;
5783        let (agent_id, status, generation) = {
5784            let runs = self.runs.lock().await;
5785            let meta = runs
5786                .get(&pending.run_id)
5787                .ok_or_else(|| format!("active run `{}` is absent", pending.run_id))?;
5788            if let Some(error) = &meta.trace_corruption {
5789                return Err(error.clone());
5790            }
5791            if !meta.accepts_proposals() {
5792                return Err(format!("active run `{}` is not writable", pending.run_id));
5793            }
5794            (
5795                meta.agent_id.clone(),
5796                meta.live_status(),
5797                meta.durability_generation,
5798            )
5799        };
5800        let store = self.run_store.clone();
5801        let durable = pending.clone();
5802        let durable_result =
5803            tokio::task::spawn_blocking(move || store.ensure_proposal_turns(&durable))
5804                .await
5805                .map_err(|error| format!("proposal trace durability task failed: {error}"))?;
5806        let mut runs = self.runs.lock().await;
5807        let ensured = match durable_result {
5808            Ok(ensured) => ensured,
5809            Err(error) if crate::run_store::is_trace_corruption_error(&error) => {
5810                let message = self
5811                    .quarantine_run_trace_locked(&mut runs, &pending.run_id, error.to_string())
5812                    .await;
5813                return Err(message);
5814            }
5815            Err(error) => return Err(format!("proposal trace durability failed: {error}")),
5816        };
5817        let meta = runs.get(&pending.run_id).ok_or_else(|| {
5818            format!(
5819                "active run `{}` disappeared after durability",
5820                pending.run_id
5821            )
5822        })?;
5823        if let Some(error) = &meta.trace_corruption {
5824            return Err(error.clone());
5825        }
5826        if meta.durability_generation != generation || !meta.accepts_proposals() {
5827            return Err(format!(
5828                "active run `{}` changed during proposal trace durability",
5829                pending.run_id
5830            ));
5831        }
5832        let already_live = !ensured.appended
5833            && meta.turns.iter().any(|record| match record {
5834                car_proto::RunRecord::Turn(turn) => {
5835                    turn.proposal_id.as_deref() == Some(pending.final_proposal_id.as_str())
5836                }
5837                _ => false,
5838            });
5839        if ensured.appended || !already_live {
5840            let meta = runs
5841                .get_mut(&pending.run_id)
5842                .expect("run registry remains locked during proposal trace commit");
5843            let base = ensured
5844                .records
5845                .first()
5846                .and_then(|record| match record {
5847                    car_proto::RunRecord::Turn(turn) => Some(turn.index),
5848                    _ => None,
5849                })
5850                .unwrap_or(meta.turns.len());
5851            meta.turns.extend(ensured.records.iter().cloned());
5852            meta.durability_generation = meta.durability_generation.wrapping_add(1);
5853            let subs = self.run_subscribers.lock().await;
5854            for (offset, record) in ensured.records.into_iter().enumerate() {
5855                Self::fanout_locked(
5856                    &subs,
5857                    &pending.run_id,
5858                    &agent_id,
5859                    record,
5860                    base + offset + 1,
5861                    status,
5862                );
5863            }
5864        }
5865        Ok(())
5866    }
5867
5868    async fn record_run_turns_owned(
5869        &self,
5870        run_id: &str,
5871        expected_active_client_id: Option<&str>,
5872        mut records: Vec<car_proto::RunRecord>,
5873    ) -> RecordRunTurnsOutcome {
5874        // Serialize durability only with this run. The global registry lock is
5875        // held for the admission snapshot and memory/fanout commit, never for
5876        // trace scanning, append, flush, or fsync.
5877        let durability_lock = self.run_durability_lock(run_id).await;
5878        let _durability_guard = durability_lock.lock().await;
5879        let (agent_id, base, new_total, status, generation) = {
5880            let mut runs = self.runs.lock().await;
5881            match runs.get_mut(run_id) {
5882                Some(meta) => {
5883                    if let Some(error) = &meta.trace_corruption {
5884                        return RecordRunTurnsOutcome::PersistenceFailed(error.clone());
5885                    }
5886                    if expected_active_client_id
5887                        .is_some_and(|expected| meta.active_client_id != expected)
5888                    {
5889                        return RecordRunTurnsOutcome::UnknownOrTerminal;
5890                    }
5891                    if !meta.accepts_proposals() {
5892                        return RecordRunTurnsOutcome::UnknownOrTerminal;
5893                    }
5894                    // ADV-1: hard ceiling, under the lock. Count only the
5895                    // `Turn` records — Started/Ended are not buffered here,
5896                    // and the ceiling is a turn-count cap. Refuse the whole
5897                    // batch if appending it would cross the ceiling.
5898                    let incoming_turns = records
5899                        .iter()
5900                        .filter(|r| matches!(r, car_proto::RunRecord::Turn(_)))
5901                        .count();
5902                    if meta.turns.len() + incoming_turns > RECORD_TURNS_RUN_CEILING {
5903                        return RecordRunTurnsOutcome::RefusedCeiling;
5904                    } else {
5905                        // Position of the first record about to be appended —
5906                        // its post-append cursor is `base + 1`.
5907                        let base = meta.turns.len();
5908                        // FIX 6: re-stamp each turn's `index` from the LIVE
5909                        // append position under the `runs` lock. The caller
5910                        // computes a provisional `start_index` from a pre-read
5911                        // `run_turn_count` OUTSIDE this lock — a TOCTOU: two
5912                        // concurrent proposals on one connection can read the
5913                        // same `start_index` and bake overlapping indices.
5914                        // Authoritatively assigning the index here, from
5915                        // `base + offset`, makes it contiguous and correct
5916                        // regardless of the racing reads upstream.
5917                        for (offset, record) in records.iter_mut().enumerate() {
5918                            if let car_proto::RunRecord::Turn(turn) = record {
5919                                turn.index = base + offset;
5920                            }
5921                        }
5922                        let agent_id = meta.agent_id.clone();
5923                        let status = meta.live_status();
5924                        (
5925                            agent_id,
5926                            base,
5927                            base + records.len(),
5928                            status,
5929                            meta.durability_generation,
5930                        )
5931                    }
5932                }
5933                None => return RecordRunTurnsOutcome::UnknownOrTerminal,
5934            }
5935        };
5936
5937        let store = self.run_store.clone();
5938        let durable_agent = agent_id.clone();
5939        let durable_run = run_id.to_string();
5940        let durable_records = records.clone();
5941        let append = tokio::task::spawn_blocking(move || {
5942            store.append_turns(&durable_agent, &durable_run, &durable_records)
5943        })
5944        .await;
5945        let mut runs = self.runs.lock().await;
5946        match append {
5947            Ok(Ok(())) => {}
5948            Ok(Err(error)) => {
5949                tracing::warn!(run_id, %error, "run-store: failed to persist turns");
5950                if crate::run_store::is_trace_corruption_error(&error) {
5951                    let message = self
5952                        .quarantine_run_trace_locked(&mut runs, run_id, error.to_string())
5953                        .await;
5954                    return RecordRunTurnsOutcome::PersistenceFailed(message);
5955                }
5956                return RecordRunTurnsOutcome::PersistenceFailed(error.to_string());
5957            }
5958            Err(error) => {
5959                tracing::warn!(run_id, %error, "run-store: durable turn task failed");
5960                return RecordRunTurnsOutcome::PersistenceFailed(error.to_string());
5961            }
5962        }
5963
5964        let meta = runs
5965            .get_mut(run_id)
5966            .expect("run registry entry survives a committed turn append");
5967        if let Some(error) = &meta.trace_corruption {
5968            return RecordRunTurnsOutcome::PersistenceFailed(error.clone());
5969        }
5970        if meta.durability_generation != generation || !meta.accepts_proposals() {
5971            return RecordRunTurnsOutcome::PersistenceFailed(format!(
5972                "run `{run_id}` changed during durable turn append"
5973            ));
5974        }
5975        meta.turns.extend(records.iter().cloned());
5976        meta.durability_generation = meta.durability_generation.wrapping_add(1);
5977        let subs = self.run_subscribers.lock().await;
5978        for (offset, record) in records.into_iter().enumerate() {
5979            Self::fanout_locked(&subs, run_id, &agent_id, record, base + offset + 1, status);
5980        }
5981        RecordRunTurnsOutcome::Appended { new_total }
5982    }
5983
5984    /// Number of turns already recorded for a run — the `start_index` the
5985    /// recorder passes so per-turn `index` stays monotonic across the
5986    /// run's proposals. `0` for an unknown run.
5987    pub async fn run_turn_count(&self, run_id: &str) -> usize {
5988        self.runs
5989            .lock()
5990            .await
5991            .get(run_id)
5992            .map(|m| m.turns.len())
5993            .unwrap_or(0)
5994    }
5995
5996    /// Clone of a run's ordered per-turn trace (agent run tracing, U2).
5997    /// This is the accessor U3 reads to flush turns to disk and U4 reads
5998    /// to broadcast them. Empty Vec for an unknown run or a run with no
5999    /// turns yet.
6000    pub async fn run_turns(&self, run_id: &str) -> Vec<car_proto::RunRecord> {
6001        self.runs
6002            .lock()
6003            .await
6004            .get(run_id)
6005            .map(|m| m.turns.clone())
6006            .unwrap_or_default()
6007    }
6008
6009    /// Atomically snapshot a run's turns AND register a live
6010    /// `runs.trace.event` subscriber for `(run_id, host_client_id)` —
6011    /// the load-bearing invariant #1 (agent run tracing, U4).
6012    ///
6013    /// Holds the `runs` lock across BOTH (a) reading the run's current
6014    /// turn buffer as the snapshot at cursor `C = turns.len()` and (b)
6015    /// inserting the subscriber into `run_subscribers`. Because the
6016    /// lifecycle path (`record_run_turns` / `complete_run` /
6017    /// `mark_run_incomplete`) appends-and-notifies under the SAME lock,
6018    /// the snapshot contains exactly the turns ≤ C and every turn appended
6019    /// after registration is delivered — no turn in the snapshot/register
6020    /// window is dropped (gap) or double-delivered (dup).
6021    ///
6022    /// Spawns the subscriber's drain task (bounded channel → WS) so the
6023    /// producer never writes the socket directly (invariant #2). Returns
6024    /// the snapshot response shape. `None` if the `run_id` is unknown (the
6025    /// handler maps that to a not-found error). A re-subscribe by the same
6026    /// `(run_id, host_client_id)` replaces the prior subscriber (its drain
6027    /// task ends when the old sender drops) and re-snapshots — the R8
6028    /// reconnect path: the fresh snapshot covers any turns emitted during
6029    /// the gap with no dup.
6030    ///
6031    /// Terminal-run disk fallback: a completed/incomplete run has its
6032    /// in-memory `turns` evicted for heap hygiene (see
6033    /// [`Self::clear_terminal_run_turns`]). When the run is terminal AND its
6034    /// in-memory buffer is empty, the snapshot is re-sourced from the disk
6035    /// store ([`crate::run_store::RunStore::get_run_trace`], the same source
6036    /// `runs.get_trace` reads), filtered to `RunRecord::Turn` so the shape
6037    /// matches the live in-memory snapshot. A terminal run takes no further
6038    /// appends, so the disk trail is final and there is no gap/dup concern —
6039    /// the snapshot/register atomicity the `runs` lock provides is only
6040    /// load-bearing for the in-progress path, which is unchanged.
6041    pub(crate) async fn subscribe_run_page(
6042        &self,
6043        run_id: &str,
6044        host_client_id: &str,
6045        channel: Arc<WsChannel>,
6046        cursor: usize,
6047        limit: usize,
6048    ) -> Result<Option<RunSubscribePageResult>, String> {
6049        // Build the subscriber (and its drain task) OUTSIDE the runs lock
6050        // so spawning never happens under the hot lock; registration
6051        // itself is the only thing serialized.
6052        let subscriber =
6053            crate::host::RunTraceSubscriber::spawn(host_client_id.to_string(), channel);
6054
6055        let (agent_id, mut expected_state) = {
6056            let runs = self.runs.lock().await;
6057            let Some(meta) = runs.get(run_id) else {
6058                return Ok(None);
6059            };
6060            if let Some(error) = &meta.trace_corruption {
6061                return Err(error.clone());
6062            }
6063            if meta.is_terminal() {
6064                return Ok(Some(RunSubscribePageResult::Durable {
6065                    agent_id: meta.agent_id.clone(),
6066                    status: meta.live_status(),
6067                }));
6068            }
6069            (
6070                meta.agent_id.clone(),
6071                (meta.turns.len(), meta.pending_terminal.is_some()),
6072            )
6073        };
6074
6075        for attempt in 0..=RUN_SUBSCRIBE_SUMMARY_STATE_RETRY_LIMIT {
6076            // Keep disk I/O off the global run-registry lock. A concurrent
6077            // mutation advances the captured state under that lock; retrying
6078            // the summary read then gives snapshot/registration one coherent
6079            // durable/live boundary.
6080            let store = self.run_store.clone();
6081            let durable_agent = agent_id.clone();
6082            let durable_run = run_id.to_string();
6083            let durable_corruption = tokio::task::spawn_blocking(move || {
6084                store.run_trace_corruption_for(&durable_agent, &durable_run)
6085            })
6086            .await
6087            .map_err(|error| format!("run trace corruption check task failed: {error}"))?
6088            .map_err(|error| format!("run trace corruption check failed: {error}"))?;
6089            let mut runs = self.runs.lock().await;
6090            let Some(meta) = runs.get(run_id) else {
6091                return Ok(None);
6092            };
6093            if let Some(error) = &meta.trace_corruption {
6094                return Err(error.clone());
6095            }
6096            if meta.agent_id != agent_id {
6097                return Err(format!("run `{run_id}` changed ownership during subscribe"));
6098            }
6099            if let Some(corruption) = durable_corruption {
6100                let message = self
6101                    .quarantine_run_trace_locked(
6102                        &mut runs,
6103                        run_id,
6104                        format!("malformed run trace record at line {}", corruption.line),
6105                    )
6106                    .await;
6107                return Err(message);
6108            }
6109            let status = meta.live_status();
6110            if meta.is_terminal() {
6111                return Ok(Some(RunSubscribePageResult::Durable { agent_id, status }));
6112            }
6113            let current_state = (meta.turns.len(), meta.pending_terminal.is_some());
6114            if current_state != expected_state {
6115                if attempt == RUN_SUBSCRIBE_SUMMARY_STATE_RETRY_LIMIT {
6116                    return Err(format!(
6117                        "run `{run_id}` changed repeatedly during durable subscribe validation; retry"
6118                    ));
6119                }
6120                expected_state = current_state;
6121                drop(runs);
6122                continue;
6123            }
6124            let live_cursor = meta.turns.len();
6125            if cursor > live_cursor {
6126                return Err(format!(
6127                    "runs.subscribe cursor {cursor} exceeds live_cursor {live_cursor}"
6128                ));
6129            }
6130            let end = cursor.saturating_add(limit).min(live_cursor);
6131            let turns = meta.turns[cursor..end].to_vec();
6132            let next_cursor = (end < live_cursor).then_some(end);
6133            let subscribed = next_cursor.is_none();
6134            if subscribed {
6135                let mut subs = self.run_subscribers.lock().await;
6136                subs.insert((run_id.to_string(), host_client_id.to_string()), subscriber);
6137            }
6138            drop(runs);
6139
6140            return Ok(Some(RunSubscribePageResult::Ready(
6141                car_proto::RunSubscribeResponse {
6142                    run_id: run_id.to_string(),
6143                    agent_id,
6144                    turns,
6145                    cursor,
6146                    limit,
6147                    next_cursor,
6148                    live_cursor,
6149                    subscribed,
6150                    status,
6151                },
6152            )));
6153        }
6154        unreachable!("bounded subscribe validation loop always returns")
6155    }
6156
6157    /// Remove a live run-trace subscriber for `(run_id, host_client_id)`
6158    /// (agent run tracing, U4). Returns `true` if a subscription existed.
6159    /// Dropping the [`crate::host::RunTraceSubscriber`] drops its channel
6160    /// sender, which ends the drain task.
6161    pub async fn unsubscribe_run(&self, run_id: &str, host_client_id: &str) -> bool {
6162        self.run_subscribers
6163            .lock()
6164            .await
6165            .remove(&(run_id.to_string(), host_client_id.to_string()))
6166            .is_some()
6167    }
6168
6169    /// Drop every live run-trace subscription owned by `host_client_id`
6170    /// (agent run tracing, U4 — R8 cleanup). Called from
6171    /// [`remove_session`](crate::session::ServerState::remove_session) on disconnect so a CarHost that drops doesn't
6172    /// leave dangling drain tasks. Reconnect-durability is client-side:
6173    /// the run stays subscribable while it lives, and the CarHost re-issues
6174    /// `runs.subscribe {run_id}` on its new connection — the server never
6175    /// synthesizes a failure on subscriber drop.
6176    pub async fn drop_run_subscribers_for_client(&self, host_client_id: &str) {
6177        self.run_subscribers
6178            .lock()
6179            .await
6180            .retain(|(_run, client), _| client != host_client_id);
6181    }
6182
6183    /// Disconnect cleanup for agent runs (R5). Called from
6184    /// [`remove_session`] with the `client_id` of the dropping
6185    /// connection. For every run this connection owns that is **not**
6186    /// yet terminal, wait a short grace window then re-check: if a
6187    /// concurrently-dispatched `runs.complete` made it terminal in the
6188    /// meantime, leave it; otherwise mark it `Incomplete`. The grace
6189    /// window is the fix for the serve-mode false-positive where a
6190    /// healthy `runs.complete` is still in flight (in a spawned
6191    /// dispatch task) when the close frame arrives (Risk: run identity
6192    /// races).
6193    async fn sweep_runs_for_disconnect(&self, session: &ClientSession) {
6194        let client_id = session.client_id.as_str();
6195        // Snapshot the still-open runs owned by this connection.
6196        let pending: Vec<(String, Option<tokio::time::Instant>)> = {
6197            let runs = self.runs.lock().await;
6198            runs.values()
6199                .filter(|m| m.active_client_id == client_id && !m.is_terminal())
6200                .map(|m| {
6201                    let resume_deadline = m.resume_lease.as_ref().and_then(|lease| {
6202                        (lease.disconnected_client_id == client_id).then_some(lease.expires_at)
6203                    });
6204                    (m.run_id.clone(), resume_deadline)
6205                })
6206                .collect()
6207        };
6208        if pending.is_empty() {
6209            return;
6210        }
6211        for (run_id, resume_deadline) in pending {
6212            // The lease deadline was installed atomically with removal from
6213            // `sessions`; sleeping to that absolute instant keeps teardown
6214            // work before this point from silently extending the contract.
6215            match resume_deadline {
6216                Some(deadline) => tokio::time::sleep_until(deadline).await,
6217                None => tokio::time::sleep(RUN_COMPLETE_GRACE).await,
6218            }
6219            match self.run_lifecycle_state(&run_id).await {
6220                None | Some((_, true, _, _)) => continue,
6221                Some((owner, false, _, _)) if owner != client_id => {
6222                    // An authenticated replacement claimed this run during the
6223                    // disconnect grace window. The old owner is fenced and
6224                    // must not be allowed to close the replacement's run.
6225                    continue;
6226                }
6227                Some((_, false, true, _)) => {}
6228                Some((_, false, false, _)) => {
6229                    match self
6230                        .reconcile_or_release_unacknowledged_start(session, &run_id)
6231                        .await
6232                    {
6233                        Ok(true) => continue,
6234                        Ok(false) => {}
6235                        Err(error) => {
6236                            tracing::error!(run_id, client_id, %error, "failed to reconcile or release an unacknowledged RunStarted transaction");
6237                            continue;
6238                        }
6239                    }
6240                }
6241            }
6242            match self.run_store.pending_proposal(&run_id) {
6243                Ok(Some(_)) => {
6244                    tracing::warn!(
6245                        run_id,
6246                        client_id,
6247                        "leaving run open because proposal finalization is pending durable replay"
6248                    );
6249                    continue;
6250                }
6251                Err(error) => {
6252                    tracing::error!(run_id, client_id, %error, "leaving run open because proposal finalization state is unreadable and outcome is unknown");
6253                    continue;
6254                }
6255                Ok(None) => {}
6256            }
6257            match self.run_store.execution_marker(&run_id) {
6258                Ok(Some(marker)) => {
6259                    tracing::warn!(
6260                        run_id,
6261                        client_id,
6262                        original_proposal_id = marker.original_proposal_id,
6263                        "leaving run open because proposal execution outcome is unknown"
6264                    );
6265                    continue;
6266                }
6267                Err(error) => {
6268                    tracing::error!(run_id, client_id, %error, "leaving run open because proposal execution marker is unreadable and outcome is unknown");
6269                    continue;
6270                }
6271                Ok(None) => {}
6272            }
6273            if let Some(ended) = self
6274                .prepare_run_incomplete_for_active_owner(&run_id, Some(client_id))
6275                .await
6276            {
6277                let durable_client_id = ended.client_id.as_deref().unwrap_or(client_id);
6278                let append_result = match session
6279                    .append_run_terminal_event_once(&ended, durable_client_id)
6280                    .await
6281                {
6282                    Err(error) if error.is_retry_safe() => {
6283                        tracing::warn!(
6284                            run_id,
6285                            client_id,
6286                            %error,
6287                            "disconnect terminal durability is unknown; retrying the exact journal row once"
6288                        );
6289                        session
6290                            .append_run_terminal_event_once(&ended, durable_client_id)
6291                            .await
6292                    }
6293                    result => result,
6294                };
6295                if let Err(error) = append_result {
6296                    tracing::error!(run_id, client_id, %error, "failed to bind disconnect terminal journal event");
6297                    continue;
6298                }
6299                if let Err(error) = self.commit_run_completion(&ended).await {
6300                    tracing::error!(run_id, client_id, %error, "failed to commit disconnect terminal");
6301                    continue;
6302                }
6303                if let Err(error) = session.clear_run_journal_binding(&run_id).await {
6304                    tracing::error!(run_id, client_id, %error, "failed to clear durable disconnect journal binding");
6305                    continue;
6306                }
6307                let mut current = session.current_run_id.lock().await;
6308                if current.as_deref() == Some(run_id.as_str()) {
6309                    *current = None;
6310                }
6311            }
6312        }
6313    }
6314
6315    /// Fails only when `<home>/.car/policies/` holds a malformed rule file —
6316    /// see [`apply_project_policies`] for why that is fatal rather than a
6317    /// warning.
6318    pub async fn create_session(
6319        &self,
6320        client_id: &str,
6321        channel: Arc<WsChannel>,
6322    ) -> Result<Arc<ClientSession>, String> {
6323        let journal_path = self.journal_dir.join(format!("{}.jsonl", client_id));
6324        let event_log = match self.journal_failures.clone() {
6325            Some(failures) => EventLog::with_journal_failure_injector(journal_path, failures),
6326            None => EventLog::with_journal(journal_path),
6327        };
6328
6329        let negotiated_capabilities =
6330            Arc::new(std::sync::RwLock::new(std::collections::BTreeSet::new()));
6331        let halted = Arc::new(AtomicBool::new(false));
6332        let ws_executor = Arc::new(WsToolExecutor::new(
6333            channel.clone(),
6334            negotiated_capabilities.clone(),
6335            halted.clone(),
6336        ));
6337
6338        // Compose: connector (remote MCP) tools route to the shared
6339        // process-wide executor; everything else falls back to this
6340        // session's WS tool executor. Both views share the same routing
6341        // table, so a connector enabled in any session is callable here.
6342        let executor: Arc<dyn ToolExecutor> =
6343            Arc::new(self.mcp_executor.share_with_fallback(ws_executor));
6344
6345        // Outbound human messaging. Built-in iMessage adapter over the same
6346        // `RealMessageSender` + `~/.car` config store the daemon's
6347        // `messaging.*` handlers use, plus a host fallback so a channel CAR has
6348        // no built-in transport for (Teams, per Parslee-ai/car#885) is
6349        // delivered by the host rather than compiled into the runtime.
6350        //
6351        // NO APPROVAL LOOP. `RealMessageSender` deliberately calls the *un-gated*
6352        // `car_ffi_common::integrations::messages_send` so that the approval
6353        // transport's own prompt does not raise an approval for itself. That
6354        // choice is safe to reuse here, and the recursion this new path might
6355        // look like it opens does not exist, because the two directions never
6356        // meet: `messaging.send` → registry → adapter → `messages_send` is a
6357        // straight line into the JXA bridge with no re-entry into the runtime,
6358        // and an approval raised *for* `messaging.send` is delivered by the
6359        // orchestrator/fanout path, which calls `MessageSender` directly and
6360        // never dispatches a runtime tool. Neither leg can call the other, so
6361        // there is no cycle to bound. Keep it that way: routing approval
6362        // delivery through `messaging.send` WOULD create one.
6363        let outbound = Arc::new(car_messaging::outbound::OutboundRegistry::new());
6364        outbound.register(Arc::new(
6365            car_messaging::outbound::ImessageOutboundAdapter::new(
6366                Arc::new(crate::messaging_orchestrator::RealMessageSender),
6367                crate::messaging_config::MessagingConfigStore::from_home(),
6368            ),
6369        ));
6370        outbound.set_fallback(Arc::new(crate::host_channel::HostChannelAdapter::new(
6371            executor.clone(),
6372        )));
6373
6374        // The trajectory store is the daemon-wide one, so traces from every
6375        // session accumulate in a single history. Without this the executor's
6376        // `persist_trajectory` short-circuits on its `self.trajectory_store
6377        // .as_ref()?` and nothing is ever recorded — which is why the
6378        // per-tool success rates had no data behind them.
6379        let runtime = Runtime::new()
6380            .with_event_log(event_log)
6381            .with_executor(executor)
6382            .with_trajectory_store(self.trajectory_store.clone())
6383            .with_message_sink(outbound);
6384
6385        // Declarative rules from `~/.car/policies/*.toml`. Same `~/.car`
6386        // convention the messaging config and the approval journal use — the
6387        // daemon serves whatever project a client happens to be working in, so
6388        // its policy set is the operator's, not a per-project one. (The
6389        // assistant, which IS rooted in a project, loads `<root>/.car`.)
6390        if let Some(car_dir) = car_home_dir() {
6391            apply_project_policies(&runtime, &car_dir).await?;
6392        }
6393
6394        // The session's permission-tier gate. Built here rather than inline in
6395        // the `ClientSession` literal below because the admission gate
6396        // registered a few lines down has to hold the SAME object the
6397        // `permission.*` RPCs mutate — see the field doc on
6398        // `ClientSession::permission_gate`.
6399        //
6400        // `SandboxEdit` is the default `docs/websocket-protocol.md` publishes,
6401        // and the code had drifted to `ReadOnly` while nothing enforced the
6402        // tier. `RiskClassifier::baseline` maps both `StateWrite` and
6403        // `ToolCall` to `SandboxEdit`, so keeping the drifted `ReadOnly`
6404        // default the moment enforcement lands would escalate essentially
6405        // every action every existing binding client submits. `full_access`
6406        // actions still escalate — mandatory HITL, exactly as documented.
6407        let permission_gate = Arc::new(tokio::sync::RwLock::new(car_policy::PermissionGate::new(
6408            car_policy::PermissionTier::SandboxEdit,
6409        )));
6410        // These handles are shared with the permission admission gate so it
6411        // observes the identity established later by `session.auth`; taking a
6412        // snapshot here would leave every newly-created session unbound.
6413        let authenticated = Arc::new(std::sync::atomic::AtomicBool::new(false));
6414        let agent_id = Arc::new(tokio::sync::Mutex::new(None));
6415        let callback_tool_schema_digests = Arc::new(tokio::sync::RwLock::new(HashMap::new()));
6416
6417        // If the embedder supplied a shared memgine, every session uses it.
6418        // Otherwise each session gets its own — matches pre-extraction behavior.
6419        // Create it before the admission gate so proposal.submit and the
6420        // permission advisory RPCs consult the exact same live skill ceilings.
6421        let memgine = match &self.shared_memgine {
6422            Some(eng) => eng.clone(),
6423            None => Arc::new(Mutex::new(car_memgine::MemgineEngine::new(None))),
6424        };
6425
6426        // A terminal callback poisons this WebSocket session after the current
6427        // proposal's engine-level abort and rollback. Register the latch as an
6428        // ordinary admission gate so a later proposal gets the same complete
6429        // rejected-result shape as every other pre-execution refusal.
6430        runtime
6431            .register_admission_gate(Arc::new(SessionHaltAdmissionGate {
6432                halted: halted.clone(),
6433            }))
6434            .await;
6435
6436        // Statically verify submitted proposals before any action dispatches.
6437        //
6438        // This is the surface that matters for it: `proposal.submit` accepts
6439        // caller-authored, **multi-action** proposals, and per-action validation
6440        // only rejects an action once execution reaches it — so a bad tool name
6441        // in action 5 is caught after actions 1–4 have already had their side
6442        // effects. The gate refuses the whole proposal up front, so nothing
6443        // partial happens. (On a runtime that submits one action at a time, like
6444        // the assistant loop, it adds nothing the validator wasn't already
6445        // doing; see `car_engine::verify_gate` for what blocks and why.)
6446        runtime
6447            .register_admission_gate(Arc::new(car_engine::StaticVerificationGate::new(
6448                runtime.tools.clone(),
6449            )))
6450            .await;
6451
6452        // Out-of-process supervision (proposal item 5). Registered
6453        // unconditionally: with no supervisor subscribed the gate short-circuits
6454        // to Allow on an empty-map read, so leaving it on costs nothing and
6455        // means a supervisor can attach to a session that was already running
6456        // rather than only to sessions created after it connected.
6457        runtime
6458            .register_admission_gate(Arc::new(crate::supervision::SupervisionGate::new(
6459                self.supervision.clone(),
6460            )))
6461            .await;
6462
6463        // Enforce the session's granted permission tier before any action
6464        // dispatches (Parslee-ai/car#890).
6465        //
6466        // Everything this needs already existed — the per-session tier gate,
6467        // the durable daemon-wide approval ledger, `permission.approve` — but
6468        // nothing consulted any of it on the way to the executor, so an action
6469        // the daemon's own `permission.evaluate` called `needs_approval`
6470        // executed anyway and its state write persisted. This is the missing
6471        // call site: it asks the SAME gate the same question `permission.
6472        // evaluate` asks, against the SAME shared ledger, so the advisory
6473        // answer and the enforced one cannot diverge. An escalation blocks the
6474        // whole proposal until an operator approves the named fingerprint;
6475        // approving it makes the next submit come back `Allow` with no
6476        // escalation raised at all.
6477        runtime
6478            .register_admission_gate(Arc::new(
6479                crate::permission_gate::PermissionAdmissionGate::new(
6480                    permission_gate.clone(),
6481                    self.approval_ledger.clone(),
6482                )
6483                .with_authenticated_agent(authenticated.clone(), agent_id.clone())
6484                .with_callback_tools(
6485                    callback_tool_schema_digests.clone(),
6486                    runtime.registry.clone(),
6487                )
6488                .with_skill_memgine(memgine.clone())
6489                .with_event_log(runtime.log.clone()),
6490            ))
6491            .await;
6492
6493        let session = Arc::new(ClientSession {
6494            client_id: client_id.to_string(),
6495            runtime: Arc::new(runtime),
6496            channel,
6497            host: self.host.clone(),
6498            memgine,
6499            browser: car_ffi_common::browser::BrowserSessionSlot::new(),
6500            // When auth is disabled (no token installed), every
6501            // session is "authenticated" by default — preserves the
6502            // pre-#32 behaviour. When auth is enabled, the value is
6503            // ignored on creation; the dispatcher's gate checks
6504            // `ServerState::auth_token.is_some()` to decide whether
6505            // to enforce.
6506            authenticated,
6507            negotiated_protocol_version: std::sync::atomic::AtomicU32::new(0),
6508            negotiated_capabilities,
6509            inference_control: Arc::new(crate::inference_control::InferenceRegistry::default()),
6510            is_host: std::sync::atomic::AtomicBool::new(false),
6511            agent_id,
6512            agent_method_allowlist: Arc::new(std::sync::RwLock::new(None)),
6513            callback_tool_schema_digests,
6514            memory_namespace: tokio::sync::Mutex::new(None),
6515            bound_memgine: tokio::sync::Mutex::new(None),
6516            current_run_id: tokio::sync::Mutex::new(None),
6517            run_lifecycle_guard: Arc::new(tokio::sync::Mutex::new(())),
6518            permission_gate,
6519            halted,
6520            evolution_guard: crate::evolution::CycleGuard::default(),
6521            last_chat_turn: tokio::sync::Mutex::new(None),
6522            chat_inflight: std::sync::atomic::AtomicUsize::new(0),
6523            tenant: tokio::sync::Mutex::new(None),
6524            tool_stream_subscribed: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)),
6525        });
6526
6527        // Seed connector tools already enabled (in any session) into this
6528        // new session's runtime registry so the model sees them at once.
6529        // Only touches the manager if connectors have been initialized —
6530        // embedders that never use connectors pay nothing here.
6531        if self.connectors.get().is_some() {
6532            for entry in self.connectors().enabled_tool_entries().await {
6533                session.runtime.register_tool_entry(entry).await;
6534            }
6535        }
6536
6537        {
6538            let _liveness_guard = self.run_resume_liveness.lock().await;
6539            self.sessions
6540                .lock()
6541                .await
6542                .insert(client_id.to_string(), session.clone());
6543        }
6544
6545        Ok(session)
6546    }
6547
6548    /// Bind a session's runtime to the [`car_engine::McpSubstrate`] backed by an
6549    /// already-connected MCP connector, making that session **coherent**:
6550    /// its commodity built-ins (`read_file`/`write_file`/…/`run_command`)
6551    /// act on the *same* environment as its `mcp_{slug}_*` connector tools,
6552    /// instead of being split between the host/WS client and the remote MCP
6553    /// server (`docs/execution-substrate.md` §3, phase 3).
6554    ///
6555    /// This is **opt-in and explicit** — it is reached only via the
6556    /// `session.bindSubstrate` JSON-RPC method, never inferred from a
6557    /// connector merely being enabled. Sessions that never call it keep the
6558    /// exact historic composition (`share_with_fallback(ws_executor)` over a
6559    /// default `LocalSubstrate`), so GUI / A2A-over-WS / connectors-as-tools
6560    /// / coder behavior is unchanged.
6561    ///
6562    /// On bind we:
6563    /// 1. Wrap the live connector session (shared with the
6564    ///    `mcp_{slug}_*` routes — no second dial) as an [`car_engine::McpSubstrate`] and
6565    ///    `set_substrate` it on the session runtime.
6566    /// 2. `register_agent_basics()` so the model sees portable names
6567    ///    (`read_file`/`run_command`/…) that the engine routes to the bound
6568    ///    substrate.
6569    /// 3. Swap the session executor to a [`SubstrateShadowExecutor`] over the
6570    ///    same `share_with_fallback(ws_executor)` composition, so the bare
6571    ///    substrate-owned names fall through to the substrate while connector
6572    ///    routes and host tool callbacks keep working unchanged (option (a)).
6573    ///
6574    /// Returns the substrate's environment name (the connector slug) on
6575    /// success, or an error string if the connector is not connected.
6576    pub async fn bind_substrate_to_connector(
6577        &self,
6578        session: &Arc<ClientSession>,
6579        slug: &str,
6580    ) -> Result<String, String> {
6581        self.ensure_connectors_loaded().await;
6582
6583        // The connector's live MCP session is the same handle the
6584        // `mcp_{slug}_*` routes dispatch through — wrap it, don't re-dial.
6585        let mcp_session = self
6586            .mcp_executor
6587            .session(slug)
6588            .await
6589            .ok_or_else(|| format!("connector '{slug}' is not connected"))?;
6590
6591        let substrate: Arc<dyn car_engine::Substrate> =
6592            Arc::new(car_engine::McpSubstrate::new(mcp_session, slug.to_string()));
6593        let name = substrate.name().to_string();
6594
6595        // 1. Bind the substrate; 2. register substrate-backed built-ins.
6596        //
6597        // These three steps (set_substrate → register_agent_basics →
6598        // set_executor below) are not one critical section, so a tool `execute`
6599        // racing on this same session mid-swap could observe a half-applied
6600        // state. That's benign here: a WS connection processes its JSON-RPC
6601        // requests sequentially, and `bindSubstrate` is a client control call
6602        // issued before tool work — so no in-flight built-in overlaps the swap.
6603        session.runtime.set_substrate(substrate).await;
6604        session.runtime.register_agent_basics().await;
6605
6606        // 3. Shadow the substrate-owned names so they fall through to the
6607        //    bound substrate, keeping connector routes + WS host callbacks.
6608        let ws_executor = Arc::new(WsToolExecutor::new(
6609            session.channel.clone(),
6610            session.negotiated_capabilities.clone(),
6611            session.halted.clone(),
6612        ));
6613        let composed: Arc<dyn ToolExecutor> =
6614            Arc::new(self.mcp_executor.share_with_fallback(ws_executor));
6615        let shadowed: Arc<dyn ToolExecutor> = Arc::new(SubstrateShadowExecutor::new(composed));
6616        session.runtime.set_executor(shadowed).await;
6617
6618        Ok(name)
6619    }
6620
6621    /// Remove a per-client session from the registry on disconnect.
6622    /// Returns the removed session if present so callers can drop any
6623    /// remaining strong refs (e.g. drain pending tool callbacks). Fix
6624    /// for MULTI-4 / WS-3 — without this, `state.sessions` retains
6625    /// `Arc<ClientSession>` for every connection that ever existed.
6626    pub async fn remove_session(&self, client_id: &str) -> Option<Arc<ClientSession>> {
6627        // Cancel any live detached tool invocations on this session's
6628        // runtime (linus review D3): after disconnect the per-session
6629        // registry is unreachable, so a still-running tool would be
6630        // uncancellable until daemon restart. Same teardown discipline
6631        // as run subscribers / chat sessions below.
6632        if let Some(session) = self.sessions.lock().await.get(client_id).cloned() {
6633            let n = session.runtime.tool_handles.cancel_all().await;
6634            if n > 0 {
6635                tracing::info!(
6636                    client_id,
6637                    cancelled = n,
6638                    "cancelled detached tools at session teardown"
6639                );
6640            }
6641        }
6642
6643        let removed = {
6644            let _liveness_guard = self.run_resume_liveness.lock().await;
6645            let removed = self.sessions.lock().await.remove(client_id);
6646            let resume_negotiated = removed.as_ref().is_some_and(|session| {
6647                session
6648                    .negotiated_capabilities
6649                    .read()
6650                    .map(|caps| caps.contains(car_proto::RUNS_RESUME_CAPABILITY))
6651                    .unwrap_or(false)
6652            });
6653            if resume_negotiated {
6654                let expires_at = tokio::time::Instant::now() + self.run_resume_lease;
6655                let mut runs = self.runs.lock().await;
6656                for meta in runs
6657                    .values_mut()
6658                    .filter(|meta| meta.active_client_id == client_id && !meta.is_terminal())
6659                {
6660                    meta.resume_lease = Some(RunResumeLease {
6661                        disconnected_client_id: client_id.to_string(),
6662                        expires_at,
6663                    });
6664                }
6665            }
6666            removed
6667        };
6668        if let Some(session) = &removed {
6669            // #169: drop the agent_id → client_id binding so a
6670            // disconnected lifecycle agent can reconnect (or its
6671            // supervisor-respawned replacement can take the slot)
6672            // without colliding with the stale claim.
6673            let bound = session.agent_id.lock().await.clone();
6674            if let Some(id) = bound {
6675                let mut attached = self.attached_agents.lock().await;
6676                if attached.get(&id).map(String::as_str) == Some(client_id) {
6677                    attached.remove(&id);
6678                }
6679            }
6680            // Drop the peer-messaging guard with the binding. The guard holds a
6681            // rate budget and a dedupe window keyed by agent *name*, so a
6682            // supervisor-respawned replacement taking the same name would
6683            // otherwise inherit the dead process's history — arriving
6684            // pre-throttled, or silently deduped against messages it never saw.
6685            let bound_for_guards = session.agent_id.lock().await.clone();
6686            if let Some(id) = bound_for_guards {
6687                self.peer_guards.lock().await.remove(&id);
6688            }
6689            // Drop any in-flight `agents.chat` sessions bound to this
6690            // client — either side disconnecting orphans the stream,
6691            // and a respawned agent's stray `agent.chat.event`
6692            // notifications must not race into a stale routing entry.
6693            // See `docs/proposals/agent-chat-surface.md`.
6694            let bound_agent = session.agent_id.lock().await.clone();
6695            let mut chats = self.chat_sessions.lock().await;
6696            chats.retain(|_, s| {
6697                if s.host_client_id == client_id {
6698                    return false;
6699                }
6700                if let Some(agent_id) = &bound_agent {
6701                    if &s.agent_id == agent_id {
6702                        return false;
6703                    }
6704                }
6705                true
6706            });
6707            // Drop the chat lock before the run sweep, which awaits the
6708            // grace window and re-locks `runs` — keeping locks disjoint
6709            // avoids holding `chat_sessions` across the sleep.
6710            drop(chats);
6711            // Same for in-process A2A chat collectors owned by this client.
6712            // Dropping the entry drops the sole sender, so the collecting task's
6713            // `recv()` returns `None` and it falls back immediately instead of
6714            // waiting out its 180s cap. Done after `drop(chats)` so we never
6715            // hold `chat_sessions` while locking `chat_collectors` (the
6716            // interceptor locks them collectors→sessions; staying disjoint here
6717            // avoids inverting that order).
6718            self.chat_collectors
6719                .lock()
6720                .await
6721                .retain(|_, c| c.host_client_id != client_id);
6722            // Agent run tracing (U4): drop this connection's live
6723            // run-trace subscriptions so its drain tasks end and the
6724            // registry doesn't accumulate stale `(run_id, client)` entries
6725            // across reconnects. Reconnect-durability is client-side: the
6726            // run stays subscribable; CarHost re-subscribes by `run_id` on
6727            // its new connection (R8). This is exempt from the chat
6728            // drain-and-synthesize-error path — a dropped trace subscriber
6729            // never marks the underlying run failed.
6730            self.drop_run_subscribers_for_client(client_id).await;
6731            // Browser-drawer subscriptions are per-connection too. Dropping
6732            // them also starts the control grace period on any view this
6733            // connection was DRIVING, so an agent is never parked forever
6734            // behind a controller whose app just quit (see
6735            // `browser_view::CONTROL_GRACE`).
6736            self.browser_views
6737                .drop_subscriptions_for_client(client_id)
6738                .await;
6739            // Only a host-client removal can change host connectivity. This
6740            // used to fan a redundant reverse call to every producer for every
6741            // agent/CLI disconnect too. The session is already out of
6742            // `sessions`, so a real host removal reads the post-removal truth.
6743            if session.is_host.load(std::sync::atomic::Ordering::Acquire) {
6744                let host_connected = self.any_host_connected().await;
6745                self.browser_views
6746                    .broadcast_host_connected(host_connected)
6747                    .await;
6748            }
6749            // And if this connection was a supervised agent PUBLISHING a
6750            // browser, that browser went with the process. Its views stay
6751            // registered reporting an empty browser, so the drawer sees the
6752            // change and a restarted process can replace them in place (see
6753            // `browser_relay::RelayProducer::note_disconnected`).
6754            self.browser_views
6755                .note_producer_disconnected(client_id)
6756                .await;
6757            // Coder event subscriptions are per-connection too; the session
6758            // itself keeps running (reconnect + `coder.subscribe { from_seq }`
6759            // replays what was missed).
6760            // Board watchers (`coder.watch`) and discussion streams are
6761            // per-connection too, and are dropped on the same boundary.
6762            crate::coder::rpc::drop_subscriptions_for_client(self, client_id).await;
6763            crate::coder::discuss::drop_subscriptions_for_client(self, client_id).await;
6764            // Agent run tracing (R5): any run this connection owns that
6765            // has no terminal record yet is swept to `Incomplete` after
6766            // a short grace window — long enough for an in-flight
6767            // `runs.complete` to land first so a healthy close is never
6768            // mislabeled.
6769            let _run_guard = session.run_lifecycle_guard.lock().await;
6770            self.sweep_runs_for_disconnect(session).await;
6771        }
6772        removed
6773    }
6774}
6775
6776#[cfg(test)]
6777mod durable_task_tests {
6778    use super::*;
6779
6780    #[tokio::test]
6781    async fn dropping_response_waiter_does_not_cancel_operation_or_release_its_guard() {
6782        let temp = tempfile::tempdir().unwrap();
6783        let state = Arc::new(ServerState::with_config(ServerStateConfig::new(
6784            temp.path().to_path_buf(),
6785        )));
6786        let overlap_guard = Arc::new(tokio::sync::Mutex::new(()));
6787        let operation_guard = overlap_guard.clone();
6788        let (started_tx, started_rx) = tokio::sync::oneshot::channel();
6789        let (release_tx, release_rx) = tokio::sync::oneshot::channel();
6790
6791        let response = state
6792            .spawn_durable_operation("auth.test", async move {
6793                let _held = operation_guard.lock().await;
6794                let _ = started_tx.send(());
6795                let _ = release_rx.await;
6796                Ok::<_, String>(serde_json::json!({ "ok": true }))
6797            })
6798            .await;
6799        started_rx.await.unwrap();
6800        drop(response);
6801
6802        assert!(
6803            overlap_guard.try_lock().is_err(),
6804            "dropping only the connection waiter must not release the operation guard"
6805        );
6806        release_tx.send(()).unwrap();
6807        let joined = tokio::time::timeout(
6808            std::time::Duration::from_secs(1),
6809            state.durable_tasks.lock().await.join_next(),
6810        )
6811        .await
6812        .expect("daemon-owned operation should finish")
6813        .expect("task should exist")
6814        .expect("task should join");
6815        assert_eq!(joined, "auth.test");
6816        assert!(
6817            overlap_guard.try_lock().is_ok(),
6818            "the guard releases only after the daemon-owned operation finishes"
6819        );
6820    }
6821}
6822
6823#[cfg(test)]
6824mod tool_timeout_tests {
6825    use super::*;
6826
6827    #[test]
6828    fn honors_action_timeout_over_default() {
6829        // The action's budget bounds the wait (+grace so the executor's own
6830        // deadline reaps first) — no 60s ceiling, no env read.
6831        assert_eq!(
6832            tool_callback_timeout(Some(180_000)),
6833            std::time::Duration::from_millis(180_000 + TOOL_TIMEOUT_GRACE_MS)
6834        );
6835        // A budget far above the old 60s ceiling is honored (the #259 bug),
6836        // and the wait always exceeds the budget (so the executor wins).
6837        assert!(tool_callback_timeout(Some(600_000)) > std::time::Duration::from_secs(600));
6838        assert!(tool_callback_timeout(Some(180_000)) >= std::time::Duration::from_millis(180_000));
6839    }
6840
6841    #[test]
6842    fn default_is_not_the_old_60s() {
6843        // With no action budget and no env override, the fallback is the
6844        // raised default — not the hardcoded 60s that reaped real tools.
6845        // (Asserted only when CAR_TOOL_TIMEOUT is unset, which it is in CI.)
6846        if std::env::var_os("CAR_TOOL_TIMEOUT").is_none() {
6847            assert_eq!(
6848                tool_callback_timeout(None),
6849                std::time::Duration::from_millis(DEFAULT_TOOL_TIMEOUT_MS)
6850            );
6851            assert!(
6852                DEFAULT_TOOL_TIMEOUT_MS > 60_000,
6853                "default must exceed the old 60s"
6854            );
6855        }
6856    }
6857}
6858
6859#[cfg(test)]
6860mod observer_mode_tests {
6861    use super::*;
6862
6863    fn journal_dir() -> PathBuf {
6864        let target = std::env::var_os("CARGO_TARGET_DIR")
6865            .map(std::path::PathBuf::from)
6866            .unwrap_or_else(|| {
6867                std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
6868                    .join("..")
6869                    .join("..")
6870                    .join("target")
6871            });
6872        std::fs::create_dir_all(&target).ok();
6873        let target = std::fs::canonicalize(&target).unwrap_or(target);
6874        let tmp = tempfile::TempDir::new_in(&target).unwrap();
6875        let p = tmp.path().to_path_buf();
6876        std::mem::forget(tmp); // keep the dir alive for the test
6877        p
6878    }
6879
6880    #[test]
6881    fn supervisor_returns_observer_error_when_marker_set() {
6882        // Closes Parslee-ai/car-releases#44: the second car-server on
6883        // a host installs the observer marker after `with_paths`
6884        // returns AlreadyRunning. Subsequent `state.supervisor()`
6885        // calls must return a clear "observe-only" error mentioning
6886        // the manifest path — they must NOT retry user_default()
6887        // (which would re-acquire the lock and likely also fail).
6888        let state = ServerState::standalone(journal_dir());
6889        let fake_manifest = PathBuf::from("/tmp/fake-manifest-for-test.json");
6890        state
6891            .install_observer_manifest(fake_manifest.clone())
6892            .expect("install_observer_manifest succeeds on fresh state");
6893        assert_eq!(state.observer_manifest_path(), Some(&fake_manifest));
6894
6895        let err = state.supervisor().map(|_| ()).unwrap_err();
6896        assert!(
6897            err.contains("observe-only"),
6898            "error must mention observe-only mode: {err}"
6899        );
6900        assert!(
6901            err.contains("fake-manifest-for-test.json"),
6902            "error must surface the manifest path so operators know which daemon owns it: {err}"
6903        );
6904    }
6905
6906    #[test]
6907    fn install_observer_manifest_is_idempotent_per_path_collision() {
6908        let state = ServerState::standalone(journal_dir());
6909        let p = PathBuf::from("/tmp/manifest-a.json");
6910        let q = PathBuf::from("/tmp/manifest-b.json");
6911        state.install_observer_manifest(p.clone()).unwrap();
6912        // OnceLock::set returns the value back on collision.
6913        let err = state.install_observer_manifest(q.clone()).unwrap_err();
6914        assert_eq!(err, q);
6915        assert_eq!(state.observer_manifest_path(), Some(&p));
6916    }
6917
6918    #[test]
6919    fn supervisor_if_installed_does_not_lazy_init() {
6920        // The Heisenberg-subscribe guard: `host.subscribe`'s
6921        // identity path must use the non-acquiring read so a
6922        // purely observational client can't cause the daemon to
6923        // claim `<manifest>.lock` as a side effect of asking
6924        // about it. Fresh state has no supervisor installed.
6925        let state = ServerState::standalone(journal_dir());
6926        assert!(state.supervisor_if_installed().is_none());
6927        // observer_manifest_path should remain unset too — no
6928        // implicit init.
6929        assert!(state.observer_manifest_path().is_none());
6930    }
6931}
6932
6933#[cfg(test)]
6934mod substrate_binding_tests {
6935    use super::*;
6936    use car_engine::{McpSession, McpToolInfo};
6937    use car_ir::ActionProposal;
6938    use serde_json::json;
6939    use std::sync::Mutex as StdMutex;
6940
6941    fn journal_dir() -> PathBuf {
6942        let target = std::env::var_os("CARGO_TARGET_DIR")
6943            .map(std::path::PathBuf::from)
6944            .unwrap_or_else(|| {
6945                std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
6946                    .join("..")
6947                    .join("..")
6948                    .join("target")
6949            });
6950        std::fs::create_dir_all(&target).ok();
6951        let target = std::fs::canonicalize(&target).unwrap_or(target);
6952        let tmp = tempfile::TempDir::new_in(&target).unwrap();
6953        let p = tmp.path().to_path_buf();
6954        std::mem::forget(tmp);
6955        p
6956    }
6957
6958    /// A minimal in-memory MCP session standing in for a connected `vm`
6959    /// connector. `read_text`/`write_text`/`run_command` behave like the
6960    /// `vm` bridge so we can assert a substrate-bound session's built-ins
6961    /// land here rather than on the WS client (which has no responder in a
6962    /// test and would hang/fail).
6963    struct FakeVmSession {
6964        name: String,
6965        files: Arc<StdMutex<std::collections::HashMap<String, String>>>,
6966    }
6967
6968    #[async_trait::async_trait]
6969    impl McpSession for FakeVmSession {
6970        async fn list_tools(&mut self) -> Result<Vec<McpToolInfo>, String> {
6971            Ok(vec![])
6972        }
6973        async fn call_tool(
6974            &mut self,
6975            name: &str,
6976            args: serde_json::Value,
6977        ) -> Result<serde_json::Value, String> {
6978            match name {
6979                "write_text" => {
6980                    let p = args["path"].as_str().unwrap().to_string();
6981                    let c = args["content"].as_str().unwrap().to_string();
6982                    self.files.lock().unwrap().insert(p, c);
6983                    Ok(json!("ok"))
6984                }
6985                "read_text" => {
6986                    let p = args["path"].as_str().unwrap();
6987                    let c = self
6988                        .files
6989                        .lock()
6990                        .unwrap()
6991                        .get(p)
6992                        .cloned()
6993                        .ok_or_else(|| "not found".to_string())?;
6994                    Ok(json!(c))
6995                }
6996                "run_command" => {
6997                    let command = args["command"].as_str().unwrap_or_default();
6998                    if let Some(path) = command
6999                        .split_once("[ -e '")
7000                        .map(|(_, path)| path)
7001                        .and_then(|path| path.split_once("' ]").map(|(path, _)| path))
7002                    {
7003                        let exists = self.files.lock().unwrap().contains_key(path);
7004                        return Ok(json!({
7005                            "stdout": "",
7006                            "stderr": "",
7007                            "exit_code": if exists { 0 } else { 1 }
7008                        }));
7009                    }
7010                    Ok(json!({
7011                        "stdout": "from-vm",
7012                        "stderr": "",
7013                        "exit_code": 0
7014                    }))
7015                }
7016                other => Err(format!("unknown tool {other}")),
7017            }
7018        }
7019        fn name(&self) -> &str {
7020            &self.name
7021        }
7022    }
7023
7024    fn read_file_proposal(path: &str) -> ActionProposal {
7025        serde_json::from_value(json!({
7026            "source": "test",
7027            "actions": [{
7028                "id": "r0",
7029                "type": "tool_call",
7030                "tool": "read_file",
7031                "parameters": { "path": path },
7032                "dependencies": [],
7033            }],
7034        }))
7035        .expect("proposal deserializes")
7036    }
7037
7038    /// The outbound message sink is attached to every session runtime, so
7039    /// `messaging.send` is a real, executable tool rather than a schema the
7040    /// runtime would refuse. `with_message_sink` registers the sink and the
7041    /// schema together, so seeing the schema proves the sink is there.
7042    #[tokio::test]
7043    async fn session_runtime_has_the_messaging_send_tool() {
7044        let state = Arc::new(ServerState::standalone(journal_dir()));
7045        let session = state
7046            .create_session("c-messaging", Arc::new(WsChannel::test_stub()))
7047            .await
7048            .unwrap();
7049
7050        assert!(
7051            session
7052                .runtime
7053                .tools
7054                .read()
7055                .await
7056                .contains_key("messaging.send"),
7057            "every session must be able to execute messaging.send"
7058        );
7059    }
7060
7061    /// Default (no bind): the session runtime keeps a `LocalSubstrate`.
7062    /// This is the backward-compat invariant — every existing consumer
7063    /// (GUI, A2A-over-WS, connectors-as-tools, coder) sees no change.
7064    #[tokio::test]
7065    async fn default_session_substrate_is_local() {
7066        let state = Arc::new(ServerState::standalone(journal_dir()));
7067        let channel = Arc::new(WsChannel::test_stub());
7068        let session = state.create_session("c-default", channel).await.unwrap();
7069
7070        let sub = session.runtime.substrate().await;
7071        assert_eq!(
7072            sub.name(),
7073            "local",
7074            "an un-bound session must keep the default LocalSubstrate"
7075        );
7076        assert!(sub.is_local(), "default substrate must be the host");
7077    }
7078
7079    /// Binding fails cleanly when the named connector is not connected —
7080    /// no panic, a descriptive error, and the substrate stays Local.
7081    #[tokio::test]
7082    async fn bind_unknown_connector_errors_and_keeps_local() {
7083        let state = Arc::new(ServerState::standalone(journal_dir()));
7084        let channel = Arc::new(WsChannel::test_stub());
7085        let session = state.create_session("c-missing", channel).await.unwrap();
7086
7087        let err = state
7088            .bind_substrate_to_connector(&session, "nope")
7089            .await
7090            .unwrap_err();
7091        assert!(err.contains("not connected"), "got: {err}");
7092        assert_eq!(session.runtime.substrate().await.name(), "local");
7093    }
7094
7095    /// A connector-driven session that binds its substrate routes the bare
7096    /// commodity built-ins (`read_file`/`write_file`) to that connector's
7097    /// environment via the engine fall-through, NOT to the WS client. We
7098    /// prove it end-to-end: write+read through the runtime land on the fake
7099    /// VM session's file map.
7100    #[tokio::test]
7101    async fn bound_session_routes_builtins_to_substrate() {
7102        let state = Arc::new(ServerState::standalone(journal_dir()));
7103        let channel = Arc::new(WsChannel::test_stub());
7104        let session = state.create_session("c-vm", channel).await.unwrap();
7105
7106        // Register a connected "vm" connector session + its routes the way
7107        // the connector manager does on dial.
7108        let files = Arc::new(StdMutex::new(std::collections::HashMap::new()));
7109        let fake: Arc<Mutex<dyn McpSession>> = Arc::new(Mutex::new(FakeVmSession {
7110            name: "vm".into(),
7111            files: files.clone(),
7112        }));
7113        state.mcp_executor.add_session("vm", fake).await;
7114
7115        // Bind the session to the vm connector's substrate.
7116        let bound = state
7117            .bind_substrate_to_connector(&session, "vm")
7118            .await
7119            .expect("bind succeeds for a connected connector");
7120        assert_eq!(bound, "vm");
7121        assert_eq!(session.runtime.substrate().await.name(), "vm");
7122
7123        // write_file through the runtime → must hit the fake VM session,
7124        // not the WS client (which would hang/fail with no responder).
7125        let write = serde_json::from_value::<ActionProposal>(json!({
7126            "source": "test",
7127            "actions": [{
7128                "id": "w0",
7129                "type": "tool_call",
7130                "tool": "write_file",
7131                "parameters": { "path": "/vm/a.txt", "content": "vm-bytes" },
7132                "dependencies": [],
7133            }],
7134        }))
7135        .unwrap();
7136        let wres = session.runtime.execute(&write).await;
7137        assert!(
7138            wres.results[0].error.is_none(),
7139            "write_file via substrate failed: {:?}",
7140            wres.results[0].error
7141        );
7142        assert_eq!(
7143            files.lock().unwrap().get("/vm/a.txt").map(String::as_str),
7144            Some("vm-bytes"),
7145            "write_file must land on the bound VM substrate"
7146        );
7147
7148        // read_file back must return the VM's content.
7149        let rres = session
7150            .runtime
7151            .execute(&read_file_proposal("/vm/a.txt"))
7152            .await;
7153        assert!(
7154            rres.results[0].error.is_none(),
7155            "read_file via substrate failed: {:?}",
7156            rres.results[0].error
7157        );
7158        let out = rres.results[0].output.clone().unwrap_or(Value::Null);
7159        let content = out.get("content").and_then(|v| v.as_str()).unwrap_or("");
7160        // read_file output is line-numbered `cat -n` style (H1/F4-remainder,
7161        // audit 2026-07-06); the VM's single-line content becomes "     1\t<line>".
7162        assert_eq!(
7163            content, "     1\tvm-bytes",
7164            "read_file must come from the VM substrate"
7165        );
7166    }
7167
7168    /// The shadow executor: substrate-owned built-in names return the
7169    /// `"unknown tool"` prefix (so the engine falls through to the bound
7170    /// substrate), while any other tool name is delegated to the inner
7171    /// executor unchanged (connector `mcp_*` routes + host WS callbacks).
7172    #[tokio::test]
7173    async fn shadow_executor_only_shadows_builtin_names() {
7174        struct Inner;
7175        #[async_trait::async_trait]
7176        impl ToolExecutor for Inner {
7177            async fn execute(&self, tool: &str, _p: &Value) -> Result<Value, String> {
7178                Ok(json!({ "delegated": tool }))
7179            }
7180        }
7181        let shadow = SubstrateShadowExecutor::new(Arc::new(Inner));
7182
7183        for name in SUBSTRATE_OWNED_TOOLS {
7184            let err = shadow.execute(name, &json!({})).await.unwrap_err();
7185            assert!(
7186                err.starts_with("unknown tool"),
7187                "{name} must be shadowed so the engine falls through: {err}"
7188            );
7189        }
7190
7191        // A connector tool name and a pure tool pass through untouched.
7192        for passthrough in ["mcp_vm_run_command", "calculate", "browser.run"] {
7193            let out = shadow.execute(passthrough, &json!({})).await.unwrap();
7194            assert_eq!(out["delegated"], json!(passthrough));
7195        }
7196    }
7197}
7198
7199/// `.car/policies/*.toml` reaching the runtime at all — the whole point of
7200/// wiring `load_project_policies` to a real call site. Exercised against a
7201/// temp directory rather than through `create_session`, which would read the
7202/// developer's real `$HOME`.
7203#[cfg(test)]
7204mod project_policy_loading_tests {
7205    use super::*;
7206
7207    fn runtime() -> Runtime {
7208        Runtime::new()
7209    }
7210
7211    #[tokio::test]
7212    async fn a_missing_policies_directory_is_silent() {
7213        let dir = tempfile::tempdir().unwrap();
7214        // No `.car` at all, let alone `.car/policies`.
7215        apply_project_policies(&runtime(), &dir.path().join(".car"))
7216            .await
7217            .expect("a project with no rules must start normally");
7218    }
7219
7220    #[tokio::test]
7221    async fn an_empty_policies_directory_is_silent() {
7222        let dir = tempfile::tempdir().unwrap();
7223        let car = dir.path().join(".car");
7224        std::fs::create_dir_all(car.join("policies")).unwrap();
7225        apply_project_policies(&runtime(), &car).await.unwrap();
7226    }
7227
7228    #[tokio::test]
7229    async fn a_well_formed_rule_file_loads_and_takes_effect() {
7230        let dir = tempfile::tempdir().unwrap();
7231        let car = dir.path().join(".car");
7232        std::fs::create_dir_all(car.join("policies")).unwrap();
7233        // Exactly the shape `car_policy::rules`' module docs advertise: a
7234        // top-level list for `deny_tool`, a table array for the per-parameter
7235        // rules.
7236        std::fs::write(
7237            car.join("policies").join("messaging.toml"),
7238            "deny_tool = [\"messaging.send\"]\n\n\
7239             [[deny_tool_param]]\n\
7240             tool = \"messaging.send\"\n\
7241             param = \"channel\"\n\
7242             equals = \"slack\"\n",
7243        )
7244        .unwrap();
7245
7246        let rt = runtime();
7247        apply_project_policies(&rt, &car).await.unwrap();
7248
7249        // Registered on the runtime's engine, not merely parsed.
7250        let names: Vec<String> = rt
7251            .list_policies(None)
7252            .await
7253            .unwrap()
7254            .into_iter()
7255            .map(|(name, _)| name)
7256            .collect();
7257        assert!(
7258            names.len() >= 2,
7259            "both declarative rules must reach the policy engine, got {names:?}"
7260        );
7261    }
7262
7263    #[tokio::test]
7264    async fn a_malformed_rule_file_fails_loudly_and_names_the_file() {
7265        let dir = tempfile::tempdir().unwrap();
7266        let car = dir.path().join(".car");
7267        std::fs::create_dir_all(car.join("policies")).unwrap();
7268        std::fs::write(
7269            car.join("policies").join("broken.toml"),
7270            "[[deny_tool]\ntool = \"shell\"",
7271        )
7272        .unwrap();
7273
7274        let err = apply_project_policies(&runtime(), &car)
7275            .await
7276            .expect_err("a malformed policy file must NOT be swallowed");
7277        assert!(
7278            err.contains("broken.toml"),
7279            "the operator has to know which file to fix: {err}"
7280        );
7281        assert!(err.contains("refusing to start"), "{err}");
7282    }
7283}
7284
7285#[cfg(test)]
7286mod org_scope_wiring_tests {
7287    use super::*;
7288
7289    fn granter_hex_for(secret: &[u8], user: &str) -> String {
7290        let master = car_sync::StretchedMaster::from_issued_high_entropy(secret, user);
7291        let vk = car_sync::ed25519_verifying(&car_sync::derive_ed25519_identity(&master, user));
7292        vk.to_bytes().iter().map(|b| format!("{b:02x}")).collect()
7293    }
7294    fn granter_hex() -> String {
7295        granter_hex_for(b"granter-login", "acc_granter")
7296    }
7297
7298    #[test]
7299    fn unset_env_is_the_personal_only_path() {
7300        // The real-tenants-OFF invariant: no value → Ok(None) → base provider.
7301        assert!(ServerState::parse_org_scope_config(None).unwrap().is_none());
7302        assert!(ServerState::parse_org_scope_config(Some(String::new()))
7303            .unwrap()
7304            .is_none());
7305    }
7306
7307    #[test]
7308    fn wellformed_env_parses_org_and_granter() {
7309        let hex = granter_hex();
7310        let (org, granters) = ServerState::parse_org_scope_config(Some(format!("orgtest:{hex}")))
7311            .unwrap()
7312            .expect("well-formed opt-in parses");
7313        assert_eq!(org, "orgtest");
7314        assert_eq!(granters.len(), 1);
7315        let expected_master =
7316            car_sync::StretchedMaster::from_issued_high_entropy(b"granter-login", "acc_granter");
7317        let expected = car_sync::ed25519_verifying(&car_sync::derive_ed25519_identity(
7318            &expected_master,
7319            "acc_granter",
7320        ));
7321        assert_eq!(granters[0].to_bytes(), expected.to_bytes());
7322    }
7323
7324    #[test]
7325    fn multiple_granters_parse_in_order() {
7326        // The trusted set is plural: an org can designate several granters.
7327        let g1 = granter_hex_for(b"g1", "acc_g1");
7328        let g2 = granter_hex_for(b"g2", "acc_g2");
7329        let (org, granters) =
7330            ServerState::parse_org_scope_config(Some(format!("orgtest:{g1},{g2}")))
7331                .unwrap()
7332                .expect("well-formed multi-granter opt-in parses");
7333        assert_eq!(org, "orgtest");
7334        assert_eq!(granters.len(), 2);
7335        assert_ne!(granters[0].to_bytes(), granters[1].to_bytes());
7336        // Surrounding whitespace on a granter entry is tolerated.
7337        let spaced = ServerState::parse_org_scope_config(Some(format!("orgtest:{g1} , {g2}")))
7338            .unwrap()
7339            .unwrap();
7340        assert_eq!(spaced.1.len(), 2);
7341    }
7342
7343    #[test]
7344    fn one_bad_granter_in_the_list_fails_the_whole_value() {
7345        // A typo in ANY granter is a loud failure — never silently drop it and
7346        // proceed with a partial trusted set.
7347        let good = granter_hex();
7348        assert!(
7349            ServerState::parse_org_scope_config(Some(format!("orgtest:{good},not-hex"))).is_err(),
7350            "a malformed granter among valid ones must be a hard error"
7351        );
7352        // An org id with no granters at all is rejected.
7353        assert!(ServerState::parse_org_scope_config(Some("orgtest:".into())).is_err());
7354        assert!(ServerState::parse_org_scope_config(Some("orgtest: , ".into())).is_err());
7355    }
7356
7357    #[test]
7358    fn malformed_env_is_a_hard_error_never_silent() {
7359        // A typo must NOT silently fall through to a wrong/unauthenticated state.
7360        let hex = granter_hex();
7361        let bads = [
7362            "no-colon-here".to_string(),   // missing ':'
7363            ":deadbeef".to_string(),       // empty org
7364            "orgtest:not-hex".to_string(), // bad hex
7365            "orgtest:aabb".to_string(),    // wrong length
7366            format!("bad org:{hex}"),      // non-canonical org (space)
7367            format!("org/evil:{hex}"),     // non-canonical org (delimiter)
7368        ];
7369        for bad in &bads {
7370            assert!(
7371                ServerState::parse_org_scope_config(Some(bad.clone())).is_err(),
7372                "malformed value {bad:?} must be a hard error"
7373            );
7374        }
7375        // NOTE: an all-zero (identity/small-order) key PARSES — from_bytes accepts
7376        // the encoding; verify_strict rejects it at verification, so such a granter
7377        // simply grants nothing (fail-closed), which is correct, not a parse error.
7378    }
7379
7380    #[test]
7381    fn scope_routing_sends_only_matching_org_to_the_org_subsystem() {
7382        use car_sync::Scope;
7383        let acme = Scope::Shared { org: "acme".into() };
7384        let globex = Scope::Shared {
7385            org: "globex".into(),
7386        };
7387
7388        // org-scope OFF (no holder): EVERYTHING routes to the personal subsystem —
7389        // the byte-identical invariant.
7390        assert_eq!(route_for_scope(&Scope::Personal, None), SyncRoute::User);
7391        assert_eq!(route_for_scope(&acme, None), SyncRoute::User);
7392
7393        // opted into "acme": only Scope::Shared{acme} routes to the org subsystem.
7394        assert_eq!(route_for_scope(&acme, Some("acme")), SyncRoute::Org);
7395        assert_eq!(
7396            route_for_scope(&Scope::Personal, Some("acme")),
7397            SyncRoute::User
7398        );
7399        assert_eq!(
7400            route_for_scope(&globex, Some("acme")),
7401            SyncRoute::User,
7402            "a different org must NOT go to acme's delivery subsystem"
7403        );
7404    }
7405}