Skip to main content

car_server_core/
session.rs

1//! Server-side session state — shared across all connections.
2
3use car_engine::{AdmissionGate, GateContext, GateOutcome, Runtime, ToolExecutor};
4use car_eventlog::EventLog;
5use car_ir::ToolFailure;
6use car_proto::{ToolCancelRequest, ToolExecuteRequest, ToolExecuteResponse};
7use futures::Sink;
8use serde::{Deserialize, Serialize};
9use serde_json::Value;
10use sha2::{Digest, Sha256};
11use std::collections::HashMap;
12use std::path::{Path, PathBuf};
13use std::pin::Pin;
14use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
15use std::sync::Arc;
16use tokio::sync::{mpsc, oneshot, Mutex};
17use tokio_tungstenite::tungstenite::{Error as WsError, Message};
18
19/// Type-erased WebSocket sink. The dispatch loop accepts either a
20/// `WebSocketStream<TcpStream>` (the legacy car-server TCP listener)
21/// or a `WebSocketStream<UnixStream>` (the daemon-as-default UDS
22/// listener) — both implement `Sink<Message, Error = WsError>` after
23/// the tungstenite handshake. Erasing the type here avoids cascading
24/// a generic parameter through every WsChannel / Session / ServerState
25/// touchpoint in the dispatcher.
26pub type WsSink = Pin<Box<dyn Sink<Message, Error = WsError> + Send + Unpin + 'static>>;
27
28/// Grace window applied on disconnect before a still-open run is marked
29/// `Incomplete` (agent run tracing, U1 — R5). Short on purpose: it only
30/// has to cover the gap between a healthy `runs.complete` being
31/// dispatched on a spawned task and its terminal record landing, not any
32/// real work. Long enough to absorb that scheduling jitter, short enough
33/// that a genuinely abandoned run is reported `Incomplete` promptly.
34pub const RUN_COMPLETE_GRACE: std::time::Duration = std::time::Duration::from_millis(250);
35
36/// Bounded in-process lease for a disconnected run whose producer negotiated
37/// `runs.resume.v1`. Ten seconds leaves practical headroom for a loaded Mac's
38/// Python controller to establish a fresh local socket, authenticate, and
39/// negotiate capabilities without turning this into restart recovery. Runs
40/// without the capability keep the original 250 ms behavior; an unclaimed
41/// resumable run still becomes `Incomplete`.
42pub const RUN_RESUME_LEASE: std::time::Duration = std::time::Duration::from_secs(10);
43
44/// Maximum time an async run-lifecycle mutation waits for the journal writer's
45/// durability acknowledgement. Five seconds matches the server's existing
46/// acknowledgement/grace budget and remains well inside the default mutating
47/// handler deadline. If an operator override cancels earlier, the async append
48/// still marks the exact row pending before its first yield.
49const CRITICAL_JOURNAL_ACKNOWLEDGEMENT_TIMEOUT: std::time::Duration =
50    std::time::Duration::from_secs(5);
51
52/// Startup uses the same durability budget as live lifecycle mutations. A
53/// stalled journal must fail the constructor before the daemon listens rather
54/// than occupying the startup thread forever.
55const STARTUP_RECONCILIATION_ACKNOWLEDGEMENT_TIMEOUT: std::time::Duration =
56    CRITICAL_JOURNAL_ACKNOWLEDGEMENT_TIMEOUT;
57
58/// Per-run observation bound for disconnect cleanup: the ten-second resume
59/// lease, two bounded five-second journal acknowledgement attempts (the second
60/// is an exact retry after durability-unknown), and scheduler headroom. Cleanup
61/// never clears the run or binding unless an acknowledgement arrives.
62pub const RUN_DISCONNECT_CLEANUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(22);
63
64const RUN_SUBSCRIBE_SUMMARY_STATE_RETRY_LIMIT: usize = 3;
65
66/// Stable admission reason returned after a callback terminally halts its
67/// WebSocket session. The halt is intentionally connection-local and dies on
68/// reconnect; durable run outcomes remain the responsibility of `runs.complete`.
69pub const SESSION_HALTED_REASON: &str =
70    "session halted by a terminal tool failure; reconnect or ask a host to call session.clear_halt";
71
72struct SessionHaltAdmissionGate {
73    halted: Arc<AtomicBool>,
74}
75
76#[async_trait::async_trait]
77impl AdmissionGate for SessionHaltAdmissionGate {
78    fn name(&self) -> &str {
79        "session_halt"
80    }
81
82    async fn check(
83        &self,
84        _proposal: &car_ir::ActionProposal,
85        _ctx: &GateContext<'_>,
86    ) -> GateOutcome {
87        if self.halted.load(Ordering::Acquire) {
88            GateOutcome::reject_all(SESSION_HALTED_REASON)
89        } else {
90            GateOutcome::Allow
91        }
92    }
93}
94
95/// Server-side credentials for continuing an A2A-owned A2UI surface.
96///
97/// This intentionally lives outside `car_a2ui::A2uiSurfaceOwner` so
98/// renderers can inspect surface ownership without receiving secrets.
99#[derive(Debug, Clone, Serialize, Deserialize)]
100#[serde(rename_all = "camelCase", tag = "type")]
101pub enum A2aRouteAuth {
102    None,
103    Bearer { token: String },
104    Header { name: String, value: String },
105}
106
107#[cfg(test)]
108mod run_reservation_liveness_tests {
109    use super::*;
110    use crate::run_store::RunStoreLookupGate;
111    use std::time::Duration;
112
113    fn pristine_run(run_id: &str, client_id: &str) -> RunMeta {
114        RunMeta {
115            run_id: run_id.to_string(),
116            agent_id: "agent-release".to_string(),
117            client_id: client_id.to_string(),
118            active_client_id: client_id.to_string(),
119            resume_predecessor_client_id: None,
120            resume_lease: None,
121            intent: "release without blocking the registry".to_string(),
122            outcome_description: None,
123            started_at: chrono::Utc::now(),
124            termination: None,
125            ended_at: None,
126            turns: Vec::new(),
127            start_committed: false,
128            pending_terminal: None,
129            cancellation_pending: None,
130            cancellation_receipt: None,
131            trace_corruption: None,
132            durability_generation: 0,
133        }
134    }
135
136    #[tokio::test]
137    async fn blocked_release_lookup_does_not_hold_global_runs_lock() {
138        let tmp = tempfile::TempDir::new().unwrap();
139        let gate = RunStoreLookupGate::default();
140        let state = Arc::new(ServerState::with_config(
141            ServerStateConfig::new(tmp.path().join("journals"))
142                .with_run_store_lookup_gate(gate.clone()),
143        ));
144        let session = state
145            .create_session("release-client", Arc::new(WsChannel::test_stub()))
146            .await
147            .unwrap();
148        let expected = pristine_run("release-candidate", &session.client_id);
149        assert!(matches!(
150            state.reserve_run(expected.clone()).await,
151            Ok(RunReservation::New)
152        ));
153
154        gate.block_next();
155        let release_state = state.clone();
156        let release_session = session.clone();
157        let release_expected = expected.clone();
158        let release = tokio::spawn(async move {
159            release_state
160                .release_unpersisted_run_reservation(&release_session, &release_expected)
161                .await
162        });
163        let wait_gate = gate.clone();
164        assert!(
165            tokio::task::spawn_blocking(move || {
166                wait_gate.wait_until_entered(Duration::from_secs(3))
167            })
168            .await
169            .unwrap(),
170            "release must reach the blocked durable lookup"
171        );
172
173        let registry = tokio::time::timeout(Duration::from_secs(1), state.runs.lock())
174            .await
175            .expect("blocked release lookup must not hold the global runs lock");
176        drop(registry);
177        gate.release();
178        assert_eq!(release.await.unwrap(), Ok(()));
179        assert!(state.run_meta(&expected.run_id).await.is_none());
180    }
181}
182
183/// Shared write half of the WebSocket, plus pending callback channels.
184/// `write` is type-erased via [`WsSink`] so the dispatcher can run
185/// against any transport-specific WebSocketStream (TCP or UDS today;
186/// axum-bridged in future) without templatizing every consumer.
187pub struct WsChannel {
188    pub write: Mutex<WsSink>,
189    /// Pending tool execution callbacks: request_id → oneshot sender
190    pub pending: Mutex<HashMap<String, oneshot::Sender<ToolExecuteResponse>>>,
191    /// Body-free correlation for active callbacks. A session owns at most one
192    /// active run, so every entry belongs to that run while its lifecycle
193    /// guard is held by proposal execution.
194    pub active_actions: Mutex<HashMap<String, String>>,
195    pub next_id: AtomicU64,
196}
197
198/// A WS connection is a host event sink (#418): `HostState` broadcasts
199/// `host.event` frames to subscribers through this trait, so it never has to
200/// know about the concrete `WsChannel` / tungstenite types.
201#[async_trait::async_trait]
202impl crate::host::EventSubscriber for WsChannel {
203    async fn send_text(&self, json: String) {
204        use futures::SinkExt;
205        // Bound both lock acquisition and the socket write. Host events are
206        // emitted from reducer transitions; a suspended client must not hold
207        // that reducer's announcement lock (and browser controls behind it)
208        // indefinitely through TCP backpressure.
209        let _ = tokio::time::timeout(std::time::Duration::from_secs(10), async {
210            self.write
211                .lock()
212                .await
213                .send(Message::Text(json.into()))
214                .await
215        })
216        .await;
217    }
218}
219
220impl WsChannel {
221    pub fn next_request_id(&self) -> String {
222        let id = self.next_id.fetch_add(1, Ordering::SeqCst);
223        format!("cb-{}", id)
224    }
225
226    /// Test-only stub that returns a WsChannel whose write sink drains
227    /// to nowhere. Used by `host.rs` tests that need a real
228    /// `Arc<WsChannel>` in the subscribers map (to exercise membership
229    /// checks like the cross-session resolve fan-out) without
230    /// constructing a tungstenite handshake. Never writes are
231    /// performed against this stub; if anything tries, the drain sink
232    /// quietly absorbs.
233    #[cfg(test)]
234    pub fn test_stub() -> Self {
235        use futures::sink::SinkExt;
236        let sink: WsSink = Box::pin(
237            futures::sink::drain()
238                .sink_map_err(|_| tokio_tungstenite::tungstenite::Error::ConnectionClosed),
239        );
240        WsChannel {
241            write: Mutex::new(sink),
242            pending: Mutex::new(HashMap::new()),
243            active_actions: Mutex::new(HashMap::new()),
244            next_id: AtomicU64::new(0),
245        }
246    }
247
248    /// Test-only channel whose write sink is *observable*: every
249    /// `Message::Text` payload written to it is appended to the returned
250    /// shared Vec. `test_stub` drains to nowhere, which makes it useless
251    /// for asserting that a specific frame — e.g. the `tools.cancel`
252    /// notification — actually reached the wire.
253    ///
254    /// The Vec is behind a `std::sync::Mutex` (not tokio's) because the
255    /// sink closure is synchronous; the lock is taken and released inside
256    /// the closure before the returned future is handed back, so a test
257    /// holding the lock to assert can never deadlock a concurrent write.
258    #[cfg(test)]
259    pub fn test_capture() -> (Self, std::sync::Arc<std::sync::Mutex<Vec<String>>>) {
260        let frames = std::sync::Arc::new(std::sync::Mutex::new(Vec::<String>::new()));
261        let sink: WsSink = Box::pin(futures::sink::unfold(
262            frames.clone(),
263            |frames: std::sync::Arc<std::sync::Mutex<Vec<String>>>, msg: Message| {
264                if let Message::Text(text) = &msg {
265                    // Scoped so the lock is released before we yield the
266                    // (already-ready) future back to the sink machinery.
267                    if let Ok(mut held) = frames.lock() {
268                        held.push(text.as_str().to_string());
269                    }
270                }
271                futures::future::ready(Ok::<_, WsError>(frames))
272            },
273        ));
274        (
275            WsChannel {
276                write: Mutex::new(sink),
277                pending: Mutex::new(HashMap::new()),
278                active_actions: Mutex::new(HashMap::new()),
279                next_id: AtomicU64::new(0),
280            },
281            frames,
282        )
283    }
284}
285
286/// In-flight `agents.chat` session bookkeeping. Created when a host
287/// client calls `agents.chat`, removed when the agent emits a terminal
288/// `agent.chat.event` (`kind: "done"` or `"error"`), when either side
289/// disconnects, or when the host cancels via `agents.chat.cancel`.
290///
291/// The session_id is host-supplied (or server-generated when omitted)
292/// and threads through every `agent.chat.event` notification so the
293/// server can route streamed deltas back to the originating host
294/// without needing per-session subscriptions. See
295/// `docs/proposals/agent-chat-surface.md` for the wire contract.
296#[derive(Debug, Clone)]
297pub struct ChatSession {
298    /// Agent that owns this chat — populated from
299    /// `attached_agents` at `agents.chat` dispatch time.
300    pub agent_id: String,
301    /// Client id of the host that issued `agents.chat`. The server
302    /// forwards `agent.chat.event` notifications back to *this* host
303    /// only, so two CarHost windows chatting with the same agent are
304    /// independent streams.
305    pub host_client_id: String,
306    /// Unix-seconds creation time — used by the future stale-session
307    /// sweeper to drop sessions whose agent died without emitting a
308    /// terminal event.
309    pub created_at: u64,
310    /// In-process cancellation flag for daemon-owned chat runs. External
311    /// attached agents receive `agent.chat.cancel` instead.
312    pub local_cancel: Option<Arc<AtomicBool>>,
313}
314
315/// Host-visible standing goal for a chat session. `goal.set` stores this keyed
316/// by `session_id`; `agents.chat` reuses it for that session when the turn does
317/// not pass an inline `goal`.
318#[derive(Debug, Clone, Serialize, Deserialize)]
319pub struct ChatGoalState {
320    pub session_id: String,
321    pub check: String,
322    pub max_iterations: u32,
323    pub status: String,
324    pub last_iteration: Option<u32>,
325    pub last_met: Option<bool>,
326    pub last_grounded: Option<bool>,
327    pub last_reason: Option<String>,
328    pub terminal_kind: Option<String>,
329    pub terminal_message: Option<String>,
330    pub updated_at: u64,
331}
332
333fn chat_goals_path_from_journal_dir(journal_dir: &Path) -> PathBuf {
334    let car_dir = journal_dir
335        .parent()
336        .map(PathBuf::from)
337        .unwrap_or_else(|| PathBuf::from("."));
338    car_dir.join("chat-goals.json")
339}
340
341fn load_chat_goals_from_disk(journal_dir: &Path) -> HashMap<String, ChatGoalState> {
342    let path = chat_goals_path_from_journal_dir(journal_dir);
343    let Ok(text) = std::fs::read_to_string(&path) else {
344        return HashMap::new();
345    };
346    match serde_json::from_str::<HashMap<String, ChatGoalState>>(&text) {
347        Ok(mut goals) => {
348            for goal in goals.values_mut() {
349                if goal.status == "running" {
350                    goal.status = "active".to_string();
351                }
352            }
353            goals
354        }
355        Err(e) => {
356            tracing::warn!(
357                path = %path.display(),
358                error = %e,
359                "chat-goals store was unreadable; starting with an empty in-memory goal registry"
360            );
361            HashMap::new()
362        }
363    }
364}
365
366fn write_chat_goals_atomic(
367    path: &std::path::Path,
368    goals: &HashMap<String, ChatGoalState>,
369) -> std::io::Result<()> {
370    use std::sync::atomic::{AtomicU64, Ordering};
371    static SEQ: AtomicU64 = AtomicU64::new(0);
372    if let Some(parent) = path.parent() {
373        std::fs::create_dir_all(parent)?;
374    }
375    let json = serde_json::to_vec_pretty(goals)?;
376    let seq = SEQ.fetch_add(1, Ordering::Relaxed);
377    let mut tmp_os = path.as_os_str().to_owned();
378    tmp_os.push(format!(".tmp.{}.{}", std::process::id(), seq));
379    let tmp = PathBuf::from(tmp_os);
380    std::fs::write(&tmp, json)?;
381    std::fs::rename(&tmp, path)
382}
383
384/// One normalized chunk of an `agent.chat` stream, delivered to an in-process
385/// [`chat_collectors`](ServerState::chat_collectors) sink (the A2A bridge). A
386/// `token` chunk carries a `delta`; a terminal chunk has `kind` `"done"`
387/// (success), `"error"` (with `error` text), or `"auth_required"` (with
388/// `message` text — the turn was refused because of the Parslee account, and
389/// nothing follows it on the wire).
390#[derive(Debug, Clone)]
391pub struct ChatStreamChunk {
392    pub kind: String,
393    pub delta: Option<String>,
394    pub error: Option<String>,
395    /// The `message` field of an `auth_required` terminal frame. Carried
396    /// separately from `error` because the two are different things to a
397    /// caller: one reports a failure, the other names a step a person can
398    /// take.
399    pub message: Option<String>,
400}
401
402/// An in-process `agent.chat` collector entry: the sink that
403/// [`try_forward_agent_chat_event`](crate::handler::try_forward_agent_chat_event)
404/// feeds, plus the host client whose disconnect should abandon it. Holding the
405/// sole sender here means dropping the entry closes the stream, so the
406/// collecting task wakes immediately instead of waiting out its timeout.
407pub struct ChatCollector {
408    pub tx: tokio::sync::mpsc::UnboundedSender<ChatStreamChunk>,
409    pub host_client_id: String,
410}
411
412#[derive(Debug, Clone)]
413#[doc(hidden)]
414pub struct RunResumeLease {
415    pub disconnected_client_id: String,
416    pub expires_at: tokio::time::Instant,
417}
418
419/// Deterministic test seam at the authenticated completion boundary. It is
420/// inert unless explicitly armed through [`ServerStateConfig`].
421#[derive(Debug, Clone, Default)]
422#[doc(hidden)]
423pub struct RunCompletionFenceGate {
424    armed: Arc<std::sync::atomic::AtomicBool>,
425    entered: Arc<tokio::sync::Notify>,
426    release: Arc<tokio::sync::Notify>,
427}
428
429impl RunCompletionFenceGate {
430    pub fn block_next(&self) {
431        self.armed.store(true, std::sync::atomic::Ordering::Release);
432    }
433
434    pub async fn wait_until_entered(&self, timeout: std::time::Duration) -> bool {
435        tokio::time::timeout(timeout, self.entered.notified())
436            .await
437            .is_ok()
438    }
439
440    pub fn release(&self) {
441        self.armed
442            .store(false, std::sync::atomic::Ordering::Release);
443        self.release.notify_one();
444    }
445
446    async fn wait_if_armed(&self) {
447        if !self.armed.load(std::sync::atomic::Ordering::Acquire) {
448            return;
449        }
450        self.entered.notify_one();
451        if self.armed.load(std::sync::atomic::Ordering::Acquire) {
452            self.release.notified().await;
453        }
454    }
455}
456
457/// Daemon-side record of a single agent run (agent run tracing, U1).
458///
459/// Keyed by `run_id` in the process-wide [`ServerState::runs`] registry
460/// so the record survives the WS connection that produced it — needed
461/// both for the disconnect grace window (R5) and so the disk store (U3)
462/// can flush a run even after its session is gone. U1 keeps this purely
463/// in memory; persistence is U3.
464#[derive(Debug, Clone)]
465pub struct RunMeta {
466    pub run_id: String,
467    /// Owning agent. Set at `runs.start` from the resolved id; used by
468    /// the read RPCs' ownership check (U5/KTD10) and the disk key (U3).
469    pub agent_id: String,
470    /// Immutable WS `client_id` that durably called `runs.start`. This remains
471    /// the provenance identity written to `RunStarted`, proposal receipts, and
472    /// `RunEnded`, even if an authenticated replacement socket resumes the
473    /// in-memory run.
474    pub client_id: String,
475    /// The one live socket currently authorized to continue the run. Equal to
476    /// `client_id` for a fresh run; changed only by authenticated orphan
477    /// reclaim after the previous owner has disappeared from `sessions`.
478    pub active_client_id: String,
479    /// Stable predecessor returned by an idempotent `runs.resume` retry on the
480    /// winning replacement socket.
481    pub resume_predecessor_client_id: Option<String>,
482    /// Ephemeral, in-process authority window installed atomically when the
483    /// active socket leaves the session registry. It is intentionally absent
484    /// from RunStore/journals and therefore never survives daemon restart.
485    #[doc(hidden)]
486    pub resume_lease: Option<RunResumeLease>,
487    pub intent: String,
488    pub outcome_description: Option<String>,
489    pub started_at: chrono::DateTime<chrono::Utc>,
490    /// `None` while the run is in progress; `Some` once a terminal
491    /// record (a reported outcome or an `Incomplete` marker) is
492    /// written. The presence of this field is the "is this run still
493    /// open?" signal the grace window checks.
494    pub termination: Option<car_proto::RunTermination>,
495    /// When the terminal record was written, if any.
496    pub ended_at: Option<chrono::DateTime<chrono::Utc>>,
497    /// The ordered per-turn trace recorded for this run (agent run
498    /// tracing, U2). Each entry is a [`car_proto::RunRecord::Turn`]
499    /// produced by the recorder from a submitted proposal + its
500    /// `ActionResult`s. U1 leaves this empty; the U2 recorder appends to
501    /// it via [`ServerState::record_run_turns`]. U3 flushes this buffer
502    /// to disk and U4 broadcasts it — both read it through
503    /// [`ServerState::run_turns`]. The `turn.index` field is monotonic
504    /// across the run's proposals, so this Vec is the clean ordered
505    /// stream those units consume.
506    pub turns: Vec<car_proto::RunRecord>,
507    /// True only after the exact RunStarted row reached flush+fsync and the
508    /// authenticated run_started journal event received the same durability
509    /// acknowledgement. A reserved-but-uncommitted run blocks all proposal
510    /// writes and can only be resumed by its owning client.
511    pub start_committed: bool,
512    /// Stable terminal transaction preimage retained across a failed
513    /// RunStore or journal durability boundary. Retries must present the same
514    /// RunTermination and finish this exact RunEnded; a conflicting terminal
515    /// is rejected.
516    pub pending_terminal: Option<car_proto::RunEnded>,
517    /// Durable cancellation request. Presence quarantines new proposals and
518    /// `runs.complete` until CAR either confirms a Cancelled terminal or an
519    /// operator independently resolves the unconfirmed stop.
520    pub cancellation_pending: Option<car_proto::RunCancellationRequested>,
521    /// Stable response for same-key idempotent retries.
522    pub cancellation_receipt: Option<car_proto::RunCancelResponse>,
523    /// Stable fail-closed error once the durable JSONL is known corrupt.
524    /// A quarantined run accepts no more proposals/turns, cannot complete,
525    /// and cannot install or retain live subscribers.
526    pub trace_corruption: Option<String>,
527    /// Monotonic in-memory commit generation for durability work which drops
528    /// the global run-registry lock while filesystem I/O executes.
529    #[doc(hidden)]
530    pub durability_generation: u64,
531}
532
533#[derive(Debug, Clone)]
534pub enum RunReservation {
535    New,
536    Existing(RunMeta),
537}
538
539/// In-memory ownership result for the authenticated WebSocket resume path.
540/// `active_client_id` is the fenced socket allowed to use the original
541/// durable run provenance now.
542#[derive(Debug, Clone, PartialEq, Eq)]
543pub(crate) struct RunResumeBinding {
544    pub run_id: String,
545    pub agent_id: String,
546    pub active_client_id: String,
547    pub resumed_from_client_id: String,
548}
549
550/// Canonical terminal identity shared by the durable `RunEnded` row, terminal
551/// journal event, and `runs.complete` response. The input is exactly the RFC
552/// 8785/JCS serialization of `RunTermination`; it deliberately does not include
553/// timestamps, run ids, client ids, or Daily artifact bytes.
554pub(crate) fn run_completion_digest(
555    termination: &car_proto::RunTermination,
556) -> Result<String, String> {
557    let canonical = car_inference::catalog_identity::canonical_json(termination)?;
558    Ok(format!("{:x}", Sha256::digest(canonical.as_bytes())))
559}
560
561fn run_trace_corruption_message(run_id: &str, detail: impl std::fmt::Display) -> String {
562    format!(
563        "{} run `{run_id}`: {detail}",
564        car_proto::RUN_TRACE_CORRUPTION_MESSAGE_PREFIX
565    )
566}
567
568fn same_run_termination(
569    left: &car_proto::RunTermination,
570    right: &car_proto::RunTermination,
571) -> bool {
572    match (run_completion_digest(left), run_completion_digest(right)) {
573        (Ok(left), Ok(right)) => left == right,
574        _ => false,
575    }
576}
577
578fn same_run_ended(left: &car_proto::RunEnded, right: &car_proto::RunEnded) -> bool {
579    left.run_id == right.run_id
580        && left.client_id == right.client_id
581        && left.agent_id == right.agent_id
582        && left.completion_digest == right.completion_digest
583        && left.ended_at == right.ended_at
584        && same_run_termination(&left.termination, &right.termination)
585}
586
587struct StartupJournalCache {
588    journal_dir: PathBuf,
589    failures: Option<car_eventlog::JournalFailureInjector>,
590    logs: HashMap<String, EventLog>,
591}
592
593impl StartupJournalCache {
594    fn new(journal_dir: &Path, failures: Option<&car_eventlog::JournalFailureInjector>) -> Self {
595        Self {
596            journal_dir: journal_dir.to_path_buf(),
597            failures: failures.cloned(),
598            logs: HashMap::new(),
599        }
600    }
601
602    fn get(&mut self, client_id: &str) -> Result<&mut EventLog, String> {
603        if !self.logs.contains_key(client_id) {
604            let path = self.journal_dir.join(format!("{client_id}.jsonl"));
605            let log = match (path.exists(), self.failures.as_ref()) {
606                (true, Some(failures)) => {
607                    EventLog::load_with_journal_failure_injector(&path, failures.clone())
608                        .map_err(|error| error.to_string())
609                }
610                (true, None) => EventLog::load(&path).map_err(|error| error.to_string()),
611                (false, Some(failures)) => Ok(EventLog::with_journal_failure_injector(
612                    path,
613                    failures.clone(),
614                )),
615                (false, None) => Ok(EventLog::with_journal(path)),
616            }?;
617            self.logs.insert(client_id.to_string(), log);
618        }
619        self.logs
620            .get_mut(client_id)
621            .ok_or_else(|| "startup journal cache insertion failed".to_string())
622    }
623}
624
625fn append_recovered_cancellation_result_journal(
626    journal_dir: PathBuf,
627    failures: Option<car_eventlog::JournalFailureInjector>,
628    client_id: String,
629    result: car_proto::RunCancelResponse,
630) -> Result<bool, String> {
631    let mut journals = StartupJournalCache::new(&journal_dir, failures.as_ref());
632    let log = journals.get(&client_id)?;
633    log.bind_run(&result.run_id, &client_id)?;
634    let Value::Object(data) = serde_json::to_value(&result).map_err(|error| error.to_string())?
635    else {
636        return Err("recovered cancellation result did not serialize as an object".into());
637    };
638    let data: HashMap<String, Value> = data.into_iter().collect();
639    let already_durable = log.events().iter().any(|event| {
640        event.kind == car_eventlog::EventKind::RunCancellationResult
641            && event.run_id.as_deref() == Some(result.run_id.as_str())
642            && event.client_id.as_deref() == Some(client_id.as_str())
643            && event.action_id.as_deref() == result.action_id.as_deref()
644            && event.proposal_id.is_none()
645            && event.data == data
646    });
647    log.append_critical_bounded(
648        car_eventlog::EventKind::RunCancellationResult,
649        result.action_id.as_deref(),
650        None,
651        data,
652        CRITICAL_JOURNAL_ACKNOWLEDGEMENT_TIMEOUT,
653    )
654    .map_err(|error| format!("recovered cancellation journal append failed: {error}"))?;
655    Ok(!already_durable)
656}
657
658fn append_recovered_terminal_cancellation_journal(
659    journal_dir: PathBuf,
660    failures: Option<car_eventlog::JournalFailureInjector>,
661    client_id: String,
662    requested: car_proto::RunCancellationRequested,
663    ended: car_proto::RunEnded,
664    result: car_proto::RunCancelResponse,
665) -> Result<(), String> {
666    let mut journals = StartupJournalCache::new(&journal_dir, failures.as_ref());
667    let log = journals.get(&client_id)?;
668    log.bind_run(&result.run_id, &client_id)?;
669
670    let Value::Object(requested_data) =
671        serde_json::to_value(&requested).map_err(|error| error.to_string())?
672    else {
673        return Err("recovered cancellation request did not serialize as an object".into());
674    };
675    log.append_critical_bounded(
676        car_eventlog::EventKind::RunCancellationRequested,
677        requested.action_id.as_deref(),
678        None,
679        requested_data.into_iter().collect(),
680        CRITICAL_JOURNAL_ACKNOWLEDGEMENT_TIMEOUT,
681    )
682    .map_err(|error| format!("recovered cancellation request journal append failed: {error}"))?;
683
684    let completion_digest = ended
685        .completion_digest
686        .as_deref()
687        .ok_or_else(|| "recovered terminal is missing completion_digest".to_string())?;
688    let completed_data = HashMap::from([
689        ("termination_kind".to_string(), Value::from("incomplete")),
690        (
691            "completion_digest".to_string(),
692            Value::from(completion_digest),
693        ),
694        (
695            "termination".to_string(),
696            serde_json::to_value(&ended.termination).map_err(|error| error.to_string())?,
697        ),
698    ]);
699    log.append_critical_bounded(
700        car_eventlog::EventKind::RunCompleted,
701        None,
702        None,
703        completed_data,
704        CRITICAL_JOURNAL_ACKNOWLEDGEMENT_TIMEOUT,
705    )
706    .map_err(|error| format!("recovered terminal journal append failed: {error}"))?;
707
708    let Value::Object(result_data) =
709        serde_json::to_value(&result).map_err(|error| error.to_string())?
710    else {
711        return Err("recovered cancellation result did not serialize as an object".into());
712    };
713    log.append_critical_bounded(
714        car_eventlog::EventKind::RunCancellationResult,
715        result.action_id.as_deref(),
716        None,
717        result_data.into_iter().collect(),
718        CRITICAL_JOURNAL_ACKNOWLEDGEMENT_TIMEOUT,
719    )
720    .map_err(|error| format!("recovered cancellation result journal append failed: {error}"))?;
721    log.clear_run_binding(&result.run_id, &client_id)
722}
723
724fn reconcile_durable_run_journals(
725    journals: &mut StartupJournalCache,
726    store: &crate::run_store::RunStore,
727    acknowledgement_timeout: std::time::Duration,
728) -> Result<(), String> {
729    type DurableRunBoundaries = (
730        car_proto::RunStarted,
731        Option<car_proto::RunEnded>,
732        Option<car_proto::RunCancellationRequested>,
733        Option<car_proto::RunCancelResponse>,
734    );
735    let mut by_client: HashMap<String, Vec<DurableRunBoundaries>> = HashMap::new();
736    store.visit_run_boundaries(|started, ended, requested, result| {
737        let Some(client_id) = started.client_id.as_deref() else {
738            return;
739        };
740        if client_id.is_empty()
741            || client_id.contains('/')
742            || client_id.contains('\\')
743            || client_id == "."
744            || client_id == ".."
745        {
746            tracing::error!(run_id = %started.run_id, "run-journal outbox has unsafe client_id");
747            return;
748        }
749        by_client
750            .entry(client_id.to_string())
751            .or_default()
752            .push((started, ended, requested, result));
753    });
754
755    for (client_id, mut runs) in by_client {
756        runs.sort_by_key(|(started, _, _, _)| started.started_at);
757        let log = match journals.get(&client_id) {
758            Ok(log) => log,
759            Err(error) => {
760                tracing::error!(%client_id, %error, "cannot load run journal for outbox reconciliation");
761                continue;
762            }
763        };
764        for (started, ended, requested, result) in runs {
765            if let Err(error) = log.bind_run(&started.run_id, &client_id) {
766                tracing::error!(run_id = %started.run_id, %error, "cannot bind run journal during outbox reconciliation");
767                continue;
768            }
769            let mut start_data = HashMap::from([
770                (
771                    "agent_id".to_string(),
772                    Value::from(started.agent_id.clone()),
773                ),
774                ("intent".to_string(), Value::from(started.intent.clone())),
775                (
776                    "started_at".to_string(),
777                    serde_json::to_value(started.started_at).unwrap_or(Value::Null),
778                ),
779            ]);
780            if let Some(description) = &started.outcome_description {
781                start_data.insert(
782                    "outcome_description".to_string(),
783                    Value::from(description.clone()),
784                );
785            }
786            log.append_critical_bounded(
787                car_eventlog::EventKind::RunStarted,
788                None,
789                None,
790                start_data,
791                acknowledgement_timeout,
792            )
793            .map_err(|error| {
794                format!(
795                    "startup reconciliation failed for run {} run_started: {error}",
796                    started.run_id
797                )
798            })?;
799
800            if let Some(requested) = requested {
801                let Value::Object(data) = serde_json::to_value(&requested).unwrap_or(Value::Null)
802                else {
803                    continue;
804                };
805                log.append_critical_bounded(
806                    car_eventlog::EventKind::RunCancellationRequested,
807                    requested.action_id.as_deref(),
808                    None,
809                    data.into_iter().collect(),
810                    acknowledgement_timeout,
811                )
812                .map_err(|error| {
813                    format!(
814                        "startup reconciliation failed for run {} cancellation request: {error}",
815                        started.run_id
816                    )
817                })?;
818            }
819
820            if ended.is_none() {
821                if let Some(result) = result {
822                    let Value::Object(data) = serde_json::to_value(&result).unwrap_or(Value::Null)
823                    else {
824                        continue;
825                    };
826                    log.append_critical_bounded(
827                        car_eventlog::EventKind::RunCancellationResult,
828                        result.action_id.as_deref(),
829                        None,
830                        data.into_iter().collect(),
831                        acknowledgement_timeout,
832                    )
833                    .map_err(|error| {
834                        format!(
835                            "startup reconciliation failed for run {} cancellation result: {error}",
836                            started.run_id
837                        )
838                    })?;
839                }
840            }
841
842            let Some(ended) = ended else {
843                continue;
844            };
845            let Some(completion_digest) = ended.completion_digest.as_deref() else {
846                log.clear_run_binding(&started.run_id, &client_id)?;
847                continue;
848            };
849            if ended.client_id.as_deref() != Some(client_id.as_str()) {
850                tracing::error!(run_id = %started.run_id, "RunEnded owner does not match RunStarted during outbox reconciliation");
851                log.clear_run_binding(&started.run_id, &client_id)?;
852                continue;
853            }
854            let termination_kind = match &ended.termination {
855                car_proto::RunTermination::Outcome { .. } => "outcome",
856                car_proto::RunTermination::Incomplete => "incomplete",
857                car_proto::RunTermination::Cancelled { .. } => "cancelled",
858            };
859            let data = HashMap::from([
860                (
861                    "termination_kind".to_string(),
862                    Value::from(termination_kind),
863                ),
864                (
865                    "completion_digest".to_string(),
866                    Value::from(completion_digest),
867                ),
868                (
869                    "termination".to_string(),
870                    serde_json::to_value(&ended.termination).unwrap_or(Value::Null),
871                ),
872            ]);
873            log.append_critical_bounded(
874                car_eventlog::EventKind::RunCompleted,
875                None,
876                None,
877                data,
878                acknowledgement_timeout,
879            )
880            .map_err(|error| {
881                format!(
882                    "startup reconciliation failed for run {} run_completed: {error}",
883                    started.run_id
884                )
885            })?;
886            log.clear_run_binding(&started.run_id, &client_id)?;
887        }
888    }
889    Ok(())
890}
891
892/// Finish proposal terminals whose actions already ran before the daemon lost
893/// the journal acknowledgement. The outbox owns the exact normal-serde result
894/// and JCS digest, so startup only replays that terminal row; it never invokes
895/// the runtime or a tool executor.
896fn reconcile_pending_proposal_journals(
897    journals: &mut StartupJournalCache,
898    store: &crate::run_store::RunStore,
899    acknowledgement_timeout: std::time::Duration,
900) -> Result<(), String> {
901    let pending_proposals = match store.all_pending_proposals() {
902        Ok(pending) => pending,
903        Err(error) => {
904            tracing::error!(%error, "cannot enumerate proposal-finalization outbox; all unresolved runs remain quarantined");
905            return Ok(());
906        }
907    };
908    for pending in pending_proposals {
909        match store.completed_proposal(
910            &pending.run_id,
911            &pending.client_id,
912            pending.requested_policy_session_id.as_deref(),
913            &pending.original_submission,
914        ) {
915            Ok(Some(receipt)) => {
916                if let Err(error) = store.cleanup_completed_proposal_guards(&receipt) {
917                    tracing::error!(run_id = %pending.run_id, %error, "completed proposal response is durable; startup guard cleanup remains pending");
918                }
919                continue;
920            }
921            Ok(None) => {}
922            Err(error) => {
923                tracing::error!(run_id = %pending.run_id, %error, "cannot validate exact completed proposal response during startup cleanup");
924                continue;
925            }
926        }
927        let (started, marker) = match store.pending_provenance(&pending) {
928            Ok(provenance) => provenance,
929            Err(error) => {
930                tracing::error!(run_id = %pending.run_id, %error, "proposal-finalization provenance is invalid; preserving outcome-unknown quarantine");
931                continue;
932            }
933        };
934        let Some(client_id) = started.client_id.as_deref() else {
935            tracing::error!(run_id = %pending.run_id, "durable RunStarted is missing client_id");
936            continue;
937        };
938        if client_id.is_empty()
939            || client_id.contains('/')
940            || client_id.contains('\\')
941            || client_id == "."
942            || client_id == ".."
943        {
944            tracing::error!(run_id = %pending.run_id, "proposal-finalization outbox has unsafe client_id");
945            continue;
946        }
947        let canonical = match car_inference::catalog_identity::canonical_json(
948            &pending.proposal_result,
949        ) {
950            Ok(canonical) => canonical,
951            Err(error) => {
952                tracing::error!(run_id = %pending.run_id, %error, "proposal-finalization result is not RFC 8785 canonicalizable");
953                continue;
954            }
955        };
956        let digest = format!("{:x}", Sha256::digest(canonical.as_bytes()));
957        if digest != pending.result_digest {
958            tracing::error!(run_id = %pending.run_id, "proposal-finalization digest does not match its durable result preimage");
959            continue;
960        }
961
962        let log = match journals.get(client_id) {
963            Ok(log) => log,
964            Err(error) => {
965                tracing::error!(run_id = %pending.run_id, %error, "cannot load proposal journal for outbox reconciliation");
966                continue;
967            }
968        };
969        if let Err(error) = log.bind_run(&started.run_id, client_id) {
970            tracing::error!(run_id = %pending.run_id, %error, "cannot bind proposal journal during outbox reconciliation");
971            continue;
972        }
973        if let Some(policy_session_id) = marker.policy_session_id.as_deref() {
974            if let Err(error) = log.bind_policy_session(policy_session_id) {
975                tracing::error!(run_id = %pending.run_id, %error, "cannot restore proposal policy binding during outbox reconciliation");
976                continue;
977            }
978        }
979        store.ensure_proposal_turns(&pending).map_err(|error| {
980            format!(
981                "startup reconciliation failed for run {} proposal trace: {error}",
982                pending.run_id
983            )
984        })?;
985        log.append_critical_bounded(
986            car_eventlog::EventKind::ProposalCompleted,
987            None,
988            Some(&pending.final_proposal_id),
989            pending.event_data(),
990            acknowledgement_timeout,
991        )
992        .map_err(|error| {
993            format!(
994                "startup reconciliation failed for run {} proposal_completed: {error}",
995                pending.run_id
996            )
997        })?;
998        if let Some(policy_session_id) = marker.policy_session_id.as_deref() {
999            log.clear_policy_session(policy_session_id)?;
1000        }
1001        let receipt = match store.write_completed_proposal(&pending) {
1002            Ok(receipt) => receipt,
1003            Err(error) => {
1004                tracing::error!(run_id = %pending.run_id, %error, "cannot persist reconciled completed proposal response");
1005                continue;
1006            }
1007        };
1008        if let Err(error) = store.cleanup_completed_proposal_guards(&receipt) {
1009            tracing::error!(run_id = %pending.run_id, %error, "completed proposal response is durable; startup guard cleanup remains pending");
1010            continue;
1011        }
1012    }
1013    Ok(())
1014}
1015
1016/// A completed response can outlive either cleanup guard when the daemon dies
1017/// between independently-fsynced unlink boundaries. Reconcile only under the
1018/// receipt's exact typed authority; never dispatch or append another event.
1019fn reconcile_completed_proposal_guards(store: &crate::run_store::RunStore) {
1020    if let Err(error) = store.reconcile_completed_proposal_migration() {
1021        tracing::error!(%error, "cannot migrate completed proposal response index; guards remain quarantined");
1022    }
1023}
1024
1025/// Per-run turn ceiling — a runaway-loop backstop sized well above any
1026/// healthy main-agent-only cycle (tens of turns), never a trimmer. This is
1027/// a TRUE hard cap, and it is enforced HERE — inside
1028/// [`ServerState::record_run_turns`], under the `runs` lock — not only in
1029/// the WS handler's pre-check. The handler keeps a fast-path pre-check off a
1030/// lock-free snapshot, but the dispatcher spawns a task per frame, so
1031/// pipelined `runs.record_turns` batches can all read a sub-ceiling snapshot
1032/// and pass that pre-check before any of them appends. Only the under-lock
1033/// check below is atomic with the append, so it is the one that actually
1034/// bounds the run (ADV-1). A batch that would take the run PAST this many
1035/// recorded turns is refused WHOLE.
1036pub const RECORD_TURNS_RUN_CEILING: usize = 2000;
1037
1038/// Outcome of [`ServerState::record_run_turns`] — distinguishes the three
1039/// reasons a batch can fail to land so the caller maps each to the right
1040/// `runs.record_turns` drop reason (ADV-1). Before this enum the function
1041/// returned a bare `usize` (the new total, or `0` for "nothing appended"),
1042/// which collapsed an under-lock CEILING refusal and an unknown/terminal run
1043/// into the same `0` — the handler then mislabeled a ceiling refusal as
1044/// `run_terminal`.
1045#[derive(Debug, Clone, PartialEq, Eq)]
1046pub enum RecordRunTurnsOutcome {
1047    /// The batch was appended; the run's new total turn count.
1048    Appended { new_total: usize },
1049    /// The batch was refused WHOLE because it would take the run past
1050    /// [`RECORD_TURNS_RUN_CEILING`] (a runaway backstop). Maps to
1051    /// `dropped: "run_turn_limit"`.
1052    RefusedCeiling,
1053    /// The run id is unknown to this process OR already terminal — nothing
1054    /// was appended. Maps to `dropped: "run_terminal"` (the benign TOCTOU:
1055    /// the run went terminal between the handler's pre-check and the
1056    /// append), matching the prior silent-zero semantics.
1057    UnknownOrTerminal,
1058    /// The batch cannot advance canonical memory/subscribers. Ordinary I/O
1059    /// failures mean the durable append boundary was not acknowledged and an
1060    /// exact retry is safe. A `run trace corruption:` error instead means the
1061    /// bytes exposed committed middle corruption; the run is quarantined and
1062    /// must not be retried or completed.
1063    PersistenceFailed(String),
1064}
1065
1066pub(crate) enum RunSubscribePageResult {
1067    Ready(car_proto::RunSubscribeResponse),
1068    Durable {
1069        agent_id: String,
1070        status: car_proto::RunLiveStatus,
1071    },
1072}
1073
1074impl RunMeta {
1075    /// True once a terminal record (outcome or `Incomplete`) is set.
1076    pub fn is_terminal(&self) -> bool {
1077        self.termination.is_some()
1078    }
1079
1080    pub fn accepts_proposals(&self) -> bool {
1081        self.start_committed
1082            && self.pending_terminal.is_none()
1083            && self.cancellation_pending.is_none()
1084            && self.trace_corruption.is_none()
1085            && !self.is_terminal()
1086    }
1087
1088    /// The coarse live status of this run for the U4 subscribe snapshot /
1089    /// `runs.trace.event` (agent run tracing). Mirrors `RunStore`'s
1090    /// `RunStatus` but reads off the in-memory `RunMeta` so the live path
1091    /// never touches disk.
1092    pub fn live_status(&self) -> car_proto::RunLiveStatus {
1093        match &self.termination {
1094            None if self.cancellation_pending.is_some() => {
1095                car_proto::RunLiveStatus::CancellationPending
1096            }
1097            None => car_proto::RunLiveStatus::InProgress,
1098            Some(car_proto::RunTermination::Outcome { .. }) => car_proto::RunLiveStatus::Completed,
1099            Some(car_proto::RunTermination::Incomplete) => car_proto::RunLiveStatus::Incomplete,
1100            Some(car_proto::RunTermination::Cancelled { .. }) => {
1101                car_proto::RunLiveStatus::Cancelled
1102            }
1103        }
1104    }
1105
1106    /// Count of `RunRecord::Turn` entries in this run's buffer — the live
1107    /// turn cursor. Equals `turns.len()` because the buffer holds only
1108    /// `Turn` records (Started/Ended are not buffered here).
1109    pub fn turn_cursor(&self) -> usize {
1110        self.turns.len()
1111    }
1112}
1113
1114/// Default ceiling for the daemon→host `tools.execute` callback wait when
1115/// an action carries no explicit `timeout_ms`. Overridable via
1116/// `CAR_TOOL_TIMEOUT` (seconds). Raised from the old hardcoded 60s — real
1117/// tools (build steps, CLI drivers, slow APIs) routinely run longer, and a
1118/// 60s ceiling reaped them regardless of the agent's budget (Parslee-ai/car#259).
1119pub const DEFAULT_TOOL_TIMEOUT_MS: u64 = 300_000;
1120
1121/// Grace added to the callback wait above an action's own `timeout_ms`. The
1122/// executor (`car-engine`) already wraps each attempt in
1123/// `timeout(action.timeout_ms, dispatch)`; if this wait used the *same*
1124/// value it would race that outer deadline with zero slack. By waiting a
1125/// little longer, the executor's deadline fires first and reaps+rolls back
1126/// cleanly with its structured "action timed out" error — this wait is only
1127/// a transport backstop for the case the executor deadline doesn't apply.
1128const TOOL_TIMEOUT_GRACE_MS: u64 = 5_000;
1129
1130/// The callback-wait budget for one tool call.
1131///
1132/// - **`Some(X)`**: the executor is the authority (it bounds the attempt at
1133///   `X`); this wait is `X + grace`, a backstop that lets the executor reap
1134///   first. So the agent's per-action budget takes effect — the bug was that
1135///   this wait used a hardcoded 60s and ignored `X` entirely (car#259).
1136/// - **`None`**: the executor applies **no** deadline, so this wait is the
1137///   *sole* bound on the call — the `CAR_TOOL_TIMEOUT`-overridable
1138///   [`DEFAULT_TOOL_TIMEOUT_MS`].
1139fn tool_callback_timeout(action_timeout_ms: Option<u64>) -> std::time::Duration {
1140    let ms = match action_timeout_ms {
1141        Some(budget) => budget.saturating_add(TOOL_TIMEOUT_GRACE_MS),
1142        None => std::env::var("CAR_TOOL_TIMEOUT")
1143            .ok()
1144            .and_then(|s| s.trim().parse::<u64>().ok())
1145            .map(|secs| secs.saturating_mul(1000))
1146            .unwrap_or(DEFAULT_TOOL_TIMEOUT_MS),
1147    };
1148    std::time::Duration::from_millis(ms)
1149}
1150
1151/// Build and write the fire-and-forget `tools.cancel` notification for a
1152/// cancelled `tools.execute` callback (Parslee-ai/car#264).
1153///
1154/// This is the ONE place that constructs a [`ToolCancelRequest`] frame and
1155/// puts it on the wire; every cancellation path routes through
1156/// [`PendingToolCall`]'s drop, which calls this. Best-effort by design — the
1157/// daemon has already given up on the call, so a failed write changes nothing
1158/// it can act on.
1159pub(crate) async fn write_tool_cancel(
1160    channel: &WsChannel,
1161    request_id: String,
1162    action_id: String,
1163    reason: String,
1164) {
1165    use futures::SinkExt;
1166    let cancel = ToolCancelRequest {
1167        request_id,
1168        action_id,
1169        reason,
1170    };
1171    let notification = serde_json::json!({
1172        "jsonrpc": "2.0",
1173        "method": "tools.cancel",
1174        "params": cancel,
1175    });
1176    let Ok(text) = serde_json::to_string(&notification) else {
1177        return;
1178    };
1179    let _ = channel
1180        .write
1181        .lock()
1182        .await
1183        .send(Message::Text(text.into()))
1184        .await;
1185}
1186
1187/// RAII guard covering one in-flight daemon→host `tools.execute` callback
1188/// (Parslee-ai/car#264).
1189///
1190/// Armed from the moment the oneshot sender lands in
1191/// [`WsChannel::pending`]; disarmed only when a `ToolExecuteResponse` is
1192/// actually received. If it is still armed when it drops, the call was
1193/// cancelled and the guard both releases the `pending` entry and tells the
1194/// host to abort the in-flight child.
1195///
1196/// Why RAII rather than an explicit cleanup branch: `tokio::time::timeout`
1197/// **drops** the inner future on expiry, and the daemon's callback wait is
1198/// deliberately the *outermost* of three deadlines
1199/// (`TOOL_TIMEOUT_GRACE_MS` above the executor's per-action budget). So on
1200/// every real reap the whole dispatch future is dropped while parked on
1201/// `rx`, and any code written after the `await` — including the explicit
1202/// timeout arm — never runs. A `Drop` impl is the only cleanup that survives
1203/// being dropped.
1204struct PendingToolCall {
1205    channel: Arc<WsChannel>,
1206    request_id: String,
1207    action_id: String,
1208    reason: String,
1209    armed: bool,
1210}
1211
1212impl PendingToolCall {
1213    fn new(channel: Arc<WsChannel>, request_id: String, action_id: String, reason: String) -> Self {
1214        Self {
1215            channel,
1216            request_id,
1217            action_id,
1218            reason,
1219            armed: true,
1220        }
1221    }
1222
1223    /// A response was delivered — this call completed, so no cancel is owed.
1224    fn disarm(&mut self) {
1225        self.armed = false;
1226    }
1227}
1228
1229impl Drop for PendingToolCall {
1230    fn drop(&mut self) {
1231        if !self.armed {
1232            return;
1233        }
1234        // `pending` is a `tokio::sync::Mutex` and the write is async, but
1235        // `Drop` is sync — so the cleanup has to be a detached task. Outside
1236        // a runtime (a plain `drop` in sync code) there is nothing to spawn
1237        // onto; skip quietly rather than panicking out of a destructor.
1238        let Ok(handle) = tokio::runtime::Handle::try_current() else {
1239            return;
1240        };
1241        let channel = self.channel.clone();
1242        let request_id = std::mem::take(&mut self.request_id);
1243        let action_id = std::mem::take(&mut self.action_id);
1244        let reason = std::mem::take(&mut self.reason);
1245        handle.spawn(async move {
1246            // Only cancel if WE claimed the pending entry. The response
1247            // demux in `handler.rs` removes it on the success path, and
1248            // connection teardown clears the whole map — in either case the
1249            // host is not waiting on anything we should abort.
1250            let claimed = channel.pending.lock().await.remove(&request_id).is_some();
1251            channel.active_actions.lock().await.remove(&request_id);
1252            if claimed {
1253                write_tool_cancel(&channel, request_id, action_id, reason).await;
1254            }
1255        });
1256    }
1257}
1258
1259/// Tool executor that sends callbacks to the client over WebSocket.
1260pub struct WsToolExecutor {
1261    pub channel: Arc<WsChannel>,
1262    negotiated_capabilities: Arc<std::sync::RwLock<std::collections::BTreeSet<String>>>,
1263    halted: Arc<AtomicBool>,
1264}
1265
1266#[async_trait::async_trait]
1267impl ToolExecutor for WsToolExecutor {
1268    async fn execute(&self, tool: &str, params: &Value) -> Result<Value, String> {
1269        // Legacy callers that don't have a proposal-level Action.id
1270        // (e.g. internal `executor.execute` chains in tests) — emit an
1271        // empty action_id so the client-side handler can still see the
1272        // payload shape and decide whether to fail loudly.
1273        self.execute_with_action(tool, params, "", None).await
1274    }
1275
1276    async fn execute_with_action(
1277        &self,
1278        tool: &str,
1279        params: &Value,
1280        action_id: &str,
1281        timeout_ms: Option<u64>,
1282    ) -> Result<Value, String> {
1283        self.execute_with_action_in_session(tool, params, action_id, timeout_ms, None, 1)
1284            .await
1285    }
1286
1287    async fn execute_with_action_in_session(
1288        &self,
1289        tool: &str,
1290        params: &Value,
1291        action_id: &str,
1292        timeout_ms: Option<u64>,
1293        session_id: Option<&str>,
1294        attempt: u32,
1295    ) -> Result<Value, String> {
1296        self.execute_callback(
1297            tool,
1298            params,
1299            action_id,
1300            timeout_ms,
1301            session_id,
1302            attempt,
1303            &HashMap::new(),
1304            None,
1305            false,
1306        )
1307        .await
1308        .map(|execution| execution.output)
1309        .map_err(|failure| failure.message)
1310    }
1311
1312    async fn execute_with_action_state_in_session(
1313        &self,
1314        tool: &str,
1315        params: &Value,
1316        action_id: &str,
1317        timeout_ms: Option<u64>,
1318        session_id: Option<&str>,
1319        attempt: u32,
1320        expected_effects: &HashMap<String, Value>,
1321        return_schema: Option<&Value>,
1322    ) -> Result<car_engine::ToolExecution, String> {
1323        self.execute_callback(
1324            tool,
1325            params,
1326            action_id,
1327            timeout_ms,
1328            session_id,
1329            attempt,
1330            expected_effects,
1331            return_schema,
1332            self.callback_state_negotiated(),
1333        )
1334        .await
1335        .map_err(|failure| failure.message)
1336    }
1337
1338    async fn execute_classified(
1339        &self,
1340        tool: &str,
1341        params: &Value,
1342        action_id: &str,
1343        timeout_ms: Option<u64>,
1344        session_id: Option<&str>,
1345        attempt: u32,
1346        expected_effects: &HashMap<String, Value>,
1347        return_schema: Option<&Value>,
1348    ) -> Result<car_engine::ToolExecution, ToolFailure> {
1349        self.execute_callback(
1350            tool,
1351            params,
1352            action_id,
1353            timeout_ms,
1354            session_id,
1355            attempt,
1356            expected_effects,
1357            return_schema,
1358            self.callback_state_negotiated(),
1359        )
1360        .await
1361    }
1362}
1363
1364impl WsToolExecutor {
1365    pub fn new(
1366        channel: Arc<WsChannel>,
1367        negotiated_capabilities: Arc<std::sync::RwLock<std::collections::BTreeSet<String>>>,
1368        halted: Arc<AtomicBool>,
1369    ) -> Self {
1370        Self {
1371            channel,
1372            negotiated_capabilities,
1373            halted,
1374        }
1375    }
1376
1377    #[cfg(test)]
1378    fn legacy(channel: Arc<WsChannel>) -> Self {
1379        Self::new(
1380            channel,
1381            Arc::new(std::sync::RwLock::new(std::collections::BTreeSet::new())),
1382            Arc::new(AtomicBool::new(false)),
1383        )
1384    }
1385
1386    fn callback_state_negotiated(&self) -> bool {
1387        self.negotiated_capabilities
1388            .read()
1389            .map(|capabilities| capabilities.contains(car_proto::TOOLS_CALLBACK_STATE_CAPABILITY))
1390            .unwrap_or(false)
1391    }
1392
1393    #[allow(clippy::too_many_arguments)]
1394    async fn execute_callback(
1395        &self,
1396        tool: &str,
1397        params: &Value,
1398        action_id: &str,
1399        timeout_ms: Option<u64>,
1400        session_id: Option<&str>,
1401        attempt: u32,
1402        expected_effects: &HashMap<String, Value>,
1403        return_schema: Option<&Value>,
1404        callback_state_negotiated: bool,
1405    ) -> Result<car_engine::ToolExecution, ToolFailure> {
1406        use futures::SinkExt;
1407
1408        // The JSON-RPC request id is the daemon's callback-routing key
1409        // (used by the pending-response map below). The `action_id`
1410        // FIELD on the payload is the originating proposal Action.id
1411        // surfaced to the host so process-wide handlers can route
1412        // concurrent callbacks back to per-call dispatchers
1413        // (Parslee-ai/car-releases#43 follow-up). They serve different
1414        // purposes and must stay distinct: routing id is daemon-side,
1415        // action id is host-side.
1416        let request_id = self.channel.next_request_id();
1417
1418        let callback = ToolExecuteRequest {
1419            action_id: action_id.to_string(),
1420            tool: tool.to_string(),
1421            parameters: params.clone(),
1422            // Surface the action's budget to the host so its own tool runner
1423            // can bound the work too (Parslee-ai/car#259) — was always None.
1424            timeout_ms,
1425            // The engine's real retry counter, 1-based. This was hardcoded to
1426            // `1`, which made the field a constant on the wire: a host building
1427            // the retry-disambiguating join the field exists for got a key that
1428            // never varied, and would only find out under the concurrent-retry
1429            // conditions the join is for (Parslee-ai/car#928).
1430            attempt,
1431            // Surface the routing id so the host can key a per-call abort
1432            // registry on it; a reap emits `tools.cancel` carrying the same id
1433            // (Parslee-ai/car#264).
1434            request_id: request_id.clone(),
1435            // Server-stamped correlation. The executor has always passed this
1436            // in; the parameter was `_session_id` and the value was discarded,
1437            // so every host that needed attribution had to rebuild it from
1438            // client-side convention (Parslee-ai/car#904).
1439            session_id: session_id.map(str::to_string),
1440        };
1441
1442        // Create a oneshot channel for the response
1443        let (tx, rx) = oneshot::channel();
1444        self.channel
1445            .pending
1446            .lock()
1447            .await
1448            .insert(request_id.clone(), tx);
1449        self.channel
1450            .active_actions
1451            .lock()
1452            .await
1453            .insert(request_id.clone(), action_id.to_string());
1454
1455        // From here on the call is in flight. The guard makes cleanup +
1456        // host-abort survive ANY cancellation of this future — the
1457        // executor's per-action `timeout_ms` deadline (the common case, it
1458        // fires `TOOL_TIMEOUT_GRACE_MS` before our own wait below), the
1459        // server's per-method deadline in `handler.rs`, or the callback wait
1460        // here — not just the one arm we can write code in
1461        // (Parslee-ai/car#264).
1462        let mut pending_call = PendingToolCall::new(
1463            self.channel.clone(),
1464            request_id.clone(),
1465            action_id.to_string(),
1466            format!("tool '{tool}' call cancelled before completion (request {request_id})"),
1467        );
1468
1469        // Send the callback to the client as a JSON-RPC request
1470        let rpc_request = serde_json::json!({
1471            "jsonrpc": "2.0",
1472            "method": "tools.execute",
1473            "params": callback,
1474            "id": request_id,
1475        });
1476
1477        let msg = Message::Text(
1478            serde_json::to_string(&rpc_request)
1479                .map_err(|e| e.to_string())?
1480                .into(),
1481        );
1482        self.channel
1483            .write
1484            .lock()
1485            .await
1486            .send(msg)
1487            .await
1488            .map_err(|e| format!("failed to send tool callback: {}", e))?;
1489
1490        // Wait for the client to respond, bounded by the action's budget
1491        // (or the configurable default) — NOT a hardcoded 60s. This is the
1492        // ceiling that previously ignored `Action.timeout_ms` and reaped
1493        // legitimately-long tool calls (Parslee-ai/car#259).
1494        let wait = tool_callback_timeout(timeout_ms);
1495        let response = match tokio::time::timeout(wait, rx).await {
1496            Ok(inner) => inner.map_err(|_| format!("tool '{}' callback channel closed", tool))?,
1497            Err(_) => {
1498                // Reaped by our own callback wait. Sharpen the guard's reason
1499                // to name the timeout, then let it do the cleanup + host abort
1500                // on the way out — the same path a dropped future takes
1501                // (Parslee-ai/car#264).
1502                let reason = format!("tool '{}' callback timed out ({}s)", tool, wait.as_secs());
1503                pending_call.reason = reason.clone();
1504                return Err(ToolFailure::ordinary(reason));
1505            }
1506        };
1507
1508        // A response was delivered (error field set or not): the call is
1509        // finished, so no cancel is owed to the host.
1510        self.channel.active_actions.lock().await.remove(&request_id);
1511        pending_call.disarm();
1512
1513        if let Some(err) = response.error {
1514            let failure = match response.terminal {
1515                false => ToolFailure::ordinary(err),
1516                true => {
1517                    // Latch at the callback boundary, before returning to the
1518                    // engine. Handler-level result inspection repeats this as
1519                    // defense in depth, but this write closes the cancellation
1520                    // gap between typed evidence arriving and execute() ending.
1521                    self.halted.store(true, Ordering::Release);
1522                    ToolFailure::terminal(err)
1523                }
1524            };
1525            return Err(failure);
1526        }
1527        decode_callback_execution(
1528            tool,
1529            action_id,
1530            response.output.unwrap_or(Value::Null),
1531            expected_effects,
1532            return_schema,
1533            callback_state_negotiated,
1534        )
1535        .map_err(ToolFailure::ordinary)
1536    }
1537}
1538
1539fn decode_callback_execution(
1540    tool: &str,
1541    action_id: &str,
1542    result: Value,
1543    expected_effects: &HashMap<String, Value>,
1544    return_schema: Option<&Value>,
1545    callback_state_negotiated: bool,
1546) -> Result<car_engine::ToolExecution, String> {
1547    if !callback_state_negotiated {
1548        return Ok(car_engine::ToolExecution::output_only(result));
1549    }
1550    let envelope = result.as_object().filter(|object| {
1551        object.len() == 2 && object.contains_key("output") && object.contains_key("state_changes")
1552    });
1553
1554    if expected_effects.is_empty() && envelope.is_none() {
1555        return Ok(car_engine::ToolExecution::output_only(result));
1556    }
1557    let envelope = envelope.ok_or_else(|| {
1558        format!(
1559            "tool '{tool}' callback for action '{action_id}' must return exact envelope {{output,state_changes}}"
1560        )
1561    })?;
1562    let output = envelope
1563        .get("output")
1564        .cloned()
1565        .expect("exact envelope contains output");
1566    let state_changes: HashMap<String, Value> = serde_json::from_value(
1567        envelope
1568            .get("state_changes")
1569            .cloned()
1570            .expect("exact envelope contains state_changes"),
1571    )
1572    .map_err(|_| {
1573        format!("tool '{tool}' callback for action '{action_id}' state_changes must be an object")
1574    })?;
1575    let expected_keys: std::collections::BTreeSet<&str> =
1576        expected_effects.keys().map(String::as_str).collect();
1577    let actual_keys: std::collections::BTreeSet<&str> =
1578        state_changes.keys().map(String::as_str).collect();
1579    if expected_keys != actual_keys {
1580        return Err(format!(
1581            "tool '{tool}' callback state_changes keys do not match action '{action_id}': expected {:?}, got {:?}",
1582            expected_keys, actual_keys
1583        ));
1584    }
1585    if let Some(schema) = return_schema {
1586        car_engine::validate_tool_output(tool, schema, &output)?;
1587    }
1588    let changes_value = serde_json::to_value(&state_changes).map_err(|error| {
1589        format!("tool '{tool}' callback state_changes serialization failed: {error}")
1590    })?;
1591    car_inference::catalog_identity::canonical_json(&changes_value).map_err(|error| {
1592        format!("tool '{tool}' callback state_changes failed JCS/I-JSON validation: {error}")
1593    })?;
1594    Ok(car_engine::ToolExecution {
1595        output,
1596        state_changes,
1597    })
1598}
1599
1600#[cfg(test)]
1601mod tool_cancel_tests {
1602    use super::*;
1603    use serde_json::json;
1604    use std::time::Duration;
1605
1606    /// Every `Message::Text` the executor wrote, parsed as JSON.
1607    fn written(frames: &std::sync::Arc<std::sync::Mutex<Vec<String>>>) -> Vec<Value> {
1608        frames
1609            .lock()
1610            .unwrap()
1611            .iter()
1612            .map(|t| serde_json::from_str::<Value>(t).expect("frame is JSON"))
1613            .collect()
1614    }
1615
1616    fn frame_with_method<'a>(frames: &'a [Value], method: &str) -> Option<&'a Value> {
1617        frames.iter().find(|f| f["method"] == json!(method))
1618    }
1619
1620    /// Let detached cleanup tasks (the `PendingToolCall` drop spawn) run.
1621    async fn settle() {
1622        for _ in 0..8 {
1623            tokio::task::yield_now().await;
1624        }
1625        tokio::time::sleep(Duration::from_millis(100)).await;
1626        for _ in 0..8 {
1627            tokio::task::yield_now().await;
1628        }
1629    }
1630
1631    /// The daemon's own callback wait expiring still cancels — the arm that
1632    /// used to hold the inline cleanup now delegates to the drop guard, and
1633    /// the observable behaviour (Err text, cancel frame, drained `pending`)
1634    /// must be unchanged.
1635    #[tokio::test(start_paused = true)]
1636    async fn callback_wait_expiry_cancels() {
1637        let (channel, frames) = WsChannel::test_capture();
1638        let channel = Arc::new(channel);
1639        let executor = WsToolExecutor::legacy(channel.clone());
1640
1641        // 1ms budget → a ~5s callback wait; paused time makes that instant.
1642        let err = executor
1643            .execute_with_action_in_session("drive_cli", &json!({}), "a0", Some(1), None, 1)
1644            .await
1645            .expect_err("the callback wait must expire");
1646        assert!(
1647            err.contains("tool 'drive_cli' callback timed out"),
1648            "unexpected error text: {err}"
1649        );
1650
1651        settle().await;
1652
1653        let frames = written(&frames);
1654        let execute = frame_with_method(&frames, "tools.execute").expect("tools.execute was sent");
1655        let cancel = frame_with_method(&frames, "tools.cancel")
1656            .expect("a tools.cancel notification must reach the wire");
1657        assert_eq!(
1658            cancel["params"]["request_id"], execute["params"]["request_id"],
1659            "cancel must correlate to the originating request_id"
1660        );
1661        assert_eq!(cancel["params"]["action_id"], json!("a0"));
1662        assert!(
1663            cancel["id"].is_null(),
1664            "cancel is a notification, not a request"
1665        );
1666        assert!(
1667            channel.pending.lock().await.is_empty(),
1668            "the pending entry must be released"
1669        );
1670    }
1671
1672    /// Parslee-ai/car#904 — the execution session must reach the wire.
1673    ///
1674    /// The executor has always passed a session id in; the parameter was named
1675    /// `_session_id` and the value was dropped on the floor, so every host that
1676    /// needed to know which mission a callback belonged to had to rebuild it
1677    /// from client-side convention. Asserted on the actual `tools.execute`
1678    /// frame rather than on the struct, because the struct field existing is
1679    /// not the property that was missing.
1680    #[tokio::test(start_paused = true)]
1681    async fn tools_execute_carries_the_execution_session() {
1682        let (channel, frames) = WsChannel::test_capture();
1683        let channel = Arc::new(channel);
1684        let executor = WsToolExecutor::legacy(channel.clone());
1685
1686        let _ = tokio::time::timeout(
1687            Duration::from_millis(50),
1688            executor.execute_with_action_in_session(
1689                "search",
1690                &json!({}),
1691                "a0",
1692                Some(10_000),
1693                Some("sess-42"),
1694                1,
1695            ),
1696        )
1697        .await;
1698        settle().await;
1699
1700        let frames = written(&frames);
1701        let execute = frame_with_method(&frames, "tools.execute").expect("tools.execute was sent");
1702        assert_eq!(
1703            execute["params"]["session_id"],
1704            json!("sess-42"),
1705            "the daemon must stamp the session it already knows, not leave \
1706             attribution to the host"
1707        );
1708    }
1709
1710    /// Parslee-ai/car#928 — the retry counter on the wire must be the engine's
1711    /// real one.
1712    ///
1713    /// It was hardcoded to `1` here, so the field a host would build its
1714    /// retry-disambiguating join on never varied. That is worse than the field
1715    /// being absent: the join looks correct and degenerates silently, and only
1716    /// under the concurrent-retry conditions it exists to handle.
1717    #[tokio::test(start_paused = true)]
1718    async fn tools_execute_carries_the_real_attempt_number() {
1719        for attempt in [1u32, 2, 7] {
1720            let (channel, frames) = WsChannel::test_capture();
1721            let channel = Arc::new(channel);
1722            let executor = WsToolExecutor::legacy(channel.clone());
1723
1724            let _ = tokio::time::timeout(
1725                Duration::from_millis(50),
1726                executor.execute_with_action_in_session(
1727                    "search",
1728                    &json!({}),
1729                    "a0",
1730                    Some(10_000),
1731                    Some("sess-1"),
1732                    attempt,
1733                ),
1734            )
1735            .await;
1736            settle().await;
1737
1738            let frames = written(&frames);
1739            let execute =
1740                frame_with_method(&frames, "tools.execute").expect("tools.execute was sent");
1741            assert_eq!(
1742                execute["params"]["attempt"],
1743                json!(attempt),
1744                "the payload must carry the engine's attempt, not a constant"
1745            );
1746        }
1747    }
1748
1749    /// The other half: a caller with no session must omit the key rather than
1750    /// send a null. `execute()` and in-process executors legitimately have
1751    /// none, and a null would make every such payload differ from the pre-#904
1752    /// shape for no gain.
1753    #[tokio::test(start_paused = true)]
1754    async fn a_sessionless_call_omits_the_session_key() {
1755        let (channel, frames) = WsChannel::test_capture();
1756        let channel = Arc::new(channel);
1757        let executor = WsToolExecutor::legacy(channel.clone());
1758
1759        let _ = tokio::time::timeout(
1760            Duration::from_millis(50),
1761            executor.execute_with_action_in_session(
1762                "search",
1763                &json!({}),
1764                "a0",
1765                Some(10_000),
1766                None,
1767                1,
1768            ),
1769        )
1770        .await;
1771        settle().await;
1772
1773        let frames = written(&frames);
1774        let execute = frame_with_method(&frames, "tools.execute").expect("tools.execute was sent");
1775        assert!(
1776            execute["params"].get("session_id").is_none(),
1777            "a sessionless call must omit the key entirely, got: {}",
1778            execute["params"]
1779        );
1780    }
1781
1782    /// REGRESSION (Parslee-ai/car#264): the executor's per-action deadline
1783    /// fires `TOOL_TIMEOUT_GRACE_MS` BEFORE the daemon's callback wait, so it
1784    /// drops this whole future while it is parked on `rx`. Before the drop
1785    /// guard, nothing ran on that path: no `tools.cancel` frame was ever
1786    /// written and the `pending` entry leaked until connection teardown,
1787    /// orphaning the host's in-flight child.
1788    #[tokio::test(start_paused = true)]
1789    async fn engine_deadline_drop_cancels() {
1790        let (channel, frames) = WsChannel::test_capture();
1791        let channel = Arc::new(channel);
1792        let executor = WsToolExecutor::legacy(channel.clone());
1793
1794        // Outer deadline (standing in for the engine's per-action timeout)
1795        // at 50ms; the callback wait would be 10s + 5s grace.
1796        let outcome = tokio::time::timeout(
1797            Duration::from_millis(50),
1798            executor.execute_with_action_in_session(
1799                "drive_cli",
1800                &json!({}),
1801                "a0",
1802                Some(10_000),
1803                None,
1804                1,
1805            ),
1806        )
1807        .await;
1808        assert!(
1809            outcome.is_err(),
1810            "the outer deadline must fire first and drop the dispatch future"
1811        );
1812
1813        settle().await;
1814
1815        let frames = written(&frames);
1816        let execute = frame_with_method(&frames, "tools.execute").expect("tools.execute was sent");
1817        let cancel = frame_with_method(&frames, "tools.cancel").expect(
1818            "dropping the dispatch future must still emit tools.cancel so the host child isn't orphaned",
1819        );
1820        assert_eq!(
1821            cancel["params"]["request_id"], execute["params"]["request_id"],
1822            "cancel must correlate to the originating request_id"
1823        );
1824        assert!(
1825            channel.pending.lock().await.is_empty(),
1826            "the pending entry must not leak"
1827        );
1828    }
1829
1830    /// The happy path must stay silent: a delivered response disarms the
1831    /// guard, so no host abort is emitted for a call that completed.
1832    #[tokio::test]
1833    async fn success_emits_no_cancel() {
1834        let (channel, frames) = WsChannel::test_capture();
1835        let channel = Arc::new(channel);
1836        let executor = WsToolExecutor::legacy(channel.clone());
1837
1838        let responder = channel.clone();
1839        let host = tokio::spawn(async move {
1840            loop {
1841                let claimed = {
1842                    let mut pending = responder.pending.lock().await;
1843                    let key = pending.keys().next().cloned();
1844                    key.and_then(|k| pending.remove(&k).map(|tx| (k, tx)))
1845                };
1846                if let Some((request_id, tx)) = claimed {
1847                    let _ = tx.send(ToolExecuteResponse {
1848                        action_id: request_id,
1849                        output: Some(json!({"ok": true})),
1850                        error: None,
1851                        terminal: false,
1852                    });
1853                    return;
1854                }
1855                tokio::task::yield_now().await;
1856            }
1857        });
1858
1859        let out = executor
1860            .execute_with_action_in_session("drive_cli", &json!({}), "a0", Some(10_000), None, 1)
1861            .await
1862            .expect("the host responded, so the call succeeds");
1863        assert_eq!(out, json!({"ok": true}));
1864        host.await.expect("responder task finishes");
1865
1866        settle().await;
1867
1868        let frames = written(&frames);
1869        assert!(
1870            frame_with_method(&frames, "tools.execute").is_some(),
1871            "tools.execute was sent"
1872        );
1873        assert!(
1874            frame_with_method(&frames, "tools.cancel").is_none(),
1875            "a completed call must not emit tools.cancel"
1876        );
1877        assert!(channel.pending.lock().await.is_empty());
1878    }
1879}
1880
1881/// The bare commodity-tool names owned by the engine's bound substrate.
1882/// When a session binds a non-Local substrate (`session.bindSubstrate`),
1883/// these names must reach the engine's substrate-after-fall_through path
1884/// (`executor.rs`), NOT the WS tool callback. Everything else — connector
1885/// `mcp_{slug}_*` routes and host-executed client tools — is untouched.
1886///
1887/// `calculate` is deliberately excluded: it is a pure, environment-free
1888/// tool that the engine already handles in-process via `agent_basics`, so
1889/// it never needs to be re-routed to a substrate.
1890pub(crate) const SUBSTRATE_OWNED_TOOLS: &[&str] = &[
1891    "read_file",
1892    "write_file",
1893    "edit_file",
1894    "list_dir",
1895    "find_files",
1896    "grep_files",
1897];
1898
1899/// Executor wrapper used **only** by substrate-bound sessions
1900/// (`session.bindSubstrate`). It is the low-blast-radius "option (a)" from
1901/// `docs/execution-substrate.md` §3: keep the existing
1902/// `share_with_fallback(ws_executor)` composition intact for connector
1903/// routes and host tool callbacks, but for the bare substrate-owned
1904/// built-in names ([`SUBSTRATE_OWNED_TOOLS`]) return an `"unknown tool:
1905/// <name>"` error so the engine's dispatch (`executor.rs`) takes the
1906/// `fall_through` branch and routes those names to the runtime's bound
1907/// substrate (the connector/VM [`car_engine::McpSubstrate`]) instead of the WS client.
1908///
1909/// This preserves every existing contract — GUI/connector/coder sessions
1910/// never construct one of these — and is what makes a connector-driven
1911/// session *coherent*: its file/exec built-ins land on the same machine as
1912/// its `mcp_{slug}_*` tools, not split between the WS client and a remote
1913/// MCP server.
1914pub struct SubstrateShadowExecutor {
1915    inner: Arc<dyn ToolExecutor>,
1916}
1917
1918impl SubstrateShadowExecutor {
1919    pub fn new(inner: Arc<dyn ToolExecutor>) -> Self {
1920        Self { inner }
1921    }
1922}
1923
1924#[async_trait::async_trait]
1925impl ToolExecutor for SubstrateShadowExecutor {
1926    async fn execute(&self, tool: &str, params: &Value) -> Result<Value, String> {
1927        self.execute_with_action(tool, params, "", None).await
1928    }
1929
1930    async fn execute_with_action(
1931        &self,
1932        tool: &str,
1933        params: &Value,
1934        action_id: &str,
1935        timeout_ms: Option<u64>,
1936    ) -> Result<Value, String> {
1937        self.execute_with_action_in_session(tool, params, action_id, timeout_ms, None, 1)
1938            .await
1939    }
1940
1941    async fn execute_with_action_in_session(
1942        &self,
1943        tool: &str,
1944        params: &Value,
1945        action_id: &str,
1946        timeout_ms: Option<u64>,
1947        session_id: Option<&str>,
1948        attempt: u32,
1949    ) -> Result<Value, String> {
1950        if SUBSTRATE_OWNED_TOOLS.contains(&tool) {
1951            // Signal the engine to fall through to the bound substrate.
1952            // The exact `"unknown tool"` prefix is the contract the engine
1953            // dispatch matches on (`executor.rs`).
1954            return Err(format!("unknown tool: {tool}"));
1955        }
1956        self.inner
1957            .execute_with_action_in_session(
1958                tool, params, action_id, timeout_ms, session_id, attempt,
1959            )
1960            .await
1961    }
1962
1963    async fn execute_with_action_state_in_session(
1964        &self,
1965        tool: &str,
1966        params: &Value,
1967        action_id: &str,
1968        timeout_ms: Option<u64>,
1969        session_id: Option<&str>,
1970        attempt: u32,
1971        expected_effects: &HashMap<String, Value>,
1972        return_schema: Option<&Value>,
1973    ) -> Result<car_engine::ToolExecution, String> {
1974        if SUBSTRATE_OWNED_TOOLS.contains(&tool) {
1975            return Err(format!("unknown tool: {tool}"));
1976        }
1977        self.inner
1978            .execute_with_action_state_in_session(
1979                tool,
1980                params,
1981                action_id,
1982                timeout_ms,
1983                session_id,
1984                attempt,
1985                expected_effects,
1986                return_schema,
1987            )
1988            .await
1989    }
1990
1991    async fn execute_classified(
1992        &self,
1993        tool: &str,
1994        params: &Value,
1995        action_id: &str,
1996        timeout_ms: Option<u64>,
1997        session_id: Option<&str>,
1998        attempt: u32,
1999        expected_effects: &HashMap<String, Value>,
2000        return_schema: Option<&Value>,
2001    ) -> Result<car_engine::ToolExecution, ToolFailure> {
2002        if SUBSTRATE_OWNED_TOOLS.contains(&tool) {
2003            return Err(ToolFailure::ordinary(format!("unknown tool: {tool}")));
2004        }
2005        self.inner
2006            .execute_classified(
2007                tool,
2008                params,
2009                action_id,
2010                timeout_ms,
2011                session_id,
2012                attempt,
2013                expected_effects,
2014                return_schema,
2015            )
2016            .await
2017    }
2018}
2019
2020/// Voice event sink that forwards events to a specific WebSocket client
2021/// as `voice.event` JSON-RPC notifications.
2022///
2023/// Each `voice.transcribe_stream.start` call constructs one of these
2024/// bound to the originating client's [`WsChannel`], so a client only
2025/// receives events for sessions it started.
2026pub struct WsVoiceEventSink {
2027    pub channel: Arc<WsChannel>,
2028}
2029
2030impl car_voice::VoiceEventSink for WsVoiceEventSink {
2031    fn send(&self, session_id: &str, event_json: String) {
2032        use futures::SinkExt;
2033        let channel = self.channel.clone();
2034        let session_id = session_id.to_string();
2035        tokio::spawn(async move {
2036            let payload: Value = serde_json::from_str(&event_json)
2037                .unwrap_or_else(|_| Value::String(event_json.clone()));
2038            let notification = serde_json::json!({
2039                "jsonrpc": "2.0",
2040                "method": "voice.event",
2041                "params": {
2042                    "session_id": session_id,
2043                    "event": payload,
2044                },
2045            });
2046            let Ok(text) = serde_json::to_string(&notification) else {
2047                return;
2048            };
2049            let _ = channel
2050                .write
2051                .lock()
2052                .await
2053                .send(Message::Text(text.into()))
2054                .await;
2055        });
2056    }
2057
2058    fn send_binary(&self, frame: Vec<u8>) {
2059        use futures::SinkExt;
2060        let channel = self.channel.clone();
2061        tokio::spawn(async move {
2062            let _ = channel
2063                .write
2064                .lock()
2065                .await
2066                .send(Message::Binary(frame.into()))
2067                .await;
2068        });
2069    }
2070}
2071
2072/// Per-meeting fanout sink that ingests transcript text into a
2073/// session-scoped memgine using the `Arc<tokio::sync::Mutex<...>>`
2074/// wrapper, then forwards every event upstream untouched.
2075///
2076/// Lives here (not in `car-ffi-common`) because the engine handle uses
2077/// `tokio::sync::Mutex` per the "one-wrapper rule" — the FFI-common
2078/// `MeetingMemgineFanout` still uses `std::sync::Mutex` for the NAPI/
2079/// PyO3 bindings, which keep their sync wrappers. Each binding owns the
2080/// fanout that matches its lock primitive; the parsing/formatting logic
2081/// itself is shared via [`car_meeting::extract_transcript_for_ingest`].
2082///
2083/// `send` is called from the voice drain task and must be non-blocking,
2084/// so the lock acquisition is shipped to a `tokio::spawn`. Transcript
2085/// events are independent so reordering across spawned tasks is fine.
2086pub struct WsMemgineIngestSink {
2087    pub meeting_id: String,
2088    pub engine: Arc<Mutex<car_memgine::MemgineEngine>>,
2089    pub upstream: Arc<dyn car_voice::VoiceEventSink>,
2090}
2091
2092impl car_voice::VoiceEventSink for WsMemgineIngestSink {
2093    fn send(&self, voice_session_id: &str, event_json: String) {
2094        if let Ok(value) = serde_json::from_str::<Value>(&event_json) {
2095            if let Some((speaker, text)) = car_meeting::extract_transcript_for_ingest(
2096                &value,
2097                &self.meeting_id,
2098                voice_session_id,
2099            ) {
2100                let engine = self.engine.clone();
2101                tokio::spawn(async move {
2102                    let mut guard = engine.lock().await;
2103                    guard.ingest_conversation(&speaker, &text, chrono::Utc::now());
2104                });
2105            }
2106        }
2107        self.upstream.send(voice_session_id, event_json);
2108    }
2109}
2110
2111/// The last assistant turn a chat session produced, remembered so the *next*
2112/// user turn can be classified against it (the conversation-outcome signal).
2113/// The `trace_id`/`model_id` are carried straight from the `InferenceResult`
2114/// that produced the turn — never reconstructed from message order — so credit
2115/// always lands on the exact model/trace that generated the judged turn.
2116#[derive(Debug, Clone)]
2117pub struct LastChatTurn {
2118    /// The user request this assistant turn answered (for restatement similarity).
2119    pub user_text: String,
2120    pub assistant_text: String,
2121    pub trace_id: String,
2122    pub model_id: String,
2123}
2124
2125/// Per-client session.
2126pub struct ClientSession {
2127    pub client_id: String,
2128    pub runtime: Arc<Runtime>,
2129    pub channel: Arc<WsChannel>,
2130    pub host: Arc<crate::host::HostState>,
2131    /// Memgine handle. Wrapped in `tokio::sync::Mutex` so dispatcher
2132    /// handlers can hold the lock across `.await` points without
2133    /// risking poisoning. Migrated from `std::sync::Mutex` in the
2134    /// car-server-core extraction (U1) per the "one-wrapper rule".
2135    pub memgine: Arc<Mutex<car_memgine::MemgineEngine>>,
2136    /// Lazy browser session — first `browser.run` call launches Chromium,
2137    /// subsequent calls reuse it so element IDs resolve across invocations
2138    /// within the same WebSocket connection.
2139    pub browser: car_ffi_common::browser::BrowserSessionSlot,
2140    /// Per-connection auth state. Starts `false`; flips to `true`
2141    /// after a successful `session.auth` handshake. Always considered
2142    /// authenticated when `ServerState::auth_token` is unset (auth
2143    /// disabled). Closes Parslee-ai/car-releases#32.
2144    pub authenticated: Arc<std::sync::atomic::AtomicBool>,
2145    /// Negotiated daemon JSON-RPC protocol for this WebSocket connection.
2146    /// Starts at `0` (unnegotiated) and is set to
2147    /// `car_proto::PROTOCOL_VERSION` only after an exact-version
2148    /// `server.handshake`. The state is connection-scoped by construction, so
2149    /// a reconnect always has to negotiate again before using handshake-gated
2150    /// host/auth methods.
2151    pub negotiated_protocol_version: std::sync::atomic::AtomicU32,
2152    /// Capability names negotiated by this connection's authenticated
2153    /// `server.handshake`. Connection-scoped for the same reason as the wire
2154    /// version: a reconnect cannot inherit another socket's proof.
2155    pub negotiated_capabilities: Arc<std::sync::RwLock<std::collections::BTreeSet<String>>>,
2156    /// Bounded lifecycle state for inferences started on this exact socket.
2157    /// Keeping it per-session prevents an authenticated peer from probing IDs
2158    /// owned by another connection.
2159    pub inference_control: Arc<crate::inference_control::InferenceRegistry>,
2160    /// Host-management role (Parslee-ai/car#254). Starts `false`; flips
2161    /// to `true` only when the connection presents the per-launch host
2162    /// token via `session.auth { host_token }` (validated against
2163    /// `ServerState::host_token`). `authorize_run_access` requires this
2164    /// for cross-agent run-trace reads — being merely `host.subscribe`d
2165    /// is no longer sufficient, which is what closes the self-elevation
2166    /// hole. Cleared implicitly when the connection drops.
2167    pub is_host: std::sync::atomic::AtomicBool,
2168    /// Bound agent identity (#169). `Some(id)` once a lifecycle-agent
2169    /// child has called `session.auth { token, agent_id }` and the
2170    /// supervisor confirmed `agent_id` is supervised + token matches.
2171    /// Used by `agents.list` to surface which managed agents have
2172    /// actually attached vs. just being marked `Running` at the
2173    /// process level. Cleared at disconnect by `remove_session`.
2174    pub agent_id: Arc<tokio::sync::Mutex<Option<String>>>,
2175    /// Optional daemon-method allowlist bound by successful supervised-agent
2176    /// authentication. `None` is the backward-compatible unrestricted state;
2177    /// `Some(empty)` denies every application method. The dispatcher reads it
2178    /// once before any method-specific handler or notification interceptor.
2179    pub agent_method_allowlist: Arc<std::sync::RwLock<Option<std::collections::BTreeSet<String>>>>,
2180    /// Canonical schema digests for the narrow reverse-callback tools this
2181    /// client registered. Server-owned registry entries never land here.
2182    pub callback_tool_schema_digests:
2183        Arc<tokio::sync::RwLock<std::collections::HashMap<String, String>>>,
2184    /// Bound memory namespace (`session.auth { memory_namespace }`, #79/#80).
2185    ///
2186    /// The namespace identity has to survive the handshake, not just the
2187    /// lookup that binds `bound_memgine`. Without it the daemon knows *which
2188    /// graph* a session is using but not *what to call it*, so nothing can
2189    /// write that graph back to
2190    /// `~/.car/memory/memory-namespaces/<encoded-ns>.json` — which is
2191    /// why namespace memory was durable only until the next restart
2192    /// (car-releases#82). Paired with `bound_memgine` exactly as `agent_id` is.
2193    pub memory_namespace: tokio::sync::Mutex<Option<String>>,
2194    /// Bound persistent memgine (#170). `Some` after `session.auth`
2195    /// successfully attaches the connection to a daemon-owned
2196    /// per-agent memgine (paired with `agent_id`). Memory handlers
2197    /// route through [`ClientSession::effective_memgine`] which
2198    /// returns this when set, falling back to the ephemeral
2199    /// `memgine` field for browser/host/CLI connections.
2200    pub bound_memgine: tokio::sync::Mutex<Option<Arc<Mutex<car_memgine::MemgineEngine>>>>,
2201    /// The run currently bracketed on this connection (agent run
2202    /// tracing, U1). Set by `runs.start` **before** that handler
2203    /// responds, so the per-turn recorder (U2) always reads the
2204    /// `run_id` the bracket established — no race across the
2205    /// concurrently-spawned dispatch tasks (KTD3). Cleared by
2206    /// `runs.complete`. On disconnect with a still-set current run and
2207    /// no recorded terminal, the daemon marks it `Incomplete` (R5).
2208    pub current_run_id: tokio::sync::Mutex<Option<String>>,
2209    /// Serializes the authenticated run bracket on one WebSocket: start,
2210    /// proposal lifecycle, complete, and disconnect terminalization cannot
2211    /// interleave and produce ambiguous journal ordering.
2212    pub run_lifecycle_guard: Arc<tokio::sync::Mutex<()>>,
2213    /// Per-session permission-tier gate (survey §3.4.3/§5.2.5). Backs the
2214    /// `permission.*` JSON-RPC methods: the session holds the granted standing
2215    /// tier + risk classifier. Defaults to `SandboxEdit`, the tier
2216    /// `docs/websocket-protocol.md` publishes as the session default.
2217    ///
2218    /// `Arc`, not a bare lock, because this is also the gate
2219    /// [`crate::permission_gate::PermissionAdmissionGate`] enforces at
2220    /// proposal admission (Parslee-ai/car#890). ONE gate object behind both:
2221    /// a second instance would let the advisory `permission.evaluate` and the
2222    /// enforcement point disagree about the same action, which is precisely
2223    /// the bug class the admission gate was added to close.
2224    ///
2225    /// NOTE (kernel review C1): approval *records* do NOT live here. The
2226    /// gate's embedded ledger is deliberately unused in the daemon — every
2227    /// fingerprint-keyed HITL read/write goes through the SHARED
2228    /// [`ServerState::approval_ledger`] (journal-backed), so an approval
2229    /// recorded on one connection is visible to every other and survives
2230    /// restart. Tier state stays per-session; the approval store is
2231    /// daemon-wide.
2232    pub permission_gate: Arc<tokio::sync::RwLock<car_policy::PermissionGate>>,
2233    /// Connection-local fail-stop latch. A typed terminal tool callback sets
2234    /// it after its proposal has aborted and rolled back; the admission gate
2235    /// sharing this exact atomic rejects every later proposal on this socket.
2236    /// It is deliberately not durable and is discarded on reconnect.
2237    pub halted: Arc<AtomicBool>,
2238    /// Non-overlap guard for `evolution.run` on this session (kernel review
2239    /// S3): the dispatcher spawns requests concurrently even on one
2240    /// connection, and two interleaved cycles would double-dispatch the
2241    /// evolution mechanisms. A second `evolution.run` while one is in flight
2242    /// errs instead of overlapping.
2243    pub evolution_guard: crate::evolution::CycleGuard,
2244    /// Last assistant turn produced on this session's chat path, so the next
2245    /// user turn can be scored as a conversation outcome (Part B). `None` until
2246    /// the first chat reply; only set for declared-chat inference.
2247    pub last_chat_turn: tokio::sync::Mutex<Option<LastChatTurn>>,
2248    /// In-flight chat-infer count. Concurrent infers on one session have no
2249    /// well-defined turn adjacency (the dispatcher spawns a task per frame), so
2250    /// the outcome scorer only records a turn that began while the session was
2251    /// idle — otherwise it would score one infer's trace against an unrelated
2252    /// concurrent user turn (an active mislabel of routing stats). neo #1.
2253    pub chat_inflight: std::sync::atomic::AtomicUsize,
2254    /// Tenant identity bound at `session.auth { tenant_id }` (linus
2255    /// review C-4). Once bound, every tenant-scoped handler uses THIS
2256    /// identity: a per-request `tenant_id` param may restate it but a
2257    /// mismatch is rejected — an authenticated connection cannot hop
2258    /// into another tenant's namespace by editing request params.
2259    /// `None` (unbound) preserves the legacy per-request behavior.
2260    pub tenant: tokio::sync::Mutex<Option<String>>,
2261    /// Whether this connection has a live `tools.stream.event` forwarder
2262    /// task (C2). Set by `tools.stream.subscribe`; a concurrent second
2263    /// subscribe is a no-op instead of spawning a duplicate forwarder
2264    /// (which would double every notification). `Arc` so the forwarder
2265    /// task can RESET it on exit (write timeout on a stalled-but-
2266    /// recovering socket, broadcast closed) — a later re-subscribe then
2267    /// spawns a fresh forwarder rather than silently no-op'ing (Q2).
2268    pub tool_stream_subscribed: std::sync::Arc<std::sync::atomic::AtomicBool>,
2269}
2270
2271impl ClientSession {
2272    /// Returns the memgine handle the memory.* handlers should use:
2273    /// the bound per-agent memgine when this session attached via
2274    /// `session.auth { agent_id }` (#169 + #170), otherwise the
2275    /// ephemeral per-WS memgine. Cheap (one async lock + Arc clone).
2276    pub async fn effective_memgine(&self) -> Arc<Mutex<car_memgine::MemgineEngine>> {
2277        if let Some(eng) = self.bound_memgine.lock().await.as_ref() {
2278            return eng.clone();
2279        }
2280        self.memgine.clone()
2281    }
2282
2283    pub async fn bind_run_journal(&self, run_id: &str) -> Result<(), String> {
2284        self.runtime
2285            .event_log_handle()
2286            .lock()
2287            .await
2288            .bind_run(run_id, &self.client_id)
2289    }
2290
2291    pub async fn require_run_journal_binding(&self, run_id: &str) -> Result<(), String> {
2292        let log = self.runtime.event_log_handle();
2293        let log = log.lock().await;
2294        match log.active_run_binding() {
2295            Some((bound_run, bound_client, _))
2296                if bound_run == run_id && bound_client == self.client_id =>
2297            {
2298                Ok(())
2299            }
2300            Some((bound_run, bound_client, _)) => Err(format!(
2301                "active journal binding mismatch: expected run_id `{run_id}` / client_id `{}`, got `{bound_run}` / `{bound_client}`",
2302                self.client_id
2303            )),
2304            None => Err(format!(
2305                "active run `{run_id}` has no authenticated journal binding"
2306            )),
2307        }
2308    }
2309
2310    pub async fn clear_run_journal_binding(&self, run_id: &str) -> Result<(), String> {
2311        self.runtime
2312            .event_log_handle()
2313            .lock()
2314            .await
2315            .clear_run_binding(run_id, &self.client_id)
2316    }
2317
2318    pub async fn append_run_terminal_event(
2319        &self,
2320        ended: &car_proto::RunEnded,
2321    ) -> Result<(), String> {
2322        self.append_run_terminal_event_once(ended, &self.client_id)
2323            .await
2324            .map_err(|error| error.to_string())
2325    }
2326
2327    /// Append a terminal from an authenticated replacement socket while
2328    /// preserving the immutable client identity in the durable run trace.
2329    pub(crate) async fn append_resumed_run_terminal_event(
2330        &self,
2331        ended: &car_proto::RunEnded,
2332        durable_client_id: &str,
2333    ) -> Result<(), String> {
2334        self.append_run_terminal_event_once(ended, durable_client_id)
2335            .await
2336            .map_err(|error| error.to_string())
2337    }
2338
2339    pub async fn append_run_cancellation_requested_event(
2340        &self,
2341        requested: &car_proto::RunCancellationRequested,
2342    ) -> Result<(), String> {
2343        self.require_run_journal_binding(&requested.run_id).await?;
2344        let value = serde_json::to_value(requested).map_err(|error| error.to_string())?;
2345        let Value::Object(data) = value else {
2346            return Err("cancellation request did not serialize as an object".into());
2347        };
2348        self.runtime
2349            .event_log_handle()
2350            .lock()
2351            .await
2352            .append_critical_async(
2353                car_eventlog::EventKind::RunCancellationRequested,
2354                requested.action_id.as_deref(),
2355                None,
2356                data.into_iter().collect(),
2357                CRITICAL_JOURNAL_ACKNOWLEDGEMENT_TIMEOUT,
2358            )
2359            .await
2360            .map_err(|error| error.to_string())?;
2361        Ok(())
2362    }
2363
2364    pub async fn append_run_cancellation_result_event(
2365        &self,
2366        result: &car_proto::RunCancelResponse,
2367    ) -> Result<(), String> {
2368        self.require_run_journal_binding(&result.run_id).await?;
2369        let value = serde_json::to_value(result).map_err(|error| error.to_string())?;
2370        let Value::Object(data) = value else {
2371            return Err("cancellation result did not serialize as an object".into());
2372        };
2373        self.runtime
2374            .event_log_handle()
2375            .lock()
2376            .await
2377            .append_critical_async(
2378                car_eventlog::EventKind::RunCancellationResult,
2379                result.action_id.as_deref(),
2380                None,
2381                data.into_iter().collect(),
2382                CRITICAL_JOURNAL_ACKNOWLEDGEMENT_TIMEOUT,
2383            )
2384            .await
2385            .map_err(|error| error.to_string())?;
2386        Ok(())
2387    }
2388
2389    async fn append_run_terminal_event_once(
2390        &self,
2391        ended: &car_proto::RunEnded,
2392        durable_client_id: &str,
2393    ) -> Result<(), car_eventlog::CriticalAppendError> {
2394        let rejected = |reason| car_eventlog::CriticalAppendError::Rejected { reason };
2395        self.require_run_journal_binding(&ended.run_id)
2396            .await
2397            .map_err(rejected)?;
2398        if ended.client_id.as_deref() != Some(durable_client_id) {
2399            return Err(rejected(
2400                "RunEnded client_id does not match the durable run owner".into(),
2401            ));
2402        }
2403        let completion_digest = ended
2404            .completion_digest
2405            .as_deref()
2406            .ok_or_else(|| rejected("RunEnded is missing completion_digest".into()))?;
2407        let termination_kind = match &ended.termination {
2408            car_proto::RunTermination::Outcome { .. } => "outcome",
2409            car_proto::RunTermination::Incomplete => "incomplete",
2410            car_proto::RunTermination::Cancelled { .. } => "cancelled",
2411        };
2412        let log = self.runtime.event_log_handle();
2413        let mut log = log.lock().await;
2414        log.append_critical_async(
2415            car_eventlog::EventKind::RunCompleted,
2416            None,
2417            None,
2418            [
2419                (
2420                    "termination_kind".to_string(),
2421                    Value::from(termination_kind),
2422                ),
2423                (
2424                    "completion_digest".to_string(),
2425                    Value::from(completion_digest),
2426                ),
2427                (
2428                    "termination".to_string(),
2429                    serde_json::to_value(&ended.termination)
2430                        .map_err(|error| rejected(error.to_string()))?,
2431                ),
2432            ]
2433            .into(),
2434            CRITICAL_JOURNAL_ACKNOWLEDGEMENT_TIMEOUT,
2435        )
2436        .await?;
2437        Ok(())
2438    }
2439
2440    pub async fn append_run_started_event(
2441        &self,
2442        started: &car_proto::RunStarted,
2443    ) -> Result<(), String> {
2444        self.require_run_journal_binding(&started.run_id).await?;
2445        if started.client_id.as_deref() != Some(self.client_id.as_str()) {
2446            return Err("RunStarted client_id does not match the live ClientSession".into());
2447        }
2448        let mut data = HashMap::from([
2449            (
2450                "agent_id".to_string(),
2451                Value::from(started.agent_id.clone()),
2452            ),
2453            ("intent".to_string(), Value::from(started.intent.clone())),
2454            (
2455                "started_at".to_string(),
2456                serde_json::to_value(started.started_at).unwrap_or(Value::Null),
2457            ),
2458        ]);
2459        if let Some(description) = &started.outcome_description {
2460            data.insert(
2461                "outcome_description".to_string(),
2462                Value::from(description.clone()),
2463            );
2464        }
2465        self.runtime
2466            .event_log_handle()
2467            .lock()
2468            .await
2469            .append_critical_async(
2470                car_eventlog::EventKind::RunStarted,
2471                None,
2472                None,
2473                data,
2474                CRITICAL_JOURNAL_ACKNOWLEDGEMENT_TIMEOUT,
2475            )
2476            .await
2477            .map_err(|error| error.to_string())?;
2478        Ok(())
2479    }
2480}
2481
2482/// Builder for constructing a [`ServerState`] with embedder-supplied
2483/// dependencies. Embedders (e.g. `tokhn-daemon`) use this to inject
2484/// their own memgine handle and other shared infrastructure; the
2485/// Approval-gate policy for high-risk WS methods.
2486///
2487/// Every method in `methods` must be acknowledged via
2488/// `host.resolve_approval` before the dispatcher will route the
2489/// request to its handler. The dispatcher waits up to `timeout` for
2490/// a resolution; on timeout (or any non-`approve` resolution) the
2491/// request fails with JSON-RPC error `-32003`.
2492///
2493/// Default: gate enabled, the host-automation surface
2494/// (`automation.run_applescript`, `automation.run_powershell`,
2495/// `automation.shortcuts.run`, `messages.send`, `mail.send`,
2496/// `calendar.create_event`, `calendar.update_event`, `calendar.delete_event`,
2497/// `vision.ocr`), 60-second timeout.
2498/// `car-server --no-approvals` (or embedders calling
2499/// [`ServerStateConfig::with_approval_gate`] with `enabled=false`)
2500/// turns it off — only appropriate when no untrusted caller can
2501/// reach the WS port.
2502#[derive(Debug, Clone)]
2503pub struct ApprovalGate {
2504    /// Master switch. When `false`, every method dispatches without
2505    /// raising an approval — the pre-2026-05 behaviour.
2506    pub enabled: bool,
2507    /// Methods that require approval. Match is by exact method-name
2508    /// string against the JSON-RPC `method` field.
2509    pub methods: std::collections::HashSet<String>,
2510    /// How long to wait for the user to resolve the approval before
2511    /// timing out and surfacing an error to the caller.
2512    pub timeout: std::time::Duration,
2513}
2514
2515impl Default for ApprovalGate {
2516    fn default() -> Self {
2517        let methods = [
2518            "automation.run_applescript",
2519            "automation.run_powershell",
2520            "automation.shortcuts.run",
2521            "messages.send",
2522            "mail.send",
2523            "calendar.create_event",
2524            "calendar.update_event",
2525            "calendar.delete_event",
2526            "vision.ocr",
2527        ]
2528        .iter()
2529        .map(|s| s.to_string())
2530        .collect();
2531        Self {
2532            enabled: true,
2533            methods,
2534            timeout: std::time::Duration::from_secs(60),
2535        }
2536    }
2537}
2538
2539impl ApprovalGate {
2540    /// Disable the gate entirely. Equivalent to passing
2541    /// `car-server --no-approvals`. Only appropriate when no
2542    /// untrusted caller can reach the WS port.
2543    pub fn disabled() -> Self {
2544        Self {
2545            enabled: false,
2546            methods: std::collections::HashSet::new(),
2547            timeout: std::time::Duration::from_secs(60),
2548        }
2549    }
2550
2551    /// `true` if this method must be acknowledged before dispatch.
2552    pub fn requires_approval(&self, method: &str) -> bool {
2553        self.enabled && self.methods.contains(method)
2554    }
2555}
2556
2557#[cfg(test)]
2558mod approval_gate_tests {
2559    use super::ApprovalGate;
2560
2561    #[test]
2562    fn local_calendar_mutations_are_high_risk_but_reads_are_not() {
2563        let gate = ApprovalGate::default();
2564        for method in [
2565            "calendar.create_event",
2566            "calendar.update_event",
2567            "calendar.delete_event",
2568        ] {
2569            assert!(
2570                gate.requires_approval(method),
2571                "{method} must require approval"
2572            );
2573        }
2574        assert!(!gate.requires_approval("calendar.list"));
2575        assert!(!gate.requires_approval("calendar.events"));
2576    }
2577}
2578
2579/// Mirror the authoritative user profile into memgine's always-present
2580/// Identity layer. Snapshots intentionally omit this node; callers invoke this
2581/// after loading `identity.json` instead, so only one durable copy exists.
2582pub(crate) fn mirror_identity_into_memgine(
2583    engine: &mut car_memgine::MemgineEngine,
2584    identity: &car_identity::AssistantIdentity,
2585) {
2586    let focus_areas: Vec<&str> = identity
2587        .focus_areas
2588        .iter()
2589        .map(|area| area.as_str())
2590        .collect();
2591    let apps: Vec<&str> = identity.apps.iter().map(String::as_str).collect();
2592    engine.ingest_identity_profile(
2593        identity.user_name.as_deref(),
2594        identity.role.as_deref(),
2595        &focus_areas,
2596        &apps,
2597    );
2598}
2599
2600/// standalone `car-server` binary uses [`ServerState::standalone`]
2601/// which calls `with_config` under the hood.
2602pub struct ServerStateConfig {
2603    pub journal_dir: PathBuf,
2604    /// Peer-message audit journal. `None` derives a path once from
2605    /// [`Self::journal_dir`] and the construction-time CAR state root. Tests
2606    /// and embedders can override it without mutating process-global `CAR_HOME`.
2607    pub peer_audit_journal: Option<PathBuf>,
2608    /// Optional pre-constructed memgine engine. When `None`, each
2609    /// `create_session` call builds a fresh engine; embedders that want
2610    /// to share a single engine across sessions can supply a clone of
2611    /// their `Arc<Mutex<MemgineEngine>>` here.
2612    pub shared_memgine: Option<Arc<Mutex<car_memgine::MemgineEngine>>>,
2613    /// Assistant identity/profile store. Production uses the CAR state root;
2614    /// tests and embedders inject a scratch root so identity writes and startup
2615    /// rehydration never touch the operator's real profile.
2616    pub identity_store: car_identity::IdentityStore,
2617    /// Optional pre-constructed inference engine.
2618    pub inference: Option<Arc<car_inference::InferenceEngine>>,
2619    /// Optional embedder-supplied A2A runtime. Used by the in-core
2620    /// `A2aDispatcher` to execute peer-driven proposals. When `None`,
2621    /// the dispatcher uses a fresh `Runtime` with `register_agent_basics`
2622    /// — peer agents see CAR's built-in tools and nothing else,
2623    /// matching the behaviour of the standalone `start_a2a_listener`.
2624    pub a2a_runtime: Option<Arc<car_engine::Runtime>>,
2625    /// Optional embedder-supplied A2A task store. When `None`,
2626    /// defaults to `InMemoryTaskStore`. tokhn-style embedders that
2627    /// want a polling-friendly persistent store plug it in here.
2628    pub a2a_store: Option<Arc<dyn car_a2a::TaskStore>>,
2629    /// Optional embedder-supplied agent card factory. When `None`,
2630    /// the dispatcher serves a card built from the A2A runtime's
2631    /// tool schemas at construction time, advertising its public URL
2632    /// as `ws://127.0.0.1:9100/` (the WS surface the dispatcher itself
2633    /// is reachable on).
2634    pub a2a_card_source: Option<Arc<car_a2a::AgentCardSource>>,
2635    /// Approval-gate policy. When `None`, the dispatcher uses
2636    /// [`ApprovalGate::default`] (gate ON, the macOS-automation
2637    /// surface gated, 60s timeout). Pass
2638    /// [`ApprovalGate::disabled`] to opt out — only appropriate
2639    /// when no untrusted caller can reach the WS port.
2640    pub approval_gate: Option<ApprovalGate>,
2641    /// Journal path for the daemon-wide shared HITL approval ledger
2642    /// ([`ServerState::approval_ledger`]). `None` = the default
2643    /// `~/.car/approvals.jsonl`. Tests and embedders that must not touch the
2644    /// user's real home point this at their own path.
2645    pub approval_journal: Option<PathBuf>,
2646    /// Directory for the daemon-wide trajectory store
2647    /// ([`ServerState::trajectory_store`]). `None` = the default
2648    /// `~/.car/trajectories/`. Tests and embedders that must not touch the
2649    /// user's real home point this at their own path, same as
2650    /// `approval_journal`.
2651    pub trajectory_dir: Option<PathBuf>,
2652    /// Deterministic durability fault seam used by direct lifecycle tests.
2653    pub run_store_failures: Option<crate::run_store::RunStoreFailureInjector>,
2654    /// Deterministic blocking-read seam used by run-liveness tests.
2655    #[doc(hidden)]
2656    pub run_store_summary_read_gate: Option<crate::run_store::RunStoreSummaryReadGate>,
2657    /// Test-only scheduling seam around durable run-summary publication.
2658    #[doc(hidden)]
2659    pub run_store_summary_write_gate: Option<crate::run_store::RunStoreSummaryWriteGate>,
2660    /// Deterministic blocking append seam used by run-liveness tests.
2661    #[doc(hidden)]
2662    pub run_store_append_gate: Option<crate::run_store::RunStoreAppendGate>,
2663    #[doc(hidden)]
2664    pub run_completion_fence_gate: Option<RunCompletionFenceGate>,
2665    /// Deterministic blocking durable-lookup seam used by run-liveness tests.
2666    #[doc(hidden)]
2667    pub run_store_lookup_gate: Option<crate::run_store::RunStoreLookupGate>,
2668    /// Deterministic first-use directory-entry durability seam for RunStore.
2669    pub run_store_private_path_failures: Option<car_secrets::PrivatePathDurabilityFailureInjector>,
2670    /// Deterministic critical-journal fault seam used by direct lifecycle tests.
2671    pub journal_failures: Option<car_eventlog::JournalFailureInjector>,
2672    /// Private self-heal detection and bounded-attempt ledger. The standalone daemon sets this to
2673    /// `<CAR_HOME>/selfheal/detections.jsonl`; embedders/tests can isolate it.
2674    pub selfheal_ledger: Option<PathBuf>,
2675    /// Detection cadence. Eligible default-on auto-fix runs from the same
2676    /// non-overlapping tick and is independently controlled by config.toml.
2677    pub selfheal_interval_secs: u64,
2678    /// Deterministic evidence seam for integration tests and embedders.
2679    #[doc(hidden)]
2680    pub selfheal_evidence: Option<crate::selfheal::SelfhealEvidence>,
2681    /// Bounded source-checkout probe. Embedders fail closed unless they supply
2682    /// candidates; the standalone daemon derives this from process-local paths.
2683    pub selfheal_source_probe: crate::selfheal::SelfhealSourceProbe,
2684    /// Deterministic seam for the checkout-built replay-verb eligibility probe.
2685    #[doc(hidden)]
2686    pub selfheal_replay_verb_probe: crate::selfheal::SelfhealReplayVerbProbe,
2687    /// Test-only override for the startup reconciliation acknowledgement bound.
2688    #[doc(hidden)]
2689    pub startup_reconciliation_acknowledgement_timeout: std::time::Duration,
2690    /// Test-only override for the bounded in-process resume lease.
2691    #[doc(hidden)]
2692    pub run_resume_lease: std::time::Duration,
2693}
2694
2695impl ServerStateConfig {
2696    /// Minimal config suitable for the standalone car-server binary:
2697    /// only the journal dir is required; everything else is lazily
2698    /// constructed at first use.
2699    pub fn new(journal_dir: PathBuf) -> Self {
2700        Self {
2701            journal_dir,
2702            peer_audit_journal: None,
2703            shared_memgine: None,
2704            identity_store: car_identity::IdentityStore::from_home(),
2705            inference: None,
2706            a2a_runtime: None,
2707            a2a_store: None,
2708            a2a_card_source: None,
2709            approval_gate: None,
2710            approval_journal: None,
2711            trajectory_dir: None,
2712            run_store_failures: None,
2713            run_store_summary_read_gate: None,
2714            run_store_summary_write_gate: None,
2715            run_store_append_gate: None,
2716            run_completion_fence_gate: None,
2717            run_store_lookup_gate: None,
2718            run_store_private_path_failures: None,
2719            journal_failures: None,
2720            selfheal_ledger: None,
2721            selfheal_interval_secs: crate::selfheal::DEFAULT_SELFHEAL_INTERVAL_SECS,
2722            selfheal_evidence: None,
2723            selfheal_source_probe: crate::selfheal::SelfhealSourceProbe::empty(),
2724            selfheal_replay_verb_probe: crate::selfheal::SelfhealReplayVerbProbe::default(),
2725            startup_reconciliation_acknowledgement_timeout:
2726                STARTUP_RECONCILIATION_ACKNOWLEDGEMENT_TIMEOUT,
2727            run_resume_lease: RUN_RESUME_LEASE,
2728        }
2729    }
2730
2731    pub fn with_shared_memgine(mut self, engine: Arc<Mutex<car_memgine::MemgineEngine>>) -> Self {
2732        self.shared_memgine = Some(engine);
2733        self
2734    }
2735
2736    /// Override the assistant identity/profile store.
2737    pub fn with_identity_store(mut self, store: car_identity::IdentityStore) -> Self {
2738        self.identity_store = store;
2739        self
2740    }
2741
2742    /// Override the peer-message audit journal path.
2743    pub fn with_peer_audit_journal(mut self, path: PathBuf) -> Self {
2744        self.peer_audit_journal = Some(path);
2745        self
2746    }
2747
2748    /// Override the shared approval-ledger journal path (default
2749    /// `~/.car/approvals.jsonl`).
2750    pub fn with_approval_journal(mut self, path: PathBuf) -> Self {
2751        self.approval_journal = Some(path);
2752        self
2753    }
2754
2755    pub fn with_trajectory_dir(mut self, dir: PathBuf) -> Self {
2756        self.trajectory_dir = Some(dir);
2757        self
2758    }
2759
2760    pub fn with_run_store_failures(
2761        mut self,
2762        failures: crate::run_store::RunStoreFailureInjector,
2763    ) -> Self {
2764        self.run_store_failures = Some(failures);
2765        self
2766    }
2767
2768    #[doc(hidden)]
2769    pub fn with_run_store_summary_read_gate(
2770        mut self,
2771        gate: crate::run_store::RunStoreSummaryReadGate,
2772    ) -> Self {
2773        self.run_store_summary_read_gate = Some(gate);
2774        self
2775    }
2776
2777    #[doc(hidden)]
2778    pub fn with_run_store_summary_write_gate(
2779        mut self,
2780        gate: crate::run_store::RunStoreSummaryWriteGate,
2781    ) -> Self {
2782        self.run_store_summary_write_gate = Some(gate);
2783        self
2784    }
2785
2786    #[doc(hidden)]
2787    pub fn with_run_store_append_gate(
2788        mut self,
2789        gate: crate::run_store::RunStoreAppendGate,
2790    ) -> Self {
2791        self.run_store_append_gate = Some(gate);
2792        self
2793    }
2794
2795    #[doc(hidden)]
2796    pub fn with_run_completion_fence_gate(mut self, gate: RunCompletionFenceGate) -> Self {
2797        self.run_completion_fence_gate = Some(gate);
2798        self
2799    }
2800
2801    #[doc(hidden)]
2802    pub fn with_run_resume_lease(mut self, lease: std::time::Duration) -> Self {
2803        self.run_resume_lease = lease;
2804        self
2805    }
2806
2807    #[doc(hidden)]
2808    pub fn with_run_store_lookup_gate(
2809        mut self,
2810        gate: crate::run_store::RunStoreLookupGate,
2811    ) -> Self {
2812        self.run_store_lookup_gate = Some(gate);
2813        self
2814    }
2815
2816    pub fn with_run_store_private_path_failures(
2817        mut self,
2818        failures: car_secrets::PrivatePathDurabilityFailureInjector,
2819    ) -> Self {
2820        self.run_store_private_path_failures = Some(failures);
2821        self
2822    }
2823
2824    pub fn with_journal_failures(mut self, failures: car_eventlog::JournalFailureInjector) -> Self {
2825        self.journal_failures = Some(failures);
2826        self
2827    }
2828
2829    pub fn with_selfheal_ledger(mut self, path: PathBuf) -> Self {
2830        self.selfheal_ledger = Some(path);
2831        self
2832    }
2833
2834    pub fn with_selfheal_interval_secs(mut self, interval_secs: u64) -> Self {
2835        self.selfheal_interval_secs = interval_secs.max(1);
2836        self
2837    }
2838
2839    #[doc(hidden)]
2840    pub fn with_selfheal_evidence(mut self, evidence: crate::selfheal::SelfhealEvidence) -> Self {
2841        self.selfheal_evidence = Some(evidence);
2842        self
2843    }
2844
2845    pub fn with_selfheal_source_probe(
2846        mut self,
2847        probe: crate::selfheal::SelfhealSourceProbe,
2848    ) -> Self {
2849        self.selfheal_source_probe = probe;
2850        self
2851    }
2852
2853    #[doc(hidden)]
2854    pub fn with_selfheal_replay_verb_probe(
2855        mut self,
2856        probe: crate::selfheal::SelfhealReplayVerbProbe,
2857    ) -> Self {
2858        self.selfheal_replay_verb_probe = probe;
2859        self
2860    }
2861
2862    #[doc(hidden)]
2863    pub fn with_startup_reconciliation_acknowledgement_timeout(
2864        mut self,
2865        timeout: std::time::Duration,
2866    ) -> Self {
2867        self.startup_reconciliation_acknowledgement_timeout = timeout;
2868        self
2869    }
2870
2871    pub fn with_inference(mut self, engine: Arc<car_inference::InferenceEngine>) -> Self {
2872        self.inference = Some(engine);
2873        self
2874    }
2875
2876    /// Plug in an embedder-supplied runtime for the A2A dispatcher.
2877    /// Use case: tokhn-daemon wants peers to see its OPA preflight
2878    /// tooling, not just CAR's `register_agent_basics` defaults.
2879    pub fn with_a2a_runtime(mut self, runtime: Arc<car_engine::Runtime>) -> Self {
2880        self.a2a_runtime = Some(runtime);
2881        self
2882    }
2883
2884    /// Plug in an embedder-supplied task store for the A2A
2885    /// dispatcher. Use case: tokhn's polling-friendly persistent
2886    /// store keyed by their session id.
2887    pub fn with_a2a_store(mut self, store: Arc<dyn car_a2a::TaskStore>) -> Self {
2888        self.a2a_store = Some(store);
2889        self
2890    }
2891
2892    /// Plug in an embedder-supplied agent card factory. The factory
2893    /// is invoked on every `agent/getAuthenticatedExtendedCard`
2894    /// dispatch, so embedders can reflect runtime tool changes.
2895    pub fn with_a2a_card_source(mut self, source: Arc<car_a2a::AgentCardSource>) -> Self {
2896        self.a2a_card_source = Some(source);
2897        self
2898    }
2899
2900    /// Override the approval-gate policy. Pass
2901    /// [`ApprovalGate::disabled`] to skip the gate entirely (only
2902    /// appropriate when no untrusted caller can reach the WS port);
2903    /// pass a customised [`ApprovalGate`] to add or remove methods
2904    /// or to change the timeout.
2905    pub fn with_approval_gate(mut self, gate: ApprovalGate) -> Self {
2906        self.approval_gate = Some(gate);
2907        self
2908    }
2909}
2910
2911/// The opted-in shared org-scope delivery subsystem + which org it serves. See
2912/// [`ServerState::org_sync`]. Held behind the same `std::sync::Mutex` as the
2913/// personal subsystem holder (init is serialized by the oplog's advisory lock);
2914/// the subsystem itself lives behind a `tokio::sync::Mutex`.
2915pub struct OrgSyncHolder {
2916    pub org: String,
2917    pub subsystem: Arc<tokio::sync::Mutex<crate::sync::SyncSubsystem>>,
2918}
2919
2920/// Which subsystem a write routes to. Pure decision, extracted so it is unit-
2921/// testable without a full daemon.
2922#[derive(Debug, PartialEq, Eq)]
2923enum SyncRoute {
2924    User,
2925    Org,
2926}
2927
2928/// Route by scope: a `Scope::Shared{org}` whose org matches the opted-in
2929/// `org_holder` goes to the org delivery subsystem; EVERYTHING else — `Personal`,
2930/// a different org, or no opted-in holder — goes to the personal subsystem. So
2931/// with `org_holder = None` (org-scope off) every scope routes `User`, byte-
2932/// identical to before.
2933fn route_for_scope(scope: &car_sync::Scope, org_holder: Option<&str>) -> SyncRoute {
2934    if let car_sync::Scope::Shared { org } = scope {
2935        if org_holder == Some(org.as_str()) {
2936            return SyncRoute::Org;
2937        }
2938    }
2939    SyncRoute::User
2940}
2941
2942/// Global server state shared across all connections.
2943pub struct ServerState {
2944    pub journal_dir: PathBuf,
2945    /// Resolved once when the state is constructed. Peer-message writes must
2946    /// use this path and never consult process-global `CAR_HOME` themselves.
2947    pub peer_audit_journal: PathBuf,
2948    pub sessions: Mutex<HashMap<String, Arc<ClientSession>>>,
2949    /// Serializes only session registry insertion/removal with the final
2950    /// liveness check + owner swap in `runs.resume`. This avoids holding the
2951    /// global session map while waiting on per-run durability and turns the
2952    /// replacement-present / old-owner-absent decision into one atomic fence.
2953    run_resume_liveness: Mutex<()>,
2954    run_resume_lease: std::time::Duration,
2955    run_completion_fence_gate: Option<RunCompletionFenceGate>,
2956    pub inference: std::sync::OnceLock<Arc<car_inference::InferenceEngine>>,
2957    pub host: Arc<crate::host::HostState>,
2958    /// When `Some`, `create_session` clones this handle into every new
2959    /// `ClientSession.memgine` — embedders that want a single shared
2960    /// memgine across all WS sessions set this. Standalone car-server
2961    /// leaves it `None`, which gives each session its own engine
2962    /// (preserving today's behavior).
2963    pub shared_memgine: Option<Arc<Mutex<car_memgine::MemgineEngine>>>,
2964    /// Source of truth for the assistant identity and user profile. Kept on the
2965    /// state so RPC tests/embedders can isolate it from the process environment.
2966    pub identity_store: car_identity::IdentityStore,
2967    /// Feedback doctor inputs resolved through [`ServerStateConfig`].
2968    pub(crate) feedback_diagnostics: crate::feedback::FeedbackDiagnostics,
2969    /// Daemon-wide trajectory store, cloned into **every** session's
2970    /// `Runtime` so execution traces persist to one place.
2971    ///
2972    /// Shared rather than per-session on purpose. A trajectory is evidence
2973    /// about how a tool behaves, and that evidence does not belong to the
2974    /// connection that happened to produce it — per-session stores would
2975    /// scatter the history across connections and reset it on every
2976    /// reconnect, leaving the derived success rates built from a handful of
2977    /// samples. This is the same reasoning that makes the approval ledger
2978    /// daemon-wide.
2979    ///
2980    /// Until now nothing called `Runtime::with_trajectory_store`, so
2981    /// `persist_trajectory` returned early on every execution and the store
2982    /// was dead code: `ToolFeedback::from_trajectories` existed with no data
2983    /// to read. Wiring it here is what makes the feedback loop real.
2984    pub trajectory_store: Arc<car_memgine::TrajectoryStore>,
2985    /// Daemon-wide self-healing detector/fixer + CAR_HOME ledger. Shared
2986    /// across sessions so `selfheal.run` and the cadence cannot overlap and
2987    /// every caller reads the same dismissals/detections.
2988    pub selfheal: Arc<crate::selfheal::SelfhealService>,
2989    /// The self-healing *repair* loop: reads a configured tracker, runs a coder
2990    /// session, gates the result on a multi-model panel, opens a pull request.
2991    ///
2992    /// Distinct from [`Self::selfheal`], which detects and never writes. Shared
2993    /// across sessions for the same reason: `heal.run` and the cadence must not
2994    /// overlap, and both must read the same claim ledger.
2995    pub heal: Arc<crate::coder::heal_service::HealService>,
2996    /// Process-wide voice session registry. Each
2997    /// `voice.transcribe_stream.start` call registers its own per-client
2998    /// [`WsVoiceEventSink`] so events route back to the originating WS
2999    /// connection only.
3000    pub voice_sessions: Arc<car_voice::VoiceSessionRegistry>,
3001    /// Process-wide meeting registry. Meeting ids are global; each
3002    /// meeting binds to the originating client's WS for upstream
3003    /// events but persists transcripts to the resolved
3004    /// `.car/meetings/<id>/` regardless of which client started it.
3005    pub meetings: Arc<car_meeting::MeetingRegistry>,
3006    /// Process-wide A2UI surface store. Agent-produced surfaces are
3007    /// visible to every host UI subscriber, independent of the
3008    /// WebSocket session that applied the update.
3009    pub a2ui: car_a2ui::A2uiSurfaceStore,
3010    /// In-process UI-improvement agent. Invoked from
3011    /// `handle_a2ui_render_report` with each inbound report; returned
3012    /// `Decision::Patch` envelopes are applied via the standard
3013    /// `apply_a2ui_envelope` path so all subscribers see the patch.
3014    /// `Arc` so the agent's interior `DashMap` state survives across
3015    /// handler calls even when `ServerState` is cheap-cloned.
3016    pub ui_agent: Arc<car_ui_agent::UIImprovementAgent>,
3017    /// Per-surface oscillation detector for the UI-improvement
3018    /// loop. Sits between the agent's `Decision::Patch` and the
3019    /// apply path so A→B→A patch cycles get cooled down without
3020    /// the agent itself having to track history. neo's review:
3021    /// "controllers use workqueue backoff; reconcilers stay
3022    /// stateless."
3023    pub ui_agent_oscillation: Arc<crate::ui_agent_loop::OscillationDetector>,
3024    /// Per-surface iteration budget. Backstop against runaway
3025    /// loops the oscillation detector misses — caps total agent-
3026    /// driven patches per surface at `DEFAULT_MAX_ITERATIONS`.
3027    pub ui_agent_budget: Arc<crate::ui_agent_loop::IterationBudget>,
3028    /// Process-wide concurrency gate for inference RPC handlers. Sized
3029    /// from host RAM at startup, overridable via
3030    /// [`crate::admission::ENV_MAX_CONCURRENT`]. Without this, N
3031    /// concurrent users multiply KV-cache and activation memory and
3032    /// take the host out (#114-adjacent: filed alongside the daemon
3033    /// always-on rework). The semaphore lives on `ServerState` so it
3034    /// is shared across every WebSocket session in the same process.
3035    pub admission: Arc<crate::admission::InferenceAdmission>,
3036    /// Server-side A2A continuation auth keyed by A2UI surface id.
3037    /// Kept out of `A2uiSurface.owner` so host renderers never see
3038    /// bearer/API-key material.
3039    pub a2ui_route_auth: Mutex<HashMap<String, A2aRouteAuth>>,
3040    /// Lifecycle-managed agents — declarative manifest at
3041    /// `~/.car/agents.json` driving spawn/restart/stop. Closes
3042    /// Parslee-ai/car-releases#27. Lazy-initialized so embedders that
3043    /// don't want process supervision don't pay the disk-touch cost
3044    /// at server start.
3045    pub supervisor: std::sync::OnceLock<Arc<car_registry::supervisor::Supervisor>>,
3046    /// Declarative (in-daemon) agent registry — a parallel store to the
3047    /// supervisor for agents the coder→agent loop builds. Lazy-initialized on
3048    /// first use (`~/.car/declagents.json`).
3049    pub declagents: std::sync::OnceLock<Arc<car_registry::declarative::DeclRegistry>>,
3050    /// Routing learning state — per-agent success stats + agent→agent edge
3051    /// weights for capability-similarity routing (`~/.car/routing.json`).
3052    /// Lazy-initialized on first use, sibling to [`Self::declagents`].
3053    pub routing: std::sync::OnceLock<Arc<car_registry::routing::RoutingStore>>,
3054    /// Multi-device sync + execution-lease subsystem (B6). Lazily opened on
3055    /// first `sync.*`/`lease.*` contact, rooted at `<journal_dir>/sync/` — a
3056    /// daemon-held [`car_sync::SyncSession`] over an `FsRelay` (the single-user
3057    /// two-device loopback works out of the box) plus an in-process
3058    /// linearizable [`car_sync::InMemoryLeaseCoordinator`]. **One subsystem per
3059    /// daemon = one device in the sync fleet.** The `std::sync::Mutex<Option>`
3060    /// serializes the fallible first-open (the oplog journal holds an exclusive
3061    /// advisory lock, so a double-open would fail) without holding the lock
3062    /// across the subsystem's async work — the subsystem itself lives behind a
3063    /// `tokio::sync::Mutex` inside the `Arc`. A distributed relay + a
3064    /// cross-daemon linearizable `LeaseCoordinator` backend is the documented
3065    /// B6 follow-up; the traits are its contract.
3066    pub sync: std::sync::Mutex<Option<Arc<tokio::sync::Mutex<crate::sync::SyncSubsystem>>>>,
3067    /// The SHARED org-scope delivery subsystem — a SECOND device oplog whose relay
3068    /// scope is `org:{orgId}`, so `Scope::Shared{org}` ops converge across an org's
3069    /// members. `None` unless org-scope is opted in (the same gated
3070    /// `PARSLEE_SYNC_ORG_SCOPE` branch that resolves the org key builds it);
3071    /// populated by `open_sync_subsystem` alongside the personal subsystem. Only
3072    /// the `Scope::Shared` write sites + `pump` route to it — the personal path
3073    /// (`sync_subsystem()` + all readers) is untouched. Slice 8a.
3074    pub org_sync: std::sync::Mutex<Option<OrgSyncHolder>>,
3075    /// Manifest path this daemon is *observing* but does NOT own.
3076    /// Set by `car-server` when boot-time supervisor construction
3077    /// fails with [`car_registry::supervisor::SupervisorError::AlreadyRunning`]
3078    /// — another car-server process on the host holds the exclusive
3079    /// lock on this manifest. In that state, `supervisor()` returns a
3080    /// clear "observe-only" error so mutation handlers refuse
3081    /// (preventing the duplicate-spawn bug from
3082    /// Parslee-ai/car-releases#44), while read-only handlers
3083    /// (`agents.list`, `agents.health`) fall back to
3084    /// [`car_registry::supervisor::Supervisor::list_from_manifest`] /
3085    /// [`car_registry::supervisor::Supervisor::health_from_manifest`]
3086    /// so operators can still inspect what the primary daemon is
3087    /// supervising.
3088    pub observer_manifest_path: std::sync::OnceLock<PathBuf>,
3089    /// In-core A2A dispatcher — embedders that consume `car-server-core`
3090    /// get A2A reachability "for free" without standing up a separate
3091    /// HTTP listener. Closes Parslee-ai/car-releases#28. Lazy-init so
3092    /// the embedder can override the runtime / task store / agent card
3093    /// via [`ServerStateConfig::with_a2a_runtime`] etc. before the
3094    /// first dispatch.
3095    pub a2a_dispatcher: std::sync::OnceLock<Arc<car_a2a::A2aDispatcher>>,
3096    /// WS clients subscribed to A2UI envelope events. After every
3097    /// successful `a2ui.apply` / `a2ui.ingest`, the resulting
3098    /// `A2uiApplyResult` is broadcast to every subscriber as an
3099    /// `a2ui.event` JSON-RPC notification. Closes
3100    /// Parslee-ai/car-releases#29. Subscribers register via the
3101    /// `a2ui/subscribe` method and are auto-cleaned on WS disconnect.
3102    pub a2ui_subscribers: Mutex<HashMap<String, Arc<WsChannel>>>,
3103    /// Per-launch auth token. When `Some`, the WS dispatcher rejects
3104    /// non-auth methods on unauthenticated sessions until the client
3105    /// calls `session.auth` with the matching value. When `None`,
3106    /// auth is disabled and every connection works as before. Set
3107    /// at startup by `car-server` unless `--no-auth` is passed
3108    /// (default flipped 2026-05); embedders that want to enable
3109    /// auth call [`ServerState::install_auth_token`]. Closes
3110    /// Parslee-ai/car-releases#32.
3111    /// Shared MCP tool executor backing remote connectors. One per
3112    /// process: every WS session's runtime executor is a
3113    /// [`car_engine::McpToolExecutor::share_with_fallback`] view over
3114    /// this, so connector tools route here while non-connector tools
3115    /// fall back to the per-session [`WsToolExecutor`]. Connectors
3116    /// (their live sessions + routes) live on this shared instance, so
3117    /// a connector enabled in one WS session is reachable from all.
3118    pub mcp_executor: Arc<car_engine::McpToolExecutor>,
3119    /// Remote MCP connector manager (`~/.car/connectors.json`).
3120    /// Lazy-initialized over [`mcp_executor`](Self::mcp_executor) so
3121    /// embedders that never touch connectors pay no disk cost. See
3122    /// [`ServerState::connectors`].
3123    pub connectors: std::sync::OnceLock<Arc<car_connectors::ConnectorManager>>,
3124    /// One-shot guard so persisted connectors are loaded + dialed at
3125    /// most once per process (see [`ServerState::ensure_connectors_loaded`]).
3126    connectors_loaded: std::sync::atomic::AtomicBool,
3127    /// Runtime approval-transport channel supervisor (Units 1/2/3). Lazy-
3128    /// initialized at daemon boot by `spawn_channel_pollers`, which records the
3129    /// channels it spawned at startup. The host-gated `messaging.config.set`
3130    /// handler reaches it via `ServerState` to spawn a channel's watcher the
3131    /// instant the user enables it — no restart (U1). Also holds the per-channel
3132    /// liveness `messaging.status` reads (U2) and the send path writes (U3).
3133    /// `None` for embedders that never boot the channel pollers.
3134    pub channel_supervisor: std::sync::OnceLock<Arc<crate::channel_supervisor::ChannelSupervisor>>,
3135    /// Daemon-owned operations which must outlive the WebSocket request that
3136    /// started them. The tasks hold no [`ClientSession`] or [`WsChannel`]; only
3137    /// their oneshot response waiters are connection-scoped.
3138    ///
3139    /// Two kinds of work live here. Credential-backed auth operations (attempt
3140    /// completion and ordinary auth mutations), so a socket close can release
3141    /// its sink immediately without cancelling a coordinator/keychain operation
3142    /// halfway through its durable result. And `coder.start`, which registers a
3143    /// session and provisions a worktree *before* a multi-minute contract
3144    /// derivation: once that side effect exists on disk, the run's lifetime
3145    /// must not be owned by the board that asked for it.
3146    pub durable_tasks: Mutex<tokio::task::JoinSet<String>>,
3147    /// Per-daemon incarnation token persisted into redeeming auth leases.
3148    /// Awaiting-callback reservations intentionally have no daemon owner and
3149    /// survive a restart; a redeeming lease owned by a different incarnation
3150    /// is an orphan and is terminally failed by completion-status reconciliation.
3151    pub auth_completion_owner_id: String,
3152    pub auth_token: std::sync::OnceLock<String>,
3153    /// Per-launch **host** token — a credential distinct from
3154    /// `auth_token`, granting the host-management role (host-class
3155    /// reads like cross-agent run traces). Critically it is **never**
3156    /// served over `GET /auth-token`: a session becomes host-role only
3157    /// by presenting this via `session.auth { host_token }`, and the
3158    /// only way to obtain it is reading the `0600` `host-token` file,
3159    /// which a different local user cannot. This is what stops any
3160    /// authenticated local client from self-elevating to host and
3161    /// reading every agent's run traces (Parslee-ai/car#254). When
3162    /// `None`, host-role can't be granted (no host reads).
3163    pub host_token: std::sync::OnceLock<String>,
3164    /// Mobile Parslee Core runtime URL this daemon can hand to consumer host apps.
3165    /// Set by `car-server` at startup from the daemon bind address or an
3166    /// explicit public URL override. The token remains the per-launch auth
3167    /// token; this field only names the WebSocket endpoint.
3168    pub mobile_runtime_url: std::sync::OnceLock<String>,
3169    /// Explicit public mobile runtime URL to register after an authorized
3170    /// credential action succeeds. Absent for the default loopback URL, which
3171    /// is discoverable locally but must not be advertised to Parslee cloud.
3172    pub mobile_registration_url: std::sync::OnceLock<String>,
3173    /// Parslee cloud identity activated by an explicit credential-backed
3174    /// request after `car auth login` has been completed.
3175    pub parslee_session: std::sync::OnceLock<crate::parslee_auth::ParsleeSession>,
3176    /// `agent_id -> client_id` map of currently-attached lifecycle
3177    /// agents (#169). Populated by the `session.auth` handler when a
3178    /// supervised child presents its `agent_id` + per-agent token;
3179    /// drained on disconnect by `remove_session`. Single-claim:
3180    /// a second connection presenting the same `agent_id` is
3181    /// rejected so the daemon-side per-agent state stays unambiguous.
3182    pub attached_agents: Mutex<HashMap<String, String>>,
3183    /// `agent_id -> persistent memgine` map (#170). Lazy-loaded on
3184    /// first connection per id from `~/.car/memory/agents/<id>.jsonl`,
3185    /// retained across daemon restart, surviving any single
3186    /// disconnect/reconnect of the supervised child. Connections
3187    /// that auth without an `agent_id` (browser, host, ad-hoc CLI)
3188    /// keep the per-WS ephemeral memgine on `ClientSession.memgine`
3189    /// — no behaviour change.
3190    pub agent_memgines: Mutex<HashMap<String, Arc<Mutex<car_memgine::MemgineEngine>>>>,
3191    /// Daemon-owned memgines keyed by a host-declared **memory namespace**
3192    /// (`session.auth { memory_namespace }`), lazily loaded from
3193    /// `~/.car/memory/memory-namespaces/<encoded-ns>.json` — the filename is a
3194    /// percent-encoding of the namespace, injective so two namespaces can never
3195    /// share a snapshot file (#891).
3196    ///
3197    /// A separate axis from `agent_memgines`, deliberately: one agent may work
3198    /// across several namespaces, and two hosts may share a namespace without
3199    /// sharing an identity (Parslee-ai/car-releases#79). A host that binds none
3200    /// keeps today's behaviour — the daemon's shared graph — so the MCP-to-WS
3201    /// shared knowledge base is untouched.
3202    pub namespace_memgines: Mutex<HashMap<String, Arc<Mutex<car_memgine::MemgineEngine>>>>,
3203    /// Live coder sessions keyed by `session_id` (built-in coding agent).
3204    /// Process-wide so a session outlives the WS connection that started
3205    /// it; terminal sessions stay listed for `coder.get`/`coder.list`
3206    /// history until daemon restart (snapshots persist under
3207    /// `~/.car/coder/`).
3208    pub coder_sessions: Mutex<crate::coder::rpc::CoderSessionMap>,
3209    /// Monotonic base for [`Self::coder_disk_gc_at`], taken at construction.
3210    pub coder_disk_gc_base: std::time::Instant,
3211    /// Seconds since [`Self::coder_disk_gc_base`] at the last coder state-dir
3212    /// sweep. Rate-limits the disk GC amortized onto `coder.start` (car#1339)
3213    /// so a burst of starts re-reads the directory once, not once each.
3214    ///
3215    /// Monotonic rather than Unix seconds because this is an INTERVAL: a wall
3216    /// clock that steps backwards would stamp a future value and suppress every
3217    /// later sweep for the daemon's lifetime, which is the bug this closes,
3218    /// reintroduced through the clock.
3219    ///
3220    /// On the state rather than a process-global static so each `ServerState` —
3221    /// every test builds its own — carries its own stamp, and one test's sweep
3222    /// cannot suppress another's.
3223    pub coder_disk_gc_at: AtomicU64,
3224    /// Live `coder.event` subscribers keyed by `(session_id, client_id)`
3225    /// — explicit fanout, same shape as `run_subscribers`. Lock order:
3226    /// the session's event buffer → this map; never the reverse.
3227    pub coder_subscribers: Mutex<HashMap<(String, String), Arc<WsChannel>>>,
3228    /// `coder.watch` board subscribers keyed by `client_id` — one
3229    /// subscription per connection covering EVERY session, so a board learns
3230    /// about runs started by any other client (`car code`, CarHost, milo)
3231    /// without polling. Fanned to as `coder.session_changed`. Lock order:
3232    /// a session's event buffer → `coder_subscribers` → this map; never the
3233    /// reverse.
3234    ///
3235    /// The value carries a **registration generation** alongside the channel.
3236    /// A board re-registers periodically (its `coder.watch` is idempotent and
3237    /// returns a fresh snapshot), and the fanout sheds a wedged watcher *after*
3238    /// releasing this lock — so a shed that removed by `client_id` alone would
3239    /// delete a registration made in that window and silently unwatch a healthy
3240    /// board. The shed compares the generation and removes only the entry it
3241    /// actually timed out on.
3242    pub coder_watchers: Mutex<HashMap<String, (u64, Arc<WsChannel>)>>,
3243    /// The one long-lived task that fans `coder.session_changed` out to
3244    /// [`coder_watchers`](Self::coder_watchers), lazily started on the first
3245    /// notification. Deliberately a single drain rather than a task per event:
3246    /// a running session emits on every tool call, and spawn-per-event against
3247    /// a half-open board socket accumulated blocked tasks without bound, each
3248    /// pinning the socket's write half past teardown (car#209's shape on a new
3249    /// path). Daemon-owned, so it is correctly absent from any connection's
3250    /// `conn_tasks`; it exits when this `ServerState` drops.
3251    pub(crate) coder_watch_notify: std::sync::OnceLock<mpsc::UnboundedSender<String>>,
3252    /// Live `coder.discuss.*` runtimes keyed by `discussion_id`. Model history
3253    /// is checkpointed separately; recovery rebuilds this connection-owned
3254    /// runtime after validating the saved repository and principal binding.
3255    pub coder_discussions: Mutex<crate::coder::discuss::DiscussionMap>,
3256    /// Serializes recovery admission, without holding the live registry lock
3257    /// while a repository runtime is rebuilt.
3258    pub(crate) coder_discussion_recovery: Mutex<()>,
3259    /// Open-discussion slots, one permit per allowed discussion. A permit is
3260    /// taken BEFORE `coder.discuss.start` does any of its expensive async work
3261    /// and lives inside the [`DiscussionEntry`](crate::coder::discuss::DiscussionEntry)
3262    /// it admitted, so it comes back when the discussion is dropped. Counting
3263    /// the map instead was a TOCTOU check — the count was read, the lock
3264    /// released, and two awaits (substrate bind + runtime build) ran before the
3265    /// insert, so N pipelined starts all passed a cap of N=8.
3266    pub(crate) coder_discussion_slots: Arc<tokio::sync::Semaphore>,
3267    /// In-flight `agents.chat` sessions keyed by `session_id`. See
3268    /// [`ChatSession`] for shape. Populated by `agents.chat`,
3269    /// cleared on terminal `agent.chat.event` or
3270    /// `agents.chat.cancel`. Disconnect cleanup happens in
3271    /// `remove_session` — any in-flight session bound to either the
3272    /// disconnecting host or agent client is dropped so subsequent
3273    /// stray notifications from a respawned agent fall on the floor
3274    /// rather than racing into a stale stream.
3275    pub chat_sessions: Mutex<HashMap<String, ChatSession>>,
3276    /// Per-recipient channel guards for peer messaging (`agents.message`).
3277    ///
3278    /// Keyed by recipient agent id. Bounds the *channel* rather than the
3279    /// sender's authority: size, identical repeats, per-sender rate, and queue
3280    /// depth. Two agents answering each other form a loop that no policy rule
3281    /// catches, because neither is misbehaving — this is what terminates it.
3282    ///
3283    /// Entries are dropped in `remove_session` when the agent detaches, so a
3284    /// restarted agent starts with a clean budget rather than inheriting the
3285    /// rate history of the process that used to own its name.
3286    pub peer_guards: Mutex<HashMap<String, car_peers::DeliveryGuard>>,
3287    /// MCP protocol sessions that currently own a peer identity.
3288    ///
3289    /// The HTTP transport mints these on `initialize` and removes them on
3290    /// protocol DELETE or idle expiry. Receive-capable sessions hold a bounded
3291    /// polled inbox; CAR-spawned batch CLI sessions are recorded only long
3292    /// enough to attribute their sends and never appear as recipients.
3293    pub(crate) mcp_peer_sessions: Mutex<HashMap<String, crate::peers::McpPeerSession>>,
3294    /// Earned standing per sending principal — a peer key inbound, a local
3295    /// agent id outbound.
3296    ///
3297    /// **Deliberately not reaped on detach**, unlike [`Self::peer_guards`] one
3298    /// line above. That hook exists so a respawned agent does not inherit a
3299    /// dead process's rate and dedupe budget, which is *channel* state.
3300    /// Standing is the opposite kind: clearing it on disconnect would let a
3301    /// degraded agent wipe its record by reattaching, which is a cheaper
3302    /// erasure than the renaming this design already refuses to allow.
3303    /// Bounded by TTL and map cap instead, pruned on write.
3304    pub peer_standing: Mutex<HashMap<String, crate::peers::PeerStanding>>,
3305    /// Peer messages held for operator approval, oldest first.
3306    ///
3307    /// A message is held when the sender's `read_only` posture is
3308    /// `require_approval`. Bounded at [`car_peers::HOLD_CAP`]; past that the
3309    /// oldest is dropped, so a stream of held messages cannot grow without
3310    /// limit. Kept flat rather than per-recipient because approval is an
3311    /// operator action over the whole queue, not a per-agent one.
3312    ///
3313    /// Deliberately in memory only: a held message is a live decision awaiting a
3314    /// human, and one that outlived a daemon restart would be delivered into a
3315    /// world that had moved on.
3316    pub held_peer_messages: Mutex<std::collections::VecDeque<crate::peers::HeldPeerMessage>>,
3317    /// Live view of CAR daemons discovered on the local network.
3318    ///
3319    /// `None` when mDNS could not start (no multicast, a locked-down sandbox) —
3320    /// LAN discovery is then simply absent rather than the daemon failing to
3321    /// boot. A network with no peers and a network CAR cannot browse look the
3322    /// same to a caller, which is why the status surface reports which it is.
3323    pub lan_discovery: std::sync::Mutex<Option<car_a2a::lan::LanDirectory>>,
3324    /// This daemon's ed25519 peer identity, and the keys it accepts.
3325    ///
3326    /// The identity signs outbound cross-host requests; the trust set decides
3327    /// which inbound signers are CAR. Both are `None` until the A2A surface
3328    /// starts, because neither is meaningful without a network surface to
3329    /// authenticate.
3330    pub peer_identity: std::sync::Mutex<Option<Arc<car_a2a::peer_auth::PeerIdentity>>>,
3331    pub peer_trust: car_a2a::peer_auth::PeerTrust,
3332    /// In-process collectors for `agent.chat` streams that have **no** host UI
3333    /// to forward to — currently the A2A conversational bridge, which reverse-
3334    /// calls `agent.chat` on a host session and aggregates the streamed deltas
3335    /// into a single reply. Keyed by `session_id`. When a collector exists for a
3336    /// session, `try_forward_agent_chat_event` feeds chunks here instead of
3337    /// forwarding to a host channel. The collecting task owns the entry's
3338    /// lifetime (inserts before the reverse-call, removes when done/timed out).
3339    pub chat_collectors: Mutex<HashMap<String, ChatCollector>>,
3340    /// Standing deterministic chat goals keyed by `session_id`. This is a
3341    /// host-facing status/control registry, separate from ephemeral
3342    /// `chat_sessions` routing so a goal can be inspected after a turn finishes.
3343    pub chat_goals: Mutex<HashMap<String, ChatGoalState>>,
3344    /// Agent runs keyed by `run_id` (agent run tracing, U1). Process-
3345    /// wide (not per-session) so a run's record outlives the WS
3346    /// connection that produced it — the durable, connection-
3347    /// independent boundary `client_id` cannot be (R1). Populated by
3348    /// `runs.start`, made terminal by `runs.complete`, and swept to
3349    /// `Incomplete` on a mid-run disconnect past the grace window
3350    /// (R5). U2/U3 build the per-turn recorder and disk store on top
3351    /// of this registry.
3352    pub runs: Mutex<HashMap<String, RunMeta>>,
3353    /// Per-run serialization for trace append/fsync and terminal persistence.
3354    /// Filesystem work never holds the global [`runs`](Self::runs) lock.
3355    run_durability_locks: Mutex<HashMap<String, std::sync::Weak<Mutex<()>>>>,
3356    /// Live `runs.trace.event` subscribers keyed by `(run_id,
3357    /// host_client_id)` (agent run tracing, U4). Each value is a
3358    /// [`crate::host::RunTraceSubscriber`] — the producer side of a
3359    /// bounded channel whose dedicated drain task writes frames to that
3360    /// connection's WS. Two CarHost windows on one run register two
3361    /// distinct entries (explicit fanout — the built-in notification
3362    /// registry is single-subscriber-per-method).
3363    ///
3364    /// **Lock contract (invariant #1):** `runs.subscribe` snapshots the
3365    /// run's turns AND inserts the subscriber while holding the
3366    /// [`runs`](Self::runs) lock; the recorder (`record_run_turns`),
3367    /// `start_run`, `complete_run`, and `mark_run_incomplete` append to
3368    /// `runs` and push to `run_subscribers` while holding the SAME
3369    /// `runs` lock — so snapshot/register and append/notify are
3370    /// serialized. No turn appended in the snapshot/register window is
3371    /// dropped (gap) or double-delivered (dup). The lock order is always
3372    /// `runs` → `run_subscribers`; never the reverse.
3373    pub run_subscribers: Mutex<HashMap<(String, String), crate::host::RunTraceSubscriber>>,
3374    /// Every browser the drawer can reach (`browser.view.*`), keyed by
3375    /// conversation/agent-session — plus the one standing user session every
3376    /// conversation without an agent-attached browser shares. Owns the
3377    /// per-view snapshot/cursor/subscriber fanout; see
3378    /// [`crate::browser_view`].
3379    pub browser_views: Arc<crate::browser_view::BrowserViewRegistry>,
3380    /// Disk-backed run-trace store (agent run tracing, U3). Source of
3381    /// truth for REPLAY (U5) — persists each run's `RunStarted`, turns,
3382    /// and terminal record as JSONL under `~/.car/runs/{agent_id}/` so a
3383    /// run survives daemon restarts (R4), bounded by retention/GC (R6)
3384    /// and protected at rest with `0600`/`0700` perms + backup exclusion
3385    /// (R14). The in-memory [`runs`](Self::runs) buffer stays the source
3386    /// for the LIVE stream (U4); this store mirrors what was recorded.
3387    /// Derived from `journal_dir` at construction (sibling `runs/`).
3388    pub run_store: crate::run_store::RunStore,
3389    journal_failures: Option<car_eventlog::JournalFailureInjector>,
3390    /// Bound MCP HTTP-streamable URL (e.g.
3391    /// `"http://127.0.0.1:9102/mcp"`) — `car-server` installs this
3392    /// after binding the listener. Used by the
3393    /// `agents.invoke_external` handler to default
3394    /// `InvokeOptions.mcp_endpoint` so external agents
3395    /// (Claude Code today) load the daemon's CAR namespace via
3396    /// `--mcp-config` automatically. `None` when MCP isn't bound
3397    /// (e.g. `--mcp-bind disabled`).
3398    pub mcp_url: std::sync::OnceLock<String>,
3399    /// Approval gate for high-risk WS methods (audit 2026-05). The
3400    /// gate intercepts `automation.run_applescript`,
3401    /// `automation.shortcuts.run`, `messages.send`, `mail.send`, the three
3402    /// mutating `calendar.*` methods, and `vision.ocr` before they dispatch, raises a
3403    /// `host.create_approval` for the user to act on, and waits
3404    /// (with a timeout) for `host.resolve_approval`. Approve →
3405    /// dispatch continues; deny / timeout → JSON-RPC error code
3406    /// `-32003`. The set of gated methods and the wait timeout are
3407    /// embedder-overridable via
3408    /// [`ServerStateConfig::with_approval_gate`].
3409    pub approval_gate: ApprovalGate,
3410    /// Out-of-process supervision (proposal item 5). Holds the subscribed
3411    /// supervisors and the intents parked on their verdicts; the matching
3412    /// [`crate::supervision::SupervisionGate`] is registered as an
3413    /// `AdmissionGate` on every session runtime alongside
3414    /// `StaticVerificationGate`.
3415    ///
3416    /// Daemon-wide rather than per-session, deliberately: a supervisor connects
3417    /// on its OWN WebSocket and supervises proposals submitted on OTHER
3418    /// sessions. A per-session registry would only ever see the supervisor's
3419    /// own (empty) traffic.
3420    ///
3421    /// Costs nothing until someone subscribes — the gate returns `Allow`
3422    /// without building an intent while the subscriber set is empty.
3423    pub supervision: Arc<crate::supervision::SupervisionRegistry>,
3424    /// The daemon-wide shared HITL approval ledger (kernel review C1) —
3425    /// journal-backed at `~/.car/approvals.jsonl` (override via
3426    /// [`ServerStateConfig::with_approval_journal`]). ONE ledger for all
3427    /// sessions: `permission.approve`/`reject` on any connection records
3428    /// here, and every ledger consumer (`permission.evaluate`/`pending`,
3429    /// `evolution.run`, `cascade.run`, `skill.enforce_deployment`/
3430    /// `ingest_governed`) reads here — so an approval granted by a host
3431    /// connection is visible to the agent connection that surfaced it and
3432    /// survives daemon restart. Per-session gates keep only tier/classifier
3433    /// state (see [`ClientSession::permission_gate`]). Falls back to an
3434    /// in-memory ledger (loudly warned, approvals NOT restart-durable) only
3435    /// when the journal is unopenable.
3436    pub approval_ledger: Arc<tokio::sync::RwLock<car_policy::ApprovalLedger>>,
3437    /// The in-process harness evaluator, when the binary installed one — the
3438    /// thing that lets `evolution.run` grade a harness candidate ITSELF
3439    /// instead of waiting for an operator to run `car-bench-harness` twice and
3440    /// hand both files back in.
3441    ///
3442    /// `None` on a build that did not install one (every embedder, and any
3443    /// binary other than `car-server`): `evolution.run` with `harness_measure`
3444    /// then ERRS rather than quietly falling back to HITL, because an opt-in
3445    /// that silently does nothing would report an unattended cycle that never
3446    /// measured anything.
3447    ///
3448    /// A `std::sync::RwLock` on purpose — no caller holds it across an await;
3449    /// they clone the `Arc` out and drop the guard.
3450    pub harness_measurer: std::sync::RwLock<Option<Arc<dyn crate::evolution::HarnessMeasurer>>>,
3451    /// A2A-runtime / store / card factory carried over from the
3452    /// embedder's [`ServerStateConfig`]. Consumed lazily on first
3453    /// `a2a_dispatcher()` call so embedders can construct
3454    /// `ServerState` without paying the runtime spin-up cost when
3455    /// they don't actually use the A2A surface.
3456    pub(crate) a2a_runtime: std::sync::Mutex<Option<Arc<car_engine::Runtime>>>,
3457    pub(crate) a2a_store: std::sync::Mutex<Option<Arc<dyn car_a2a::TaskStore>>>,
3458    pub(crate) a2a_card_source: std::sync::Mutex<Option<Arc<car_a2a::AgentCardSource>>>,
3459}
3460
3461/// Default location of the shared approval-ledger journal: `approvals.jsonl`
3462/// directly under the CAR state root, so it follows `CAR_HOME` rather than
3463/// pinning itself to `~/.car` (which is still where it lands when `CAR_HOME` is
3464/// unset — HOME, or USERPROFILE on Windows). Creates the root best-effort;
3465/// `None` when neither `CAR_HOME` nor a home directory is resolvable.
3466fn default_approval_journal_path() -> Option<PathBuf> {
3467    let dir = car_home::root()?;
3468    let _ = std::fs::create_dir_all(&dir);
3469    Some(dir.join("approvals.jsonl"))
3470}
3471
3472/// Resolve the peer audit journal once, while constructing [`ServerState`].
3473///
3474/// The standalone daemon's default journal directory is `<CAR_HOME>/journals`,
3475/// so preserve the established `<CAR_HOME>/peer-messages.jsonl` location in
3476/// that configuration. A custom journal directory — including every hermetic
3477/// test tempdir — owns its peer audit journal directly. No writer re-reads
3478/// `CAR_HOME`, so process-global state cannot redirect a configured test state.
3479fn default_peer_audit_journal_path(journal_dir: &Path) -> PathBuf {
3480    if let Some(car_dir) = car_home::root() {
3481        if journal_dir == car_dir.join("journals") {
3482            return car_dir.join("peer-messages.jsonl");
3483        }
3484    }
3485    journal_dir.join("peer-messages.jsonl")
3486}
3487
3488/// The daemon's state root: `CAR_HOME` when set, otherwise `~/.car` (HOME, or
3489/// USERPROFILE on Windows).
3490///
3491/// One convention, used by [`default_approval_journal_path`], the `messaging.*`
3492/// handlers and now the project policy loader — a second one would mean an
3493/// operator's rules living somewhere their pairing config does not. Going
3494/// through `car_home` is what keeps that true once an operator relocates the
3495/// root: everything moves together or nothing does.
3496fn car_home_dir() -> Option<PathBuf> {
3497    car_home::root()
3498}
3499
3500/// Register the declarative rules in `<car_dir>/policies/*.toml` on `runtime`.
3501///
3502/// **A malformed file is fatal, on purpose. Do not downgrade this to a
3503/// warning.** The loader's own words (`car_policy::load_policy_dir`): "a
3504/// security rule that fails to parse must surface, never be silently skipped."
3505/// A rule file exists to *stop* something; a typo that turns "never message
3506/// anyone outside the allowlist" into a no-op fails open, and it fails open
3507/// silently, at exactly the moment the operator believes they are protected. A
3508/// daemon that refuses to open the session is loud, immediate, and trivially
3509/// fixed; a daemon that logs a warning nobody reads is neither. The error text
3510/// names the offending file (`PolicyLoadError`'s `Display`) so the fix is
3511/// mechanical.
3512///
3513/// A **missing** directory is not a failure — most projects have no rules, and
3514/// `load_policy_dir` already returns an empty set for it.
3515///
3516/// Split out of [`ServerState::create_session`] so the fatal-vs-silent
3517/// behaviour is testable against a temp directory instead of the real `$HOME`.
3518pub(crate) async fn apply_project_policies(
3519    runtime: &Runtime,
3520    car_dir: &std::path::Path,
3521) -> Result<(), String> {
3522    match runtime.load_project_policies(car_dir).await {
3523        Ok(_) => Ok(()),
3524        Err(e) => Err(format!(
3525            "refusing to start with unreadable project policy rules: {e}. \
3526             Fix or remove the file — a policy rule that fails to parse is a \
3527             security control that would silently not exist."
3528        )),
3529    }
3530}
3531
3532impl ServerState {
3533    async fn run_durability_lock(&self, run_id: &str) -> Arc<Mutex<()>> {
3534        let mut locks = self.run_durability_locks.lock().await;
3535        if let Some(lock) = locks.get(run_id).and_then(std::sync::Weak::upgrade) {
3536            return lock;
3537        }
3538        locks.retain(|_, lock| lock.strong_count() > 0);
3539        let lock = Arc::new(Mutex::new(()));
3540        locks.insert(run_id.to_string(), Arc::downgrade(&lock));
3541        lock
3542    }
3543
3544    async fn quarantine_run_trace_locked(
3545        &self,
3546        runs: &mut HashMap<String, RunMeta>,
3547        run_id: &str,
3548        detail: String,
3549    ) -> String {
3550        let message = run_trace_corruption_message(run_id, detail);
3551        if let Some(meta) = runs.get_mut(run_id) {
3552            if meta.trace_corruption.as_deref() != Some(message.as_str()) {
3553                meta.durability_generation = meta.durability_generation.wrapping_add(1);
3554            }
3555            meta.trace_corruption = Some(message.clone());
3556        }
3557        let mut subs = self.run_subscribers.lock().await;
3558        subs.retain(|(subscribed_run, _), _| subscribed_run != run_id);
3559        message
3560    }
3561
3562    /// Publish corruption found by a strict replay read into live lifecycle
3563    /// state. Disk scanning and marker persistence happen before this call;
3564    /// only the per-run durability gate and short registry/subscriber mutation
3565    /// execute here, so a corrupt trace cannot remain writable or subscribed.
3566    pub(crate) async fn quarantine_run_trace_from_read(
3567        &self,
3568        run_id: &str,
3569        detail: String,
3570    ) -> String {
3571        let durability_lock = self.run_durability_lock(run_id).await;
3572        let _durability_guard = durability_lock.lock().await;
3573        let mut runs = self.runs.lock().await;
3574        self.quarantine_run_trace_locked(&mut runs, run_id, detail)
3575            .await
3576    }
3577
3578    /// Constructor for the standalone `car-server` binary. Each WS
3579    /// connection gets its own per-session memgine — matches the
3580    /// pre-extraction default and is correct for a single-process
3581    /// daemon serving one user at a time.
3582    ///
3583    /// **Embedders must not call this.** It silently leaves
3584    /// `shared_memgine = None`, which re-introduces the dual-memgine
3585    /// bug U7 was created to prevent (one engine in the embedder, a
3586    /// fresh one inside every WS session). Embedders use
3587    /// [`ServerState::embedded`] instead, which makes the shared
3588    /// engine handle a required argument so it cannot be forgotten.
3589    pub fn standalone(journal_dir: PathBuf) -> Self {
3590        Self::with_config(ServerStateConfig::new(journal_dir))
3591    }
3592
3593    /// The daemon-held multi-device-sync + execution-lease subsystem (B6),
3594    /// lazily opened on first `sync.*`/`lease.*` contact and rooted at
3595    /// `<journal_dir>/sync/`. Serialized first-open (see the [`Self::sync`]
3596    /// field docs): the oplog journal's exclusive advisory lock means only one
3597    /// open may succeed, so the fallible init runs under the field's
3598    /// `std::sync::Mutex` while the subsystem itself is shared behind a
3599    /// `tokio::sync::Mutex`.
3600    pub fn sync_subsystem(
3601        &self,
3602    ) -> Result<Arc<tokio::sync::Mutex<crate::sync::SyncSubsystem>>, String> {
3603        let mut guard = self
3604            .sync
3605            .lock()
3606            .map_err(|_| "sync subsystem init lock poisoned".to_string())?;
3607        if let Some(existing) = guard.as_ref() {
3608            return Ok(existing.clone());
3609        }
3610        let root = self.journal_dir.join("sync");
3611        let subsystem = self.open_sync_subsystem(&root)?;
3612        let arc = Arc::new(tokio::sync::Mutex::new(subsystem));
3613        *guard = Some(arc.clone());
3614        Ok(arc)
3615    }
3616
3617    /// Route a write to the right subsystem BY SCOPE: an opted-in
3618    /// `Scope::Shared{org}` goes to the shared org delivery subsystem (relay scope
3619    /// `org:{orgId}`, so it converges cross-member); everything else — `Personal`,
3620    /// or an org with no opted-in delivery — goes to the personal subsystem, which
3621    /// is byte-identical to before when org-scope is off. Used ONLY at the write
3622    /// sites; every reader keeps calling [`Self::sync_subsystem`]. (8a)
3623    pub fn subsystem_for_scope(
3624        &self,
3625        scope: &car_sync::Scope,
3626    ) -> Result<Arc<tokio::sync::Mutex<crate::sync::SyncSubsystem>>, String> {
3627        // Init the personal subsystem first — this also builds + stores the org
3628        // subsystem when opted in (via `open_sync_subsystem`).
3629        let user = self.sync_subsystem()?;
3630        let guard = self
3631            .org_sync
3632            .lock()
3633            .map_err(|_| "org sync lock poisoned".to_string())?;
3634        match route_for_scope(scope, guard.as_ref().map(|h| h.org.as_str())) {
3635            SyncRoute::Org => Ok(guard
3636                .as_ref()
3637                .expect("route_for_scope returns Org only when a holder exists")
3638                .subsystem
3639                .clone()),
3640            SyncRoute::User => Ok(user),
3641        }
3642    }
3643
3644    /// The opted-in shared org delivery subsystem, if any — for pumping both scopes.
3645    pub fn org_subsystem(&self) -> Option<Arc<tokio::sync::Mutex<crate::sync::SyncSubsystem>>> {
3646        match self.org_sync.lock() {
3647            Ok(g) => g.as_ref().map(|h| h.subsystem.clone()),
3648            // Surface the poison loudly — do NOT silently degrade to personal-only
3649            // sync (linus B). Poison only occurs on a panic while locked.
3650            Err(_) => {
3651                tracing::error!(
3652                    "org_sync lock poisoned — skipping the org pump (org-scope delivery degraded)"
3653                );
3654                None
3655            }
3656        }
3657    }
3658
3659    /// Build the sync subsystem for `root`, selecting the backend from config.
3660    ///
3661    /// Default is the local shared-directory `FsRelay`. When
3662    /// `PARSLEE_SYNC_BACKEND=parslee` (env or keychain), it builds the
3663    /// Parslee-backed device: a `ParsleeSyncTransport` at
3664    /// `PARSLEE_CAR_SYNC_BASE_URL` (default `{api_base}/sync`), authed with the
3665    /// live refresh-aware bearer, scoped `user:{account_id}`, E2E-encrypted under
3666    /// a passphrase-derived key (`PARSLEE_SYNC_PASSPHRASE`, the zero-knowledge
3667    /// cross-device key — same passphrase → same key on every device, never sent
3668    /// to the server). Missing a Parslee login falls back to local (logged);
3669    /// missing the passphrase is a hard error (refusing to sync unencrypted).
3670    fn open_sync_subsystem(
3671        &self,
3672        root: &std::path::Path,
3673    ) -> Result<crate::sync::SyncSubsystem, String> {
3674        let backend = car_secrets::resolve_env_or_keychain("PARSLEE_SYNC_BACKEND")
3675            .unwrap_or_default()
3676            .to_lowercase();
3677        if backend != "parslee" {
3678            return crate::sync::SyncSubsystem::open(root);
3679        }
3680
3681        let Some(account_id) = self
3682            .parslee_session
3683            .get()
3684            .map(|s| s.identity.account_id.clone())
3685            .filter(|a| !a.is_empty())
3686        else {
3687            tracing::warn!(
3688                "sync backend=parslee but no Parslee login is present; using local sync until login"
3689            );
3690            return crate::sync::SyncSubsystem::open(root);
3691        };
3692
3693        let Some(passphrase) = car_secrets::resolve_env_or_keychain("PARSLEE_SYNC_PASSPHRASE")
3694            .filter(|p| !p.is_empty())
3695        else {
3696            return Err(
3697                "sync backend=parslee requires PARSLEE_SYNC_PASSPHRASE (the zero-knowledge \
3698                 cross-device sync key) — set it in the keychain; refusing to sync unencrypted"
3699                    .to_string(),
3700            );
3701        };
3702
3703        let base_url = car_secrets::resolve_env_or_keychain("PARSLEE_CAR_SYNC_BASE_URL")
3704            .filter(|u| !u.is_empty())
3705            .unwrap_or_else(|| {
3706                let api_base = car_auth::api_base(None);
3707                format!("{}/sync", api_base.trim_end_matches('/'))
3708            });
3709
3710        let bearer: car_parslee::sync_transport::BearerFn = Arc::new(car_auth::access_token);
3711        let transport = Arc::new(
3712            car_parslee::sync_transport::ParsleeSyncTransport::new(base_url, bearer)
3713                .map_err(|e| format!("sync: build Parslee transport: {e}"))?,
3714        );
3715        // Argon2id-stretch the passphrase into the master ONCE at open (memory-hard
3716        // work factor for a password-equivalent secret), then feed BOTH the
3717        // per-audience provider and the org identity from it — so Argon2id runs
3718        // exactly once, not once per derivation.
3719        let master = car_sync::StretchedMaster::from_passphrase(passphrase.as_bytes(), &account_id);
3720        let base: Arc<dyn car_sync::SyncKeyProvider> =
3721            Arc::new(car_sync::DerivedKeyProvider::from_master(&master));
3722
3723        // Org-scope shared-brain activation — REAL provisioning, but DEFAULT-OFF.
3724        // `PARSLEE_SYNC_ORG_SCOPE=<orgId>:<granter_hex>[,<granter_hex>...]` names the
3725        // org AND the admin-designated trusted granter key(s), held LOCALLY (never
3726        // backend-fetched — see `parse_org_scope_config`). Unset → `base`
3727        // (byte-identical to the personal-only path). Malformed → hard error above.
3728        // The configured org MUST match the signed-in `active_organization` or we
3729        // refuse loudly. Turning it ON for a real tenant is slice 12, still gated on
3730        // the cryptographer audit of the login_secret→identity entropy dependency.
3731        let mut pending_org: Option<OrgSyncHolder> = None;
3732        let provider: Arc<dyn car_sync::SyncKeyProvider> = match Self::parse_org_scope_config(
3733            car_secrets::resolve_env_or_keychain("PARSLEE_SYNC_ORG_SCOPE"),
3734        )? {
3735            None => base,
3736            Some((org_id, granters)) => {
3737                // Bind the configured org to the signed-in active organization — never
3738                // encrypt org ops for an org this account is not actually a member of.
3739                let active_org = self
3740                    .parslee_session
3741                    .get()
3742                    .and_then(|s| s.identity.active_organization.clone())
3743                    .filter(|o| !o.is_empty());
3744                if active_org.as_deref() != Some(org_id.as_str()) {
3745                    return Err(format!(
3746                        "PARSLEE_SYNC_ORG_SCOPE org {org_id:?} does not match the signed-in active \
3747                         organization {active_org:?} — refusing to activate org scope"
3748                    ));
3749                }
3750
3751                // Bring the directory trait into scope for `publish_pubkey`.
3752                use car_sync::OrgKeyDirectory as _;
3753                let my_secret = car_sync::derive_x25519_identity(&master, &account_id);
3754                let mut directory = car_sync::NetworkOrgKeyDirectory::new(
3755                    transport.clone(),
3756                    format!("org:{org_id}"),
3757                );
3758
3759                // Announce our X25519 identity pubkey so a granter can wrap K_org for
3760                // us (idempotent LWW). Non-fatal: a failure just means future grants
3761                // may not reach us; any already-resolved roots still work. This routes
3762                // to the bearer-authenticated backend, but whether the backend
3763                // ENFORCES "only this account may publish its own pubkey" (and "only a
3764                // K_org holder may publish a wrap") is the confidentiality-critical,
3765                // AUDIT-OWNED check — this is not a guarantee against a hostile backend.
3766                let my_pub = car_sync::x25519_public(&my_secret);
3767                let my_pub_hex: String = my_pub
3768                    .as_bytes()
3769                    .iter()
3770                    .map(|b| format!("{b:02x}"))
3771                    .collect();
3772                if let Err(e) = directory.publish_pubkey(&account_id, &my_pub_hex) {
3773                    tracing::warn!(org = %org_id, error = %e, "org-scope: failed to publish member pubkey — future grants may not reach this member");
3774                }
3775
3776                // Resolve EVERY epoch of K_org we can unwrap (slice 10) → the
3777                // multi-epoch map the cipher selects by kid. `org → { epoch → K_org }`.
3778                // Fail-closed: Err (directory unreachable) and empty (ungranted) both
3779                // leave org scope on DenyCipher, NEVER demoted to the personal key.
3780                let mut roots = std::collections::HashMap::new();
3781                match car_sync::resolve_all_org_roots(
3782                    &directory,
3783                    &org_id,
3784                    &my_secret,
3785                    &account_id,
3786                    &granters,
3787                ) {
3788                    Ok(map) if !map.is_empty() => {
3789                        tracing::info!(org = %org_id, epochs = ?map.keys().collect::<Vec<_>>(), "org-scope: resolved K_org generations");
3790                        roots.insert(org_id.clone(), map);
3791                    }
3792                    Ok(_) => tracing::warn!(
3793                        org = %org_id,
3794                        "org-scope: no trusted grant for this member — org scope fails closed (DenyCipher)"
3795                    ),
3796                    Err(e) => tracing::warn!(
3797                        org = %org_id, error = %e,
3798                        "org-scope: org-key directory unreachable — org scope fails closed (DenyCipher), NOT demoted to the personal key"
3799                    ),
3800                }
3801                // ALWAYS OrgAwareKeyProvider once opted in — NEVER the bare
3802                // DerivedKeyProvider, which has no Scope::Shared case and would
3803                // silently encrypt org ops under the PERSONAL key (a fail-open).
3804                let provider: Arc<dyn car_sync::SyncKeyProvider> =
3805                    Arc::new(car_sync::OrgAwareKeyProvider::new(base, roots));
3806
3807                // Open the SHARED org delivery subsystem: a SECOND device oplog
3808                // under `root/org-{orgId}/` whose relay scope is `org:{orgId}`, with
3809                // the SAME OrgAwareKeyProvider (it keys Scope::Shared{org} on K_org).
3810                // Scope::Shared{org} writes route here (subsystem_for_scope) and
3811                // converge across the org's members. `org_id` is a canonical slug
3812                // (validated on parse), so it is filesystem-safe. (8a)
3813                let org_root = root.join(format!("org-{org_id}"));
3814                let org_subsystem = crate::sync::SyncSubsystem::open_remote(
3815                    &org_root,
3816                    transport.clone(),
3817                    format!("org:{org_id}"),
3818                    provider.clone(),
3819                )?;
3820                // Do NOT commit to `org_sync` yet — hold the org subsystem in a
3821                // local and only install it AFTER the personal open below succeeds.
3822                // Else a failed personal open would orphan the org journal's
3823                // exclusive lock and permanently wedge sync retry (linus A).
3824                pending_org = Some(OrgSyncHolder {
3825                    org: org_id.clone(),
3826                    subsystem: Arc::new(tokio::sync::Mutex::new(org_subsystem)),
3827                });
3828                tracing::info!(org = %org_id, "org-scope: opened shared org delivery subsystem (relay org:{org_id})");
3829                provider
3830            }
3831        };
3832        let user = crate::sync::SyncSubsystem::open_remote(
3833            root,
3834            transport,
3835            format!("user:{account_id}"),
3836            provider,
3837        )?;
3838        // Personal open succeeded — NOW commit the org holder. On the failure path
3839        // above, `pending_org` drops here, releasing the org journal lock so a
3840        // retry can re-open cleanly.
3841        if let Some(holder) = pending_org {
3842            *self
3843                .org_sync
3844                .lock()
3845                .map_err(|_| "org sync init lock poisoned".to_string())? = Some(holder);
3846        }
3847        Ok(user)
3848    }
3849
3850    /// Parse `PARSLEE_SYNC_ORG_SCOPE` (`<orgId>:<granter_hex>[,<granter_hex>...]`).
3851    /// `Ok(None)` when unset/empty → the personal-only path. `Err` on ANY malformed
3852    /// value — a typo must be a loud failure, never a silent fall-through to a
3853    /// wrong or unauthenticated state. Returns the org id + the (≥1) admin-designated
3854    /// trusted granter verifying keys.
3855    ///
3856    /// The granter set is held LOCALLY (operator-provisioned), NEVER fetched from
3857    /// the backend: whoever fills the `trusted` slice holds the authority, and if
3858    /// the same backend that serves the directory could name the granters it could
3859    /// name ITSELF a granter and substitute `K_org`. A backend-sourced / signed
3860    /// granter manifest is an audit-owned trust decision (slice 12), not this.
3861    /// Pure over the raw value so it is unit-testable without the environment.
3862    fn parse_org_scope_config(
3863        raw: Option<String>,
3864    ) -> Result<Option<(String, Vec<car_sync::OrgVerifyingKey>)>, String> {
3865        let Some(raw) = raw.filter(|s| !s.is_empty()) else {
3866            return Ok(None);
3867        };
3868        let (org_id, granters_csv) = raw.split_once(':').ok_or_else(|| {
3869            "PARSLEE_SYNC_ORG_SCOPE must be <orgId>:<granter_ed25519_pubkey_hex>[,...]".to_string()
3870        })?;
3871        // Honor the "loud failure" rule: a non-canonical org id (space, Unicode,
3872        // the platform's raw tenant form) must error here, not silently resolve
3873        // nothing — it would never match a real wrap's canonical `org` anyway.
3874        car_sync::require_canonical_org(org_id)
3875            .map_err(|e| format!("PARSLEE_SYNC_ORG_SCOPE: bad orgId: {e}"))?;
3876        let granters = granters_csv
3877            .split(',')
3878            .map(str::trim)
3879            .filter(|h| !h.is_empty())
3880            .map(|hex| {
3881                car_sync::parse_ed25519_verifying(hex)
3882                    .map_err(|e| format!("PARSLEE_SYNC_ORG_SCOPE: bad granter key {hex:?}: {e}"))
3883            })
3884            .collect::<Result<Vec<_>, _>>()?;
3885        if granters.is_empty() {
3886            return Err(
3887                "PARSLEE_SYNC_ORG_SCOPE: at least one granter ed25519 pubkey is required".into(),
3888            );
3889        }
3890        Ok(Some((org_id.to_string(), granters)))
3891    }
3892
3893    /// Persist the daemon-wide standing chat-goal registry. The snapshot is
3894    /// taken under the async mutex, then serialized and atomically rewritten on
3895    /// the blocking pool so chat/event dispatch does not block a Tokio worker.
3896    pub async fn persist_chat_goals(&self) -> Result<(), String> {
3897        let path = chat_goals_path_from_journal_dir(&self.journal_dir);
3898        let goals = self.chat_goals.lock().await.clone();
3899        tokio::task::spawn_blocking(move || {
3900            write_chat_goals_atomic(&path, &goals)
3901                .map_err(|e| format!("write {}: {e}", path.display()))
3902        })
3903        .await
3904        .map_err(|e| format!("persist chat goals join: {e}"))?
3905    }
3906
3907    /// Constructor for embedders (e.g. `tokhn-daemon`). The shared
3908    /// memgine handle is **required**: every WS session created by
3909    /// this state will reuse the same engine, preventing the
3910    /// dual-memgine bug.
3911    ///
3912    /// For embedders that also want to inject a pre-warmed inference
3913    /// engine or other advanced wiring, build a [`ServerStateConfig`]
3914    /// directly and call [`ServerState::with_config`].
3915    pub fn embedded(
3916        journal_dir: PathBuf,
3917        shared_memgine: Arc<Mutex<car_memgine::MemgineEngine>>,
3918    ) -> Self {
3919        Self::with_config(ServerStateConfig::new(journal_dir).with_shared_memgine(shared_memgine))
3920    }
3921
3922    /// Build a `ServerState` from a [`ServerStateConfig`] — the path
3923    /// embedders use when they need to inject a shared memgine *and*
3924    /// a pre-warmed inference engine, or any other advanced wiring
3925    /// the convenience constructors don't cover.
3926    pub fn with_config(cfg: ServerStateConfig) -> Self {
3927        Self::try_with_config(cfg)
3928            .unwrap_or_else(|error| panic!("server state startup failed: {error}"))
3929    }
3930
3931    /// Build a state with test-owned feedback diagnostic roots and no ambient
3932    /// registry probes. This bypasses the production diagnostic default rather
3933    /// than constructing it and then overriding it, so state construction never
3934    /// resolves `HOME`/`USERPROFILE` for feedback. Public only because Cargo
3935    /// integration tests link `car-server-core` without `cfg(test)`.
3936    #[doc(hidden)]
3937    pub fn with_config_and_isolated_feedback_diagnostics(
3938        cfg: ServerStateConfig,
3939        models_dir: PathBuf,
3940        huggingface_hub_root: PathBuf,
3941    ) -> Self {
3942        Self::try_with_config_and_feedback_diagnostics(
3943            cfg,
3944            crate::feedback::FeedbackDiagnostics::isolated(models_dir, huggingface_hub_root),
3945        )
3946        .unwrap_or_else(|error| panic!("server state startup failed: {error}"))
3947    }
3948
3949    /// Fallible startup surface used by the daemon before it begins listening.
3950    /// Durable lifecycle outbox replay is acknowledgement-bounded; any
3951    /// durability-unknown result stops construction and leaves its RunStore
3952    /// preimage/quarantine intact for an exact later retry.
3953    pub fn try_with_config(cfg: ServerStateConfig) -> Result<Self, String> {
3954        Self::try_with_config_and_feedback_diagnostics(
3955            cfg,
3956            crate::feedback::FeedbackDiagnostics::production(),
3957        )
3958    }
3959
3960    fn try_with_config_and_feedback_diagnostics(
3961        cfg: ServerStateConfig,
3962        feedback_diagnostics: crate::feedback::FeedbackDiagnostics,
3963    ) -> Result<Self, String> {
3964        let startup_acknowledgement_timeout = cfg.startup_reconciliation_acknowledgement_timeout;
3965        if startup_acknowledgement_timeout.is_zero()
3966            || startup_acknowledgement_timeout > car_eventlog::MAX_CRITICAL_ACKNOWLEDGEMENT_TIMEOUT
3967        {
3968            return Err(format!(
3969                "startup reconciliation acknowledgement timeout must be between 1ns and {}ms, got {}ms",
3970                car_eventlog::MAX_CRITICAL_ACKNOWLEDGEMENT_TIMEOUT.as_millis(),
3971                startup_acknowledgement_timeout.as_millis()
3972            ));
3973        }
3974        let inference = std::sync::OnceLock::new();
3975        if let Some(eng) = cfg.inference {
3976            // OnceLock::set returns Err if already set — fresh OnceLock
3977            // means it's empty, so this is infallible here.
3978            let _ = inference.set(eng);
3979        }
3980        let voice_sessions = Arc::new(car_voice::VoiceSessionRegistry::new());
3981        // Reap sessions whose clients dropped without calling
3982        // voice.transcribe_stream.stop (WS disconnect, process exit,
3983        // etc.). Listener handles otherwise leak for the daemon's
3984        // lifetime. `with_config` is sync but always called from the
3985        // `#[tokio::main]` entry point, so `Handle::try_current()`
3986        // inside `start_sweeper` finds the runtime.
3987        voice_sessions.start_sweeper();
3988        // UI-improvement agent is pure decision logic — no I/O, no
3989        // persistence handle. Memgine ingest of strategy outcomes is
3990        // the caller's responsibility (handler.rs after a successful
3991        // Decision::Patch). Keeps the agent crate Mutex-flavor
3992        // agnostic so it can compose with std/tokio mutex callers.
3993        let ui_agent = Arc::new(car_ui_agent::UIImprovementAgent::with_default_strategies());
3994        let ui_agent_oscillation = Arc::new(crate::ui_agent_loop::OscillationDetector::new());
3995        let ui_agent_budget = Arc::new(crate::ui_agent_loop::IterationBudget::new());
3996        // Disk-backed run-trace store (U3). Rooted at the `runs/` sibling
3997        // of the journal dir (`~/.car/runs`), with retention read from
3998        // `~/.car/config.toml`. Boot-time GC enforces the retention cap
3999        // (R6) — best-effort; a failed GC must never block startup, so the
4000        // count is dropped. Never evicts an in-progress run.
4001        let mut run_store = crate::run_store::RunStore::from_journal_dir(&cfg.journal_dir);
4002        if let Some(failures) = cfg.run_store_failures.clone() {
4003            run_store = run_store.with_failure_injector(failures);
4004        }
4005        if let Some(gate) = cfg.run_store_summary_read_gate.clone() {
4006            run_store = run_store.with_summary_read_gate(gate);
4007        }
4008        if let Some(gate) = cfg.run_store_summary_write_gate.clone() {
4009            run_store = run_store.with_summary_write_gate(gate);
4010        }
4011        if let Some(gate) = cfg.run_store_append_gate.clone() {
4012            run_store = run_store.with_append_gate(gate);
4013        }
4014        if let Some(gate) = cfg.run_store_lookup_gate.clone() {
4015            run_store = run_store.with_lookup_gate(gate);
4016        }
4017        if let Some(failures) = cfg.run_store_private_path_failures.clone() {
4018            run_store = run_store.with_private_path_failure_injector(failures);
4019        }
4020        run_store
4021            .reconcile_proposal_retry_rollbacks()
4022            .map_err(|error| format!("proposal retry rollback reconciliation failed: {error}"))?;
4023        let chat_goals = load_chat_goals_from_disk(&cfg.journal_dir);
4024        let selfheal_ledger = cfg.selfheal_ledger.clone().unwrap_or_else(|| {
4025            cfg.journal_dir
4026                .parent()
4027                .unwrap_or(&cfg.journal_dir)
4028                .join("selfheal")
4029                .join("detections.jsonl")
4030        });
4031        // Config lives beside the daemon's other state. A missing or empty
4032        // `heal.toml` is a disabled loop, which is the default: there are no
4033        // compiled-in targets, because a loop that opened pull requests in a
4034        // repository nobody named would be a surprise in somebody else's
4035        // tracker.
4036        let heal_state_dir =
4037            crate::coder::rpc::coder_state_dir().unwrap_or_else(|_| cfg.journal_dir.join("coder"));
4038        let heal_config_dir = car_home::root_or_relative();
4039        let heal = Arc::new(crate::coder::heal_service::HealService::with_config_dir(
4040            crate::coder::heal_config::HealConfig::load(&heal_config_dir),
4041            heal_state_dir,
4042            heal_config_dir,
4043        ));
4044        let selfheal = Arc::new(crate::selfheal::SelfhealService::open(
4045            selfheal_ledger,
4046            cfg.selfheal_interval_secs,
4047            cfg.selfheal_evidence.clone(),
4048            cfg.selfheal_source_probe.clone(),
4049            cfg.selfheal_replay_verb_probe.clone(),
4050        )?);
4051        let mut startup_journals =
4052            StartupJournalCache::new(&cfg.journal_dir, cfg.journal_failures.as_ref());
4053        // Establish the durable RunStarted bracket before replaying a pending
4054        // proposal terminal. Then adopt the crash orphan and run lifecycle
4055        // reconciliation again to add RunCompleted. Both passes share the
4056        // per-client journal cache, are exact-row idempotent, and preserve:
4057        // RunStarted -> ProposalCompleted -> RunCompleted.
4058        reconcile_durable_run_journals(
4059            &mut startup_journals,
4060            &run_store,
4061            startup_acknowledgement_timeout,
4062        )?;
4063        reconcile_pending_proposal_journals(
4064            &mut startup_journals,
4065            &run_store,
4066            startup_acknowledgement_timeout,
4067        )?;
4068        reconcile_completed_proposal_guards(&run_store);
4069        // The in-memory map is empty at boot, so any remaining on-disk
4070        // InProgress run is a crashed prior process; mark it Incomplete so it
4071        // is terminal and age-GC-eligible (FIX 4). Adoption must precede the
4072        // only boot GC so those runs do not remain immortal.
4073        let adopted = run_store.adopt_orphans();
4074        tracing::info!(
4075            adopted,
4076            "adopted crash-orphaned run traces at daemon startup"
4077        );
4078        reconcile_durable_run_journals(
4079            &mut startup_journals,
4080            &run_store,
4081            startup_acknowledgement_timeout,
4082        )?;
4083        let evicted = run_store.gc();
4084        if evicted > 0 {
4085            // Said out loud, for the same reason the coder GC below says it: an
4086            // operator whose own config just ate part of their store has to be
4087            // able to see it happened. This was silent, which is what made a
4088            // zero cap a data-loss incident rather than a log line (car#1338) —
4089            // and `max_age_days = 1` where someone meant `10` still deletes nine
4090            // days of traces, which the age of the policy has nothing to do with.
4091            tracing::info!(
4092                evicted,
4093                max_per_agent = ?run_store.retention().count_cap(),
4094                max_age_days = ?run_store.retention().age_cap_days(),
4095                "pruned run traces (~/.car/config.toml [runs] retention)"
4096            );
4097        }
4098        // Adopt crash/restart-orphaned coder sessions the same way: the
4099        // in-memory `coder_sessions` map is empty at boot, so any non-terminal
4100        // snapshot under the coder state dir is a prior process's orphan. Mark
4101        // genuinely-stranded ones Failed ("daemon restarted mid-session") and
4102        // preserve a `needs_approval` orphan whose worktree still exists so the
4103        // user can inspect/approve it by hand. Best-effort — never blocks boot.
4104        if let Ok(coder_dir) = crate::coder::rpc::coder_state_dir() {
4105            let _coder_adopted = crate::coder::session::adopt_orphaned_sessions(&coder_dir);
4106            // Then prune, the same way `run_store.gc()` above prunes run
4107            // traces. Adoption runs first on purpose: it is what turns a
4108            // stranded orphan terminal, and only terminal sessions are
4109            // collectable. An orphan adopted just now carries a fresh
4110            // `updated_at`, so this boot never collects what this boot
4111            // rewrote — it ages out on a later one.
4112            let retention = crate::coder::config::CoderConfig::load().session_retention();
4113            let collected = crate::coder::session::gc_sessions(
4114                &coder_dir,
4115                &retention,
4116                crate::coder::session::SweepScope::Boot,
4117            );
4118            if collected > 0 {
4119                // Said out loud, unlike `run_store.gc()` above. That policy has
4120                // existed since the run store did; this one is NEW and applies
4121                // retroactively, so the first boot after an upgrade can delete
4122                // years of snapshots under caps the operator never chose. An
4123                // operator who did not want that has to be able to see it
4124                // happened.
4125                tracing::info!(
4126                    collected,
4127                    max_sessions = retention.max_sessions,
4128                    max_age_days = retention.max_age_days,
4129                    "pruned coder session snapshots (~/.car/coder.toml retention)"
4130                );
4131            }
4132        }
4133        // The daemon-wide shared approval ledger (kernel review C1): one
4134        // journal-backed store for every session's HITL records, so approvals
4135        // cross connections and survive restart. In-memory fallback is loud —
4136        // a silent downgrade here would fake durability.
4137        let peer_audit_journal = cfg
4138            .peer_audit_journal
4139            .clone()
4140            .unwrap_or_else(|| default_peer_audit_journal_path(&cfg.journal_dir));
4141        let approval_ledger = {
4142            let path = cfg.approval_journal.or_else(default_approval_journal_path);
4143            match path {
4144                Some(p) => {
4145                    // The journal's parent must exist before the first
4146                    // append-mode open (ApprovalLedger::record does not
4147                    // create directories).
4148                    if let Some(parent) = p.parent() {
4149                        let _ = std::fs::create_dir_all(parent);
4150                    }
4151                    match car_policy::ApprovalLedger::with_journal(&p) {
4152                        Ok(l) => {
4153                            if l.skipped_on_load() > 0 {
4154                                tracing::warn!(
4155                                    path = %p.display(),
4156                                    skipped = l.skipped_on_load(),
4157                                    "approval-ledger journal had unparseable lines; partial ledger loaded"
4158                                );
4159                            }
4160                            l
4161                        }
4162                        Err(e) => {
4163                            tracing::warn!(
4164                                path = %p.display(), error = %e,
4165                                "approval-ledger journal unopenable; falling back to IN-MEMORY \
4166                                 ledger — approvals will NOT survive restart"
4167                            );
4168                            car_policy::ApprovalLedger::new()
4169                        }
4170                    }
4171                }
4172                None => {
4173                    tracing::warn!(
4174                        "no CAR_HOME/HOME/USERPROFILE to place approvals.jsonl under the CAR \
4175                         state root; approval ledger is IN-MEMORY — approvals will NOT survive \
4176                         restart"
4177                    );
4178                    car_policy::ApprovalLedger::new()
4179                }
4180            }
4181        };
4182        // Identity nodes are deliberately absent from memgine snapshots:
4183        // identity.json is authoritative and is re-mirrored on every daemon
4184        // startup. This avoids two durable copies drifting after a profile edit.
4185        if let Some(engine) = cfg.shared_memgine.as_ref() {
4186            match cfg.identity_store.load() {
4187                Ok(identity) => {
4188                    let mut engine = engine.try_lock().map_err(|_| {
4189                        "server state startup could not initialize the shared memgine identity: \
4190                         the supplied engine is already locked"
4191                            .to_string()
4192                    })?;
4193                    mirror_identity_into_memgine(&mut engine, &identity);
4194                }
4195                Err(error) => tracing::warn!(
4196                    error = %error,
4197                    "identity profile could not be mirrored at daemon startup"
4198                ),
4199            }
4200        }
4201
4202        // Built before the struct literal so the browser-view registry can be
4203        // pointed at the SAME `HostState` it will broadcast sign-in attention
4204        // on (`host.event`) — see `crate::browser_attention`.
4205        let host = Arc::new(crate::host::HostState::new());
4206        let browser_views = Arc::new(crate::browser_view::BrowserViewRegistry::default());
4207        browser_views.set_signin_attention(Arc::new(
4208            crate::browser_attention::HostSignInAttention::new(Arc::clone(&host)),
4209        ));
4210        Ok(Self {
4211            journal_dir: cfg.journal_dir,
4212            peer_audit_journal,
4213            sessions: Mutex::new(HashMap::new()),
4214            inference,
4215            host,
4216            shared_memgine: cfg.shared_memgine,
4217            identity_store: cfg.identity_store,
4218            feedback_diagnostics,
4219            trajectory_store: Arc::new(car_memgine::TrajectoryStore::new(
4220                &cfg.trajectory_dir
4221                    .unwrap_or_else(car_memgine::TrajectoryStore::default_path),
4222            )),
4223            selfheal,
4224            heal,
4225            voice_sessions,
4226            meetings: Arc::new(car_meeting::MeetingRegistry::new()),
4227            a2ui: car_a2ui::A2uiSurfaceStore::new(),
4228            ui_agent,
4229            ui_agent_oscillation,
4230            ui_agent_budget,
4231            admission: Arc::new(crate::admission::InferenceAdmission::new()),
4232            a2ui_route_auth: Mutex::new(HashMap::new()),
4233            supervisor: std::sync::OnceLock::new(),
4234            declagents: std::sync::OnceLock::new(),
4235            routing: std::sync::OnceLock::new(),
4236            sync: std::sync::Mutex::new(None),
4237            org_sync: std::sync::Mutex::new(None),
4238            observer_manifest_path: std::sync::OnceLock::new(),
4239            a2a_dispatcher: std::sync::OnceLock::new(),
4240            a2a_runtime: std::sync::Mutex::new(cfg.a2a_runtime),
4241            a2a_store: std::sync::Mutex::new(cfg.a2a_store),
4242            a2a_card_source: std::sync::Mutex::new(cfg.a2a_card_source),
4243            a2ui_subscribers: Mutex::new(HashMap::new()),
4244            mcp_executor: Arc::new(car_engine::McpToolExecutor::new()),
4245            connectors: std::sync::OnceLock::new(),
4246            connectors_loaded: std::sync::atomic::AtomicBool::new(false),
4247            channel_supervisor: std::sync::OnceLock::new(),
4248            durable_tasks: Mutex::new(tokio::task::JoinSet::new()),
4249            auth_completion_owner_id: uuid::Uuid::new_v4().simple().to_string(),
4250            auth_token: std::sync::OnceLock::new(),
4251            host_token: std::sync::OnceLock::new(),
4252            mobile_runtime_url: std::sync::OnceLock::new(),
4253            mobile_registration_url: std::sync::OnceLock::new(),
4254            parslee_session: std::sync::OnceLock::new(),
4255            attached_agents: Mutex::new(HashMap::new()),
4256            agent_memgines: Mutex::new(HashMap::new()),
4257            namespace_memgines: Mutex::new(HashMap::new()),
4258            coder_sessions: Mutex::new(HashMap::new()),
4259            coder_disk_gc_base: std::time::Instant::now(),
4260            // `0` IS "just swept": boot ran the same sweep a few lines above,
4261            // and zero seconds have elapsed since the base taken on the line
4262            // above this one. The first `coder.start` is therefore a no-op
4263            // rather than an immediate re-scan.
4264            coder_disk_gc_at: AtomicU64::new(0),
4265            coder_subscribers: Mutex::new(HashMap::new()),
4266            coder_watchers: Mutex::new(HashMap::new()),
4267            coder_watch_notify: std::sync::OnceLock::new(),
4268            coder_discussions: Mutex::new(HashMap::new()),
4269            coder_discussion_recovery: Mutex::new(()),
4270            coder_discussion_slots: Arc::new(tokio::sync::Semaphore::new(
4271                crate::coder::discuss::MAX_OPEN_DISCUSSIONS,
4272            )),
4273            chat_sessions: Mutex::new(HashMap::new()),
4274            peer_guards: Mutex::new(HashMap::new()),
4275            mcp_peer_sessions: Mutex::new(HashMap::new()),
4276            peer_standing: Mutex::new(HashMap::new()),
4277            held_peer_messages: Mutex::new(std::collections::VecDeque::new()),
4278            lan_discovery: std::sync::Mutex::new(None),
4279            peer_identity: std::sync::Mutex::new(None),
4280            peer_trust: car_a2a::peer_auth::PeerTrust::new(),
4281            chat_collectors: Mutex::new(HashMap::new()),
4282            chat_goals: Mutex::new(chat_goals),
4283            runs: Mutex::new(HashMap::new()),
4284            run_resume_liveness: Mutex::new(()),
4285            run_resume_lease: cfg.run_resume_lease,
4286            run_completion_fence_gate: cfg.run_completion_fence_gate,
4287            run_durability_locks: Mutex::new(HashMap::new()),
4288            run_subscribers: Mutex::new(HashMap::new()),
4289            browser_views,
4290            run_store,
4291            journal_failures: cfg.journal_failures,
4292            mcp_url: std::sync::OnceLock::new(),
4293            approval_gate: cfg.approval_gate.unwrap_or_default(),
4294            supervision: Arc::new(crate::supervision::SupervisionRegistry::default()),
4295            approval_ledger: Arc::new(tokio::sync::RwLock::new(approval_ledger)),
4296            // Installed by the daemon binary at startup (see
4297            // `car-server/src/main.rs`), never here: the implementation lives
4298            // in `car-bench`, which depends on this crate.
4299            harness_measurer: std::sync::RwLock::new(None),
4300        })
4301    }
4302
4303    /// Install the in-process harness evaluator `evolution.run`'s
4304    /// `harness_measure` path grades candidates with. Called once at daemon
4305    /// startup; a later call replaces it.
4306    pub fn set_harness_measurer(&self, m: Arc<dyn crate::evolution::HarnessMeasurer>) {
4307        match self.harness_measurer.write() {
4308            Ok(mut guard) => *guard = Some(m),
4309            // Poisoned only if a panic happened while the lock was held.
4310            // Refusing loudly beats leaving `evolution.run` to believe an
4311            // evaluator is installed when none is.
4312            Err(e) => tracing::error!(
4313                error = %e,
4314                "harness_measurer lock poisoned; the in-process harness evaluator was NOT installed"
4315            ),
4316        }
4317    }
4318
4319    /// The installed harness evaluator, if any. Clones the `Arc` out and drops
4320    /// the guard so callers never hold a `std::sync` lock across an await.
4321    pub fn harness_measurer(&self) -> Option<Arc<dyn crate::evolution::HarnessMeasurer>> {
4322        match self.harness_measurer.read() {
4323            Ok(guard) => guard.clone(),
4324            Err(e) => {
4325                tracing::error!(error = %e, "harness_measurer lock poisoned; treating as absent");
4326                None
4327            }
4328        }
4329    }
4330
4331    /// Start one daemon-owned operation and return a connection-scoped result
4332    /// receiver. Dropping the receiver never cancels the operation.
4333    ///
4334    /// The caller awaits the receiver exactly as it would have awaited the
4335    /// operation, so response shape and timing are unchanged for a connection
4336    /// that stays alive. What changes is the failure mode: when the connection
4337    /// goes away, the per-connection `JoinSet` aborts only this waiter, and the
4338    /// operation itself runs to completion on the daemon.
4339    pub async fn spawn_durable_operation<F, E>(
4340        &self,
4341        operation_name: impl Into<String>,
4342        operation: F,
4343    ) -> tokio::sync::oneshot::Receiver<Result<serde_json::Value, E>>
4344    where
4345        F: std::future::Future<Output = Result<serde_json::Value, E>> + Send + 'static,
4346        E: Send + 'static,
4347    {
4348        let (result_tx, result_rx) = tokio::sync::oneshot::channel();
4349        let operation_name = operation_name.into();
4350        let mut tasks = self.durable_tasks.lock().await;
4351        while let Some(result) = tasks.try_join_next() {
4352            if let Err(error) = result {
4353                tracing::warn!(error = %error, "daemon-owned task failed to join");
4354            }
4355        }
4356        tasks.spawn(async move {
4357            let result = operation.await;
4358            if result_tx.send(result).is_err() {
4359                tracing::debug!(
4360                    operation = %operation_name,
4361                    "daemon-owned operation finished after its response waiter closed"
4362                );
4363            }
4364            operation_name
4365        });
4366        result_rx
4367    }
4368
4369    /// Start one fire-and-reconcile daemon-owned operation.
4370    pub async fn spawn_durable_task<F>(&self, operation_name: impl Into<String>, operation: F)
4371    where
4372        F: std::future::Future<Output = ()> + Send + 'static,
4373    {
4374        let operation_name = operation_name.into();
4375        let mut tasks = self.durable_tasks.lock().await;
4376        while let Some(result) = tasks.try_join_next() {
4377            if let Err(error) = result {
4378                tracing::warn!(error = %error, "daemon-owned task failed to join");
4379            }
4380        }
4381        tasks.spawn(async move {
4382            operation.await;
4383            operation_name
4384        });
4385    }
4386
4387    /// Enable the per-launch auth handshake. After this call, every
4388    /// new WS connection must call `session.auth` with `token` as
4389    /// the first frame; otherwise the connection is closed. Called
4390    /// by `car-server` at startup unless `--no-auth` is set
4391    /// (default flipped 2026-05); embedders supply their own token
4392    /// if they want the same posture. Returns `Err(token)` when
4393    /// auth was already installed.
4394    pub fn install_auth_token(&self, token: String) -> Result<(), String> {
4395        self.auth_token.set(token)
4396    }
4397
4398    /// Install the per-launch host token (Parslee-ai/car#254). A
4399    /// session that later presents this via `session.auth { host_token }`
4400    /// is granted the host-management role (`ClientSession::is_host`),
4401    /// which `authorize_run_access` requires for cross-agent run-trace
4402    /// reads. Set by `car-server` at startup (mints + writes the `0600`
4403    /// `host-token` file) unless `--no-auth` is set. Returns `Err(token)`
4404    /// when a host token was already installed.
4405    pub fn install_host_token(&self, token: String) -> Result<(), String> {
4406        self.host_token.set(token)
4407    }
4408
4409    pub fn install_mobile_runtime_url(&self, url: String) -> Result<(), String> {
4410        self.mobile_runtime_url.set(url)
4411    }
4412
4413    pub fn install_mobile_registration_url(&self, url: String) -> Result<(), String> {
4414        self.mobile_registration_url.set(url)
4415    }
4416
4417    pub fn install_parslee_session(
4418        &self,
4419        session: crate::parslee_auth::ParsleeSession,
4420    ) -> Result<(), crate::parslee_auth::ParsleeSession> {
4421        self.parslee_session.set(session)
4422    }
4423
4424    /// Install the runtime channel supervisor (Units 1/2/3). Called by
4425    /// `car-server` at boot right after `spawn_channel_pollers` builds it, so
4426    /// the host-gated `messaging.config.set` handler can reach it via
4427    /// `ServerState` to spawn a channel's watcher on enable. Idempotent on the
4428    /// first call; a second install is rejected (returns the supervisor back).
4429    pub fn install_channel_supervisor(
4430        &self,
4431        supervisor: Arc<crate::channel_supervisor::ChannelSupervisor>,
4432    ) -> Result<(), Arc<crate::channel_supervisor::ChannelSupervisor>> {
4433        self.channel_supervisor.set(supervisor)
4434    }
4435
4436    /// Install the bound MCP URL after car-server's listener is up.
4437    /// Idempotent on the first call; subsequent calls are accepted
4438    /// silently (matches the supervisor / a2a_dispatcher install
4439    /// idiom). Returns `Err(())` when an MCP URL was already
4440    /// installed — embedders should treat this as "another
4441    /// component beat us to it" and use whichever value is now set.
4442    pub fn install_mcp_url(&self, url: String) -> Result<(), String> {
4443        self.mcp_url.set(url)
4444    }
4445
4446    /// Lazy-initialize and return the remote MCP connector manager,
4447    /// backed by `~/.car/connectors.json` over the shared
4448    /// [`mcp_executor`](Self::mcp_executor). Construction is sync and
4449    /// touches no network — call [`ensure_connectors_loaded`](crate::session::ServerState::ensure_connectors_loaded) to dial
4450    /// persisted connectors.
4451    pub fn connectors(&self) -> Arc<car_connectors::ConnectorManager> {
4452        self.connectors
4453            .get_or_init(|| {
4454                let mgr = car_connectors::ConnectorManager::new(self.mcp_executor.clone())
4455                    .unwrap_or_else(|_| {
4456                        // HOME unresolved (rare): fall back to a CWD-relative
4457                        // manifest so connectors still work this session.
4458                        car_connectors::ConnectorManager::with_path(
4459                            self.mcp_executor.clone(),
4460                            std::path::PathBuf::from("connectors.json"),
4461                        )
4462                    });
4463                Arc::new(mgr)
4464            })
4465            .clone()
4466    }
4467
4468    /// Load persisted connectors and dial them, **once** per process.
4469    /// Idempotent: the first caller does the work; later callers return
4470    /// immediately. `car-server` calls this at boot so connector tools
4471    /// are live before clients connect; the `connectors.*` handlers call
4472    /// it too, so an embedder that skips the boot call still gets a
4473    /// lazy load on first use. Enabled tools discovered here are
4474    /// registered into any already-open sessions.
4475    pub async fn ensure_connectors_loaded(&self) {
4476        use std::sync::atomic::Ordering;
4477        if self.connectors_loaded.swap(true, Ordering::SeqCst) {
4478            return;
4479        }
4480        let mgr = self.connectors();
4481        match mgr.load_and_connect().await {
4482            Ok(entries) => self.register_connector_entries(&entries).await,
4483            Err(e) => tracing::warn!("connector load failed: {e}"),
4484        }
4485    }
4486
4487    /// Register connector [`car_engine::ToolEntry`]s into every currently-open
4488    /// session's runtime registry, so a tool enabled mid-session
4489    /// becomes visible without a reconnect. New sessions pick up the
4490    /// enabled set in [`create_session`](crate::session::ServerState::create_session).
4491    pub async fn register_connector_entries(&self, entries: &[car_engine::ToolEntry]) {
4492        if entries.is_empty() {
4493            return;
4494        }
4495        let sessions: Vec<Arc<ClientSession>> =
4496            self.sessions.lock().await.values().cloned().collect();
4497        for session in sessions {
4498            for entry in entries {
4499                session.runtime.register_tool_entry(entry.clone()).await;
4500            }
4501        }
4502    }
4503
4504    /// Unregister connector tools (by canonical name) from every open
4505    /// session's runtime, so a disabled or removed connector's tools
4506    /// stop being visible to the model and accepted by the validator.
4507    pub async fn unregister_connector_tools(&self, canonical_names: &[String]) {
4508        if canonical_names.is_empty() {
4509            return;
4510        }
4511        let sessions: Vec<Arc<ClientSession>> =
4512            self.sessions.lock().await.values().cloned().collect();
4513        for session in sessions {
4514            for name in canonical_names {
4515                session.runtime.unregister_tool(name).await;
4516            }
4517        }
4518    }
4519
4520    /// Whether any currently connected WS client holds the host-management
4521    /// role — authenticated via `session.auth { host_token }`
4522    /// (`ClientSession::is_host`), NOT the same thing as `host.subscribe`
4523    /// membership (see the doc comment on the auth check this mirrors,
4524    /// around `handle_session_auth`). In practice: is a CarHost / Command
4525    /// Deck app connected right now, so its browser drawer is a real
4526    /// visible surface?
4527    ///
4528    /// Scans the (small) live session set fresh on every call — no caching,
4529    /// so this is always current. The deciding signal for Task 7's
4530    /// headless/headed browser-launch default and its sign-in host-gone
4531    /// fallback (`assistant::browser_tools::HostConnectivity`).
4532    pub async fn any_host_connected(&self) -> bool {
4533        self.sessions
4534            .lock()
4535            .await
4536            .values()
4537            .any(|s| s.is_host.load(std::sync::atomic::Ordering::Acquire))
4538    }
4539
4540    /// Lazy-initialize and return the agent supervisor. The first
4541    /// call constructs a [`car_registry::supervisor::Supervisor`] backed by
4542    /// `~/.car/agents.json` + `~/.car/logs/`. Embedders that need a
4543    /// non-default location should call
4544    /// [`ServerState::install_supervisor`] before any handler runs.
4545    ///
4546    /// In observer mode (set via [`install_observer_manifest`](crate::session::ServerState::install_observer_manifest)),
4547    /// returns a clear error mentioning the manifest path the
4548    /// primary daemon owns. This prevents the second daemon from
4549    /// re-attempting `user_default()` (which would also fail with
4550    /// `AlreadyRunning`) on every WS call, and gives mutation
4551    /// handlers a stable refusal path. Read-only handlers
4552    /// (`agents.list`, `agents.health`) should call
4553    /// [`Self::observer_manifest_path`] first and fall back to
4554    /// [`car_registry::supervisor::Supervisor::list_from_manifest`] /
4555    /// `health_from_manifest` when set. Closes
4556    /// Parslee-ai/car-releases#44.
4557    /// The declarative-agent registry, lazy-initialized on first use.
4558    pub fn declagents(&self) -> Result<Arc<car_registry::declarative::DeclRegistry>, String> {
4559        if let Some(r) = self.declagents.get() {
4560            return Ok(r.clone());
4561        }
4562        let r = Arc::new(car_registry::declarative::DeclRegistry::user_default()?);
4563        let _ = self.declagents.set(r);
4564        Ok(self.declagents.get().expect("set or pre-existing").clone())
4565    }
4566
4567    pub fn routing(&self) -> Result<Arc<car_registry::routing::RoutingStore>, String> {
4568        if let Some(r) = self.routing.get() {
4569            return Ok(r.clone());
4570        }
4571        let r = Arc::new(car_registry::routing::RoutingStore::user_default()?);
4572        let _ = self.routing.set(r);
4573        Ok(self.routing.get().expect("set or pre-existing").clone())
4574    }
4575
4576    pub fn supervisor(&self) -> Result<Arc<car_registry::supervisor::Supervisor>, String> {
4577        if let Some(s) = self.supervisor.get() {
4578            return Ok(s.clone());
4579        }
4580        if let Some(p) = self.observer_manifest_path.get() {
4581            return Err(format!(
4582                "this car-server is observe-only — another car-server process \
4583                 holds the supervisor lock for {}. Mutations refuse here; route \
4584                 them to the primary daemon, or stop the other car-server first.",
4585                p.display()
4586            ));
4587        }
4588        let s = car_registry::supervisor::Supervisor::user_default()
4589            .map(Arc::new)
4590            .map_err(|e| e.to_string())?;
4591        // OnceLock::set returns the original arg back on collision —
4592        // a concurrent caller racing through user_default. Take
4593        // whichever wins.
4594        let _ = self.supervisor.set(s);
4595        Ok(self.supervisor.get().expect("set or pre-existing").clone())
4596    }
4597
4598    /// Replace the lazy default with a caller-supplied supervisor.
4599    /// Returns `Err(())` when a supervisor was already installed.
4600    /// Used by the standalone `car-server` binary to call
4601    /// `start_all()` on a known-good handle without paying the
4602    /// lazy-init lookup cost.
4603    pub fn install_supervisor(
4604        &self,
4605        supervisor: Arc<car_registry::supervisor::Supervisor>,
4606    ) -> Result<(), Arc<car_registry::supervisor::Supervisor>> {
4607        self.supervisor.set(supervisor)
4608    }
4609
4610    /// Non-acquiring read of the currently-installed supervisor.
4611    /// Unlike [`supervisor`](Self::supervisor), this does NOT lazy-
4612    /// init via `user_default()` — it returns `None` instead of
4613    /// constructing a fresh `Supervisor` and acquiring the
4614    /// `<manifest>.lock` as a side effect. Use this from read-only
4615    /// metadata paths (`host.subscribe` identity, status surfaces)
4616    /// where causing lock acquisition on observation would be a
4617    /// Heisenberg subscribe — the act of asking "do you own the
4618    /// lock?" must not be the act of taking it.
4619    pub fn supervisor_if_installed(&self) -> Option<Arc<car_registry::supervisor::Supervisor>> {
4620        self.supervisor.get().cloned()
4621    }
4622
4623    /// Mark this daemon as *observing* a manifest owned by another
4624    /// car-server process. After this call, `supervisor()` returns
4625    /// an "observe-only" error and read-only handlers
4626    /// (`agents.list`, `agents.health`) fall back to the static
4627    /// `Supervisor::list_from_manifest` / `health_from_manifest`
4628    /// paths. Idempotent — subsequent calls with the same path are
4629    /// no-ops; a different path returns `Err(())`. Closes
4630    /// Parslee-ai/car-releases#44.
4631    pub fn install_observer_manifest(&self, path: PathBuf) -> Result<(), PathBuf> {
4632        self.observer_manifest_path.set(path)
4633    }
4634
4635    /// Path of the manifest this daemon is observing but not
4636    /// supervising. `None` when this daemon owns the supervisor
4637    /// (the normal case) or when no manifest is configured at all
4638    /// (no `HOME`, embedder didn't install one).
4639    pub fn observer_manifest_path(&self) -> Option<&PathBuf> {
4640        self.observer_manifest_path.get()
4641    }
4642
4643    /// Lazy-initialize and return the in-core A2A dispatcher. The
4644    /// first call constructs an [`car_a2a::A2aDispatcher`] from
4645    /// either the embedder's overrides (set via
4646    /// [`ServerStateConfig::with_a2a_runtime`] / `with_a2a_store` /
4647    /// `with_a2a_card_source`) or sensible defaults: a fresh
4648    /// `Runtime` with `register_agent_basics` registered, an
4649    /// `InMemoryTaskStore`, and a card built from the runtime's
4650    /// tool schemas advertising `ws://127.0.0.1:9100/` as the
4651    /// public URL. Closes Parslee-ai/car-releases#28.
4652    pub async fn a2a_dispatcher(&self) -> Arc<car_a2a::A2aDispatcher> {
4653        if let Some(d) = self.a2a_dispatcher.get() {
4654            return d.clone();
4655        }
4656
4657        // Embedder overrides take precedence; fall back to defaults
4658        // for each slot independently (so an embedder that only
4659        // wants a custom card can leave the runtime + store at
4660        // defaults). `Mutex::take()` consumes the slot so the
4661        // defaults aren't reconstructed on a racing init that loses
4662        // the OnceLock::set call below.
4663        let runtime = self
4664            .a2a_runtime
4665            .lock()
4666            .expect("a2a_runtime mutex poisoned")
4667            .take();
4668        let runtime = match runtime {
4669            Some(r) => r,
4670            None => {
4671                let r = Arc::new(car_engine::Runtime::new());
4672                r.register_agent_basics().await;
4673                r
4674            }
4675        };
4676
4677        let store = self
4678            .a2a_store
4679            .lock()
4680            .expect("a2a_store mutex poisoned")
4681            .take()
4682            .unwrap_or_else(|| Arc::new(car_a2a::InMemoryTaskStore::new()));
4683
4684        let card_source = self
4685            .a2a_card_source
4686            .lock()
4687            .expect("a2a_card_source mutex poisoned")
4688            .take();
4689        let card_source = match card_source {
4690            Some(c) => c,
4691            None => {
4692                let card = car_a2a::build_default_agent_card(
4693                    &runtime,
4694                    car_a2a::AgentCardConfig::minimal(
4695                        "Common Agent Runtime",
4696                        "Embedded CAR daemon — A2A v1.0 reachable over WebSocket JSON-RPC.",
4697                        "ws://127.0.0.1:9100/",
4698                        car_a2a::AgentProvider {
4699                            organization: "Parslee".into(),
4700                            url: Some("https://github.com/Parslee-ai/car".into()),
4701                        },
4702                    ),
4703                )
4704                .await;
4705                Arc::new(move || card.clone()) as Arc<car_a2a::AgentCardSource>
4706            }
4707        };
4708
4709        let dispatcher = Arc::new(car_a2a::A2aDispatcher::new(runtime, store, card_source));
4710        // OnceLock::set returns Err on race — accept whichever
4711        // dispatcher won and clone-return that one.
4712        let _ = self.a2a_dispatcher.set(dispatcher);
4713        self.a2a_dispatcher
4714            .get()
4715            .expect("a2a_dispatcher set or pre-existing")
4716            .clone()
4717    }
4718
4719    /// Record the start of a run and return its [`RunMeta`]. The
4720    /// `run_id` is minted by the caller (the `runs.start` handler)
4721    /// so it can set `session.current_run_id` to the same value
4722    /// **before** responding (KTD3). Idempotent collision on an
4723    /// already-present `run_id` is treated as a fresh insert (uuids
4724    /// don't collide in practice; if one did, the latest start wins).
4725    ///
4726    /// U3: also writes the `RunStarted` line to the disk store + creates
4727    /// the run file, so the run is on disk from the first record (REPLAY
4728    /// survives a restart even before any turn lands). Disk failures are
4729    /// logged, never fatal — the in-memory registry is the live path.
4730    /// Fan one `runs.trace.event` out to every live subscriber of
4731    /// `run_id` (agent run tracing, U4). Called by the lifecycle methods
4732    /// **while they hold the `runs` lock** so append-and-notify is
4733    /// serialized with subscribe's snapshot-and-register (invariant #1).
4734    ///
4735    /// Each push is a non-blocking `try_send` onto the subscriber's
4736    /// bounded channel — the WS socket is written only by that
4737    /// subscriber's dedicated drain task, never here (invariant #2). A
4738    /// full channel drops the event (the slow-subscriber case); the
4739    /// client detects the cursor gap and re-subscribes (R8). Takes the
4740    /// already-acquired subscribers guard so the caller controls the lock
4741    /// scope and the `runs` → `run_subscribers` order.
4742    fn fanout_locked(
4743        subscribers: &HashMap<(String, String), crate::host::RunTraceSubscriber>,
4744        run_id: &str,
4745        agent_id: &str,
4746        record: car_proto::RunRecord,
4747        cursor: usize,
4748        status: car_proto::RunLiveStatus,
4749    ) {
4750        for ((sub_run, _client), sub) in subscribers.iter() {
4751            if sub_run != run_id {
4752                continue;
4753            }
4754            let event = car_proto::RunTraceEvent {
4755                run_id: run_id.to_string(),
4756                agent_id: agent_id.to_string(),
4757                record: record.clone(),
4758                cursor,
4759                status,
4760            };
4761            // Best-effort: a wedged subscriber's full channel drops the
4762            // event rather than stalling the producer. Logged at debug —
4763            // the client backfills via re-subscribe.
4764            if !sub.push(event) {
4765                tracing::debug!(
4766                    run_id,
4767                    "run-trace: dropped event for slow subscriber (channel full)"
4768                );
4769            }
4770        }
4771    }
4772
4773    /// Atomically reserve a run id/idempotency key across every live session.
4774    /// A per-run durability lock serializes the disk absence check with the
4775    /// final process-wide map insert. The global `runs` lock is held only for
4776    /// the two bounded map checks, never for filesystem enumeration/replay.
4777    pub async fn reserve_run(&self, meta: RunMeta) -> Result<RunReservation, String> {
4778        fn existing_reservation(
4779            existing: &RunMeta,
4780            requested: &RunMeta,
4781        ) -> Result<RunReservation, String> {
4782            if existing.client_id != requested.client_id {
4783                return Err(format!(
4784                    "{} run `{}` belongs to client_id `{}`",
4785                    car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX,
4786                    requested.run_id,
4787                    existing.client_id
4788                ));
4789            }
4790            if existing.agent_id != requested.agent_id {
4791                return Err(format!(
4792                    "{} run `{}` was retried under a different agent",
4793                    car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX,
4794                    requested.run_id
4795                ));
4796            }
4797            let changed_fields = [
4798                (existing.intent != requested.intent).then_some("intent"),
4799                (existing.outcome_description != requested.outcome_description)
4800                    .then_some("outcome_description"),
4801            ]
4802            .into_iter()
4803            .flatten()
4804            .collect::<Vec<_>>();
4805            if !changed_fields.is_empty() {
4806                return Err(format!(
4807                    "{} idempotency conflict for run `{}`: retry changed occurrence-defining {}",
4808                    car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX,
4809                    requested.run_id,
4810                    changed_fields.join(" and ")
4811                ));
4812            }
4813            Ok(RunReservation::Existing(existing.clone()))
4814        }
4815
4816        let durability_lock = self.run_durability_lock(&meta.run_id).await;
4817        let _durability_guard = durability_lock.lock().await;
4818        {
4819            let runs = self.runs.lock().await;
4820            if let Some(existing) = runs.get(&meta.run_id) {
4821                return existing_reservation(existing, &meta);
4822            }
4823        }
4824        let store = self.run_store.clone();
4825        let durable_run_id = meta.run_id.clone();
4826        let durable = tokio::task::spawn_blocking(move || store.run_started(&durable_run_id))
4827            .await
4828            .map_err(|error| format!("RunStarted reservation read task failed: {error}"))?
4829            .map_err(|error| format!("RunStarted reservation read failed: {error}"))?;
4830        if let Some(started) = durable {
4831            return Err(format!(
4832                "{} persisted run `{}` belongs to client_id `{}` and cannot be adopted",
4833                car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX,
4834                meta.run_id,
4835                started.client_id.as_deref().unwrap_or("historical/unknown")
4836            ));
4837        }
4838        let mut runs = self.runs.lock().await;
4839        if let Some(existing) = runs.get(&meta.run_id) {
4840            return existing_reservation(existing, &meta);
4841        }
4842        runs.insert(meta.run_id.clone(), meta);
4843        Ok(RunReservation::New)
4844    }
4845
4846    /// Release a newly reserved candidate when `runs.start` rejects before it
4847    /// binds or persists that candidate. The exact pristine metadata and the
4848    /// absent RunStore boundary are both re-checked under the per-run durability
4849    /// lock. This deliberately does not touch `current_run_id` or the journal
4850    /// binding, which may still belong to an older live run on this session.
4851    pub(crate) async fn release_unpersisted_run_reservation(
4852        &self,
4853        session: &ClientSession,
4854        expected: &RunMeta,
4855    ) -> Result<(), String> {
4856        if expected.client_id != session.client_id
4857            || expected.start_committed
4858            || expected.termination.is_some()
4859            || expected.ended_at.is_some()
4860            || !expected.turns.is_empty()
4861            || expected.pending_terminal.is_some()
4862        {
4863            return Err(format!(
4864                "run `{}` is not a pristine reservation owned by this session",
4865                expected.run_id
4866            ));
4867        }
4868        let still_expected = |existing: &RunMeta| {
4869            existing.run_id == expected.run_id
4870                && existing.client_id == expected.client_id
4871                && existing.agent_id == expected.agent_id
4872                && existing.intent == expected.intent
4873                && existing.outcome_description == expected.outcome_description
4874                && existing.started_at == expected.started_at
4875                && existing.durability_generation == expected.durability_generation
4876                && !existing.start_committed
4877                && existing.termination.is_none()
4878                && existing.ended_at.is_none()
4879                && existing.turns.is_empty()
4880                && existing.pending_terminal.is_none()
4881        };
4882        let durability_lock = self.run_durability_lock(&expected.run_id).await;
4883        let _durability_guard = durability_lock.lock().await;
4884        {
4885            let runs = self.runs.lock().await;
4886            let existing = runs
4887                .get(&expected.run_id)
4888                .ok_or_else(|| format!("reserved run `{}` is absent", expected.run_id))?;
4889            if !still_expected(existing) {
4890                return Err(format!(
4891                    "reserved run `{}` changed before pre-start release",
4892                    expected.run_id
4893                ));
4894            }
4895        }
4896        let store = self.run_store.clone();
4897        let durable_run_id = expected.run_id.clone();
4898        if tokio::task::spawn_blocking(move || store.run_started(&durable_run_id))
4899            .await
4900            .map_err(|error| format!("RunStarted release read task failed: {error}"))?
4901            .map_err(|error| format!("RunStarted release read failed: {error}"))?
4902            .is_some()
4903        {
4904            return Err(format!(
4905                "reserved run `{}` reached durable storage and cannot be released",
4906                expected.run_id
4907            ));
4908        }
4909        let mut runs = self.runs.lock().await;
4910        let existing = runs
4911            .get(&expected.run_id)
4912            .ok_or_else(|| format!("reserved run `{}` is absent", expected.run_id))?;
4913        if !still_expected(existing) {
4914            return Err(format!(
4915                "reserved run `{}` changed during pre-start release",
4916                expected.run_id
4917            ));
4918        }
4919        runs.remove(&expected.run_id);
4920        Ok(())
4921    }
4922
4923    pub async fn persist_run_start(&self, run_id: &str) -> Result<car_proto::RunStarted, String> {
4924        let durability_lock = self.run_durability_lock(run_id).await;
4925        let _durability_guard = durability_lock.lock().await;
4926        let started = {
4927            let runs = self.runs.lock().await;
4928            let meta = runs
4929                .get(run_id)
4930                .ok_or_else(|| format!("unknown reserved run_id `{run_id}`"))?;
4931            car_proto::RunStarted {
4932                run_id: meta.run_id.clone(),
4933                client_id: Some(meta.client_id.clone()),
4934                agent_id: meta.agent_id.clone(),
4935                intent: meta.intent.clone(),
4936                outcome_description: meta.outcome_description.clone(),
4937                started_at: meta.started_at,
4938            }
4939        };
4940        let store = self.run_store.clone();
4941        let durable_started = started.clone();
4942        tokio::task::spawn_blocking(move || store.write_started(&durable_started))
4943            .await
4944            .map_err(|error| format!("RunStarted durability task failed: {error}"))?
4945            .map_err(|error| format!("RunStarted durability failed: {error}"))?;
4946        Ok(started)
4947    }
4948
4949    pub async fn commit_run_start(&self, run_id: &str) -> Result<(), String> {
4950        let mut runs = self.runs.lock().await;
4951        let meta = runs
4952            .get_mut(run_id)
4953            .ok_or_else(|| format!("unknown reserved run_id `{run_id}`"))?;
4954        if meta.start_committed {
4955            return Ok(());
4956        }
4957        meta.start_committed = true;
4958        meta.durability_generation = meta.durability_generation.wrapping_add(1);
4959        let started = car_proto::RunStarted {
4960            run_id: meta.run_id.clone(),
4961            client_id: Some(meta.client_id.clone()),
4962            agent_id: meta.agent_id.clone(),
4963            intent: meta.intent.clone(),
4964            outcome_description: meta.outcome_description.clone(),
4965            started_at: meta.started_at,
4966        };
4967        let subs = self.run_subscribers.lock().await;
4968        Self::fanout_locked(
4969            &subs,
4970            &started.run_id,
4971            &started.agent_id,
4972            car_proto::RunRecord::Started(started.clone()),
4973            0,
4974            car_proto::RunLiveStatus::InProgress,
4975        );
4976        Ok(())
4977    }
4978
4979    /// Resolve a `runs.start` transaction whose acknowledgement failed before
4980    /// the owning socket disconnected. Returns `true` when no durable start
4981    /// boundary existed and the reservation was released. Returns `false`
4982    /// after reconciling an exact durable `RunStarted`, which the disconnect
4983    /// sweep must then close as one historical `Incomplete` occurrence.
4984    pub(crate) async fn reconcile_or_release_unacknowledged_start(
4985        &self,
4986        session: &ClientSession,
4987        run_id: &str,
4988    ) -> Result<bool, String> {
4989        let durability_lock = self.run_durability_lock(run_id).await;
4990        let _durability_guard = durability_lock.lock().await;
4991        let expected = {
4992            let runs = self.runs.lock().await;
4993            let meta = runs
4994                .get(run_id)
4995                .ok_or_else(|| format!("unknown reserved run_id `{run_id}`"))?;
4996            if meta.client_id != session.client_id {
4997                return Err(format!(
4998                    "{} run `{run_id}` belongs to client_id `{}`",
4999                    car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX,
5000                    meta.client_id
5001                ));
5002            }
5003            if meta.start_committed {
5004                return Ok(false);
5005            }
5006            if meta.is_terminal() || meta.pending_terminal.is_some() {
5007                return Err(format!(
5008                    "uncommitted run `{run_id}` unexpectedly has terminal state"
5009                ));
5010            }
5011            car_proto::RunStarted {
5012                run_id: meta.run_id.clone(),
5013                client_id: Some(meta.client_id.clone()),
5014                agent_id: meta.agent_id.clone(),
5015                intent: meta.intent.clone(),
5016                outcome_description: meta.outcome_description.clone(),
5017                started_at: meta.started_at,
5018            }
5019        };
5020
5021        let store = self.run_store.clone();
5022        let durable_run_id = run_id.to_string();
5023        let durable = tokio::task::spawn_blocking(move || store.run_started(&durable_run_id))
5024            .await
5025            .map_err(|error| format!("RunStarted recovery read task failed: {error}"))?
5026            .map_err(|error| format!("RunStarted recovery read failed: {error}"))?;
5027        match durable {
5028            Some(durable) => {
5029                if durable != expected {
5030                    return Err(format!(
5031                        "durable RunStarted identity for `{run_id}` does not match its live reservation"
5032                    ));
5033                }
5034                // Re-run both exact idempotent durability barriers. The first
5035                // retry turns a prior write-without-fsync into an acknowledged
5036                // RunStore boundary; the second either appends RunStarted or
5037                // fsyncs the exact journal row already pending in memory.
5038                let store = self.run_store.clone();
5039                let durable_expected = expected.clone();
5040                tokio::task::spawn_blocking(move || store.write_started(&durable_expected))
5041                    .await
5042                    .map_err(|error| {
5043                        format!("RunStarted recovery durability task failed: {error}")
5044                    })?
5045                    .map_err(|error| format!("RunStarted recovery durability failed: {error}"))?;
5046                session.append_run_started_event(&expected).await?;
5047                self.commit_run_start(run_id).await?;
5048                Ok(false)
5049            }
5050            None => {
5051                // `runs.start` orders RunStore before journal append, so an
5052                // absent RunStore boundary proves no lifecycle row was ever
5053                // admitted to the journal. Remove only the zero-byte file an
5054                // open-before-write failure may have left behind.
5055                let store = self.run_store.clone();
5056                let durable_expected = expected.clone();
5057                tokio::task::spawn_blocking(move || {
5058                    store.rollback_empty_run_start(&durable_expected)
5059                })
5060                .await
5061                .map_err(|error| format!("empty RunStarted rollback task failed: {error}"))?
5062                .map_err(|error| format!("empty RunStarted rollback failed: {error}"))?;
5063                {
5064                    let mut runs = self.runs.lock().await;
5065                    let meta = runs.get(run_id).ok_or_else(|| {
5066                        format!("reserved run `{run_id}` disappeared during rollback")
5067                    })?;
5068                    let still_expected = meta.client_id == session.client_id
5069                        && !meta.start_committed
5070                        && !meta.is_terminal()
5071                        && meta.pending_terminal.is_none()
5072                        && meta.agent_id == expected.agent_id
5073                        && meta.intent == expected.intent
5074                        && meta.outcome_description == expected.outcome_description
5075                        && meta.started_at == expected.started_at;
5076                    if !still_expected {
5077                        return Err(format!(
5078                            "reserved run `{run_id}` changed during empty-start rollback"
5079                        ));
5080                    }
5081                    runs.remove(run_id);
5082                }
5083                session.clear_run_journal_binding(run_id).await?;
5084                let mut current = session.current_run_id.lock().await;
5085                if current.as_deref() == Some(run_id) {
5086                    *current = None;
5087                }
5088                Ok(true)
5089            }
5090        }
5091    }
5092
5093    /// Compatibility/internal path for CAR-owned in-process callers that do
5094    /// not expose `runs.start`. The public RPC uses reserve/persist/journal/
5095    /// commit explicitly so its acknowledgement includes both durable stores.
5096    pub async fn start_run(&self, meta: RunMeta) -> Result<(), String> {
5097        let run_id = meta.run_id.clone();
5098        self.reserve_run(meta).await?;
5099        self.persist_run_start(&run_id).await?;
5100        self.commit_run_start(&run_id).await
5101    }
5102
5103    /// Make a run terminal with a harness-reported outcome
5104    /// (`runs.complete`). Returns `Err` if the `run_id` is unknown or
5105    /// already terminal — the handler maps that to a JSON-RPC error so
5106    /// a double-complete or stale id is visible, not silently swallowed.
5107    ///
5108    /// U3: also appends the terminal `RunEnded` line to the run's JSONL
5109    /// file so REPLAY (U5) reports the right status (Completed) after a
5110    /// restart. The disk write happens after the lock is released; disk
5111    /// failures are logged, never fatal.
5112    pub async fn prepare_run_completion(
5113        &self,
5114        run_id: &str,
5115        termination: car_proto::RunTermination,
5116    ) -> Result<car_proto::RunEnded, String> {
5117        self.prepare_run_completion_for_active_owner(run_id, termination, None)
5118            .await
5119    }
5120
5121    pub(crate) async fn prepare_run_completion_for_active_owner(
5122        &self,
5123        run_id: &str,
5124        termination: car_proto::RunTermination,
5125        expected_active_client_id: Option<&str>,
5126    ) -> Result<car_proto::RunEnded, String> {
5127        let completion_digest = run_completion_digest(&termination)?;
5128        if expected_active_client_id.is_some() {
5129            if let Some(gate) = &self.run_completion_fence_gate {
5130                gate.wait_if_armed().await;
5131            }
5132        }
5133        let durability_lock = self.run_durability_lock(run_id).await;
5134        let _durability_guard = durability_lock.lock().await;
5135        let ended = {
5136            let mut runs = self.runs.lock().await;
5137            let meta = runs
5138                .get_mut(run_id)
5139                .ok_or_else(|| format!("unknown run_id `{run_id}`"))?;
5140            if let Some(error) = &meta.trace_corruption {
5141                return Err(error.clone());
5142            }
5143            if !meta.start_committed {
5144                return Err(format!("run `{run_id}` start is not durably committed"));
5145            }
5146            if expected_active_client_id.is_some_and(|expected| meta.active_client_id != expected) {
5147                return Err(format!(
5148                    "run `{run_id}` active owner changed before terminalization"
5149                ));
5150            }
5151            if let Some(requested) = &meta.cancellation_pending {
5152                let matches_cancel = matches!(
5153                    &termination,
5154                    car_proto::RunTermination::Cancelled { cancellation }
5155                        if cancellation.run_id == requested.run_id
5156                            && cancellation.idempotency_key == requested.idempotency_key
5157                            && cancellation.reason_digest == requested.reason_digest
5158                            && cancellation.principal == requested.principal
5159                            && cancellation.action_id == requested.action_id
5160                            && cancellation.request_id == requested.request_id
5161                );
5162                if !matches_cancel {
5163                    return Err(format!(
5164                        "run `{run_id}` has a durable cancellation request and is quarantined"
5165                    ));
5166                }
5167            }
5168            if let Some(existing) = &meta.termination {
5169                if !same_run_termination(existing, &termination) {
5170                    return Err(format!("run `{run_id}` already has a different terminal"));
5171                }
5172                meta.resume_lease = None;
5173                car_proto::RunEnded {
5174                    run_id: run_id.to_string(),
5175                    client_id: Some(meta.client_id.clone()),
5176                    agent_id: meta.agent_id.clone(),
5177                    termination,
5178                    completion_digest: Some(completion_digest),
5179                    ended_at: meta
5180                        .ended_at
5181                        .ok_or_else(|| "terminal run is missing ended_at".to_string())?,
5182                }
5183            } else if let Some(pending) = &meta.pending_terminal {
5184                if !same_run_termination(&pending.termination, &termination)
5185                    || pending.completion_digest.as_deref() != Some(completion_digest.as_str())
5186                {
5187                    return Err(format!(
5188                        "run `{run_id}` has a different pending terminal transaction"
5189                    ));
5190                }
5191                meta.resume_lease = None;
5192                pending.clone()
5193            } else {
5194                let ended = car_proto::RunEnded {
5195                    run_id: run_id.to_string(),
5196                    client_id: Some(meta.client_id.clone()),
5197                    agent_id: meta.agent_id.clone(),
5198                    termination,
5199                    completion_digest: Some(completion_digest),
5200                    ended_at: chrono::Utc::now(),
5201                };
5202                meta.pending_terminal = Some(ended.clone());
5203                meta.resume_lease = None;
5204                meta.durability_generation = meta.durability_generation.wrapping_add(1);
5205                ended
5206            }
5207        };
5208        let store = self.run_store.clone();
5209        let durable_ended = ended.clone();
5210        let persisted = tokio::task::spawn_blocking(move || store.write_ended(&durable_ended))
5211            .await
5212            .map_err(|error| format!("RunEnded durability task failed: {error}"))?;
5213        if let Err(error) = persisted {
5214            if crate::run_store::is_trace_corruption_error(&error) {
5215                let mut runs = self.runs.lock().await;
5216                let message = self
5217                    .quarantine_run_trace_locked(&mut runs, run_id, error.to_string())
5218                    .await;
5219                return Err(message);
5220            }
5221            return Err(format!("RunEnded durability failed: {error}"));
5222        }
5223        Ok(ended)
5224    }
5225
5226    pub async fn commit_run_completion(&self, ended: &car_proto::RunEnded) -> Result<(), String> {
5227        {
5228            let mut runs = self.runs.lock().await;
5229            let meta = runs
5230                .get_mut(&ended.run_id)
5231                .ok_or_else(|| format!("unknown run_id `{}`", ended.run_id))?;
5232            if let Some(existing) = &meta.termination {
5233                if same_run_termination(existing, &ended.termination)
5234                    && meta.ended_at == Some(ended.ended_at)
5235                {
5236                    return Ok(());
5237                }
5238                return Err(format!(
5239                    "run `{}` already has a different terminal",
5240                    ended.run_id
5241                ));
5242            }
5243            if !meta
5244                .pending_terminal
5245                .as_ref()
5246                .is_some_and(|pending| same_run_ended(pending, ended))
5247            {
5248                return Err(format!(
5249                    "run `{}` terminal commit does not match its prepared transaction",
5250                    ended.run_id
5251                ));
5252            }
5253            meta.termination = Some(ended.termination.clone());
5254            meta.ended_at = Some(ended.ended_at);
5255            meta.pending_terminal = None;
5256            meta.resume_lease = None;
5257            meta.cancellation_pending = None;
5258            meta.cancellation_receipt = None;
5259            meta.durability_generation = meta.durability_generation.wrapping_add(1);
5260            let cursor = meta.turn_cursor();
5261            let status = meta.live_status();
5262            let agent_id = meta.agent_id.clone();
5263            let subs = self.run_subscribers.lock().await;
5264            Self::fanout_locked(
5265                &subs,
5266                &ended.run_id,
5267                &agent_id,
5268                car_proto::RunRecord::Ended(ended.clone()),
5269                cursor,
5270                status,
5271            );
5272        }
5273        // Heap hygiene: the run is now terminal and fully flushed to disk,
5274        // and the terminal `runs.trace.event` has already fanned out to any
5275        // live subscriber. Drop the in-memory per-turn buffer (each entry
5276        // holds a full prompt + CLI output, potentially MBs) so completed
5277        // runs don't pin the heaviest payloads for the daemon's lifetime.
5278        // The lightweight `RunMeta` header (agent_id, status, started_at,
5279        // termination) stays resident so `run_meta`/status lookups still
5280        // work. A late `runs.subscribe` to this terminal run re-sources its
5281        // snapshot from disk (see `subscribe_run`). Only terminal runs are
5282        // cleared; an in-progress run keeps its turns (the live snapshot
5283        // source). `Vec::new()` frees the buffer's capacity, not just its
5284        // length.
5285        self.clear_terminal_run_turns(&ended.run_id).await;
5286        Ok(())
5287    }
5288
5289    pub async fn complete_run(
5290        &self,
5291        run_id: &str,
5292        termination: car_proto::RunTermination,
5293    ) -> Result<car_proto::RunEnded, String> {
5294        let ended = self.prepare_run_completion(run_id, termination).await?;
5295        self.commit_run_completion(&ended).await?;
5296        Ok(ended)
5297    }
5298
5299    /// Commit the body-free request to RunStore and the authenticated opening
5300    /// client's journal under the same per-run durability lock used by
5301    /// terminal completion. Memory changes only after both receipts exist.
5302    pub async fn persist_run_cancellation_requested(
5303        &self,
5304        owner_session: &ClientSession,
5305        requested: &car_proto::RunCancellationRequested,
5306    ) -> Result<(), String> {
5307        let run_id = requested.run_id.as_str();
5308        let durability_lock = self.run_durability_lock(run_id).await;
5309        let _durability_guard = durability_lock.lock().await;
5310        let agent_id = {
5311            let runs = self.runs.lock().await;
5312            let meta = runs
5313                .get(run_id)
5314                .ok_or_else(|| format!("unknown run_id `{run_id}`"))?;
5315            if meta.is_terminal() || meta.pending_terminal.is_some() {
5316                return Err(format!("run `{run_id}` is already terminal"));
5317            }
5318            if meta.client_id != owner_session.client_id || !meta.start_committed {
5319                return Err("run cancellation client/run binding mismatch".into());
5320            }
5321            if let Some(existing) = &meta.cancellation_pending {
5322                if existing != requested {
5323                    return Err(format!(
5324                        "run `{run_id}` already has a different cancellation request"
5325                    ));
5326                }
5327                return Ok(());
5328            }
5329            meta.agent_id.clone()
5330        };
5331        let store = self.run_store.clone();
5332        let durable_agent = agent_id.clone();
5333        let durable_requested = requested.clone();
5334        tokio::task::spawn_blocking(move || {
5335            store.write_cancellation_requested(&durable_agent, &durable_requested)
5336        })
5337        .await
5338        .map_err(|error| format!("cancellation request durability task failed: {error}"))?
5339        .map_err(|error| format!("cancellation request durability failed: {error}"))?;
5340        owner_session
5341            .append_run_cancellation_requested_event(requested)
5342            .await?;
5343        let mut runs = self.runs.lock().await;
5344        let meta = runs
5345            .get_mut(run_id)
5346            .ok_or_else(|| format!("unknown run_id `{run_id}`"))?;
5347        meta.cancellation_pending = Some(requested.clone());
5348        meta.durability_generation = meta.durability_generation.wrapping_add(1);
5349        let cursor = meta.turn_cursor();
5350        let agent_id = meta.agent_id.clone();
5351        let subs = self.run_subscribers.lock().await;
5352        Self::fanout_locked(
5353            &subs,
5354            run_id,
5355            &agent_id,
5356            car_proto::RunRecord::CancellationRequested(requested.clone()),
5357            cursor,
5358            car_proto::RunLiveStatus::CancellationPending,
5359        );
5360        Ok(())
5361    }
5362
5363    pub async fn persist_run_cancellation_result(
5364        &self,
5365        owner_session: &ClientSession,
5366        result: &car_proto::RunCancelResponse,
5367    ) -> Result<(), String> {
5368        let run_id = result.run_id.as_str();
5369        let durability_lock = self.run_durability_lock(run_id).await;
5370        let _durability_guard = durability_lock.lock().await;
5371        let agent_id = {
5372            let runs = self.runs.lock().await;
5373            let meta = runs
5374                .get(run_id)
5375                .ok_or_else(|| format!("unknown run_id `{run_id}`"))?;
5376            if meta.is_terminal() {
5377                return Err(format!("run `{run_id}` is already terminal"));
5378            }
5379            let requested = meta
5380                .cancellation_pending
5381                .as_ref()
5382                .ok_or_else(|| format!("run `{run_id}` has no cancellation request"))?;
5383            if requested.idempotency_key != result.idempotency_key
5384                || requested.reason_digest != result.reason_digest
5385                || requested.principal != result.principal
5386            {
5387                return Err(format!("run `{run_id}` cancellation receipt mismatch"));
5388            }
5389            if let Some(existing) = &meta.cancellation_receipt {
5390                if existing != result {
5391                    return Err(format!(
5392                        "run `{run_id}` has a different cancellation receipt"
5393                    ));
5394                }
5395                return Ok(());
5396            }
5397            meta.agent_id.clone()
5398        };
5399        let store = self.run_store.clone();
5400        let durable_agent = agent_id.clone();
5401        let durable_result = result.clone();
5402        tokio::task::spawn_blocking(move || {
5403            store.write_cancellation_result(&durable_agent, &durable_result)
5404        })
5405        .await
5406        .map_err(|error| format!("cancellation result durability task failed: {error}"))?
5407        .map_err(|error| format!("cancellation result durability failed: {error}"))?;
5408        owner_session
5409            .append_run_cancellation_result_event(result)
5410            .await?;
5411        let mut runs = self.runs.lock().await;
5412        let meta = runs
5413            .get_mut(run_id)
5414            .ok_or_else(|| format!("unknown run_id `{run_id}`"))?;
5415        meta.cancellation_receipt = Some(result.clone());
5416        meta.durability_generation = meta.durability_generation.wrapping_add(1);
5417        let cursor = meta.turn_cursor();
5418        let agent_id = meta.agent_id.clone();
5419        let subs = self.run_subscribers.lock().await;
5420        Self::fanout_locked(
5421            &subs,
5422            run_id,
5423            &agent_id,
5424            car_proto::RunRecord::CancellationResult(result.clone()),
5425            cursor,
5426            car_proto::RunLiveStatus::CancellationPending,
5427        );
5428        Ok(())
5429    }
5430
5431    /// Finish an unconfirmed cancellation after the opening socket has gone
5432    /// away. The durable `RunStarted` client id is the journal authority: a
5433    /// reconnect may request recovery, but it may not redirect the receipt to
5434    /// its own journal. RunStore, that authenticated journal, live state, and
5435    /// subscribers advance as one retryable transaction before success is
5436    /// returned.
5437    pub async fn persist_recovered_run_cancellation_result(
5438        &self,
5439        agent_id: &str,
5440        result: &car_proto::RunCancelResponse,
5441    ) -> Result<(), String> {
5442        if result.status == car_proto::RunCancellationStatus::AlreadyTerminal
5443            && result.terminal_digest.is_some()
5444        {
5445            return self
5446                .persist_recovered_orphan_terminal_cancellation(agent_id, result)
5447                .await;
5448        }
5449        if result.status != car_proto::RunCancellationStatus::TerminationUnconfirmed
5450            || result.terminal_digest.is_some()
5451        {
5452            return Err(
5453                "recovered cancellation receipt must be termination_unconfirmed or a host orphan terminal"
5454                    .into(),
5455            );
5456        }
5457        let run_id = result.run_id.as_str();
5458        let durability_lock = self.run_durability_lock(run_id).await;
5459        let _durability_guard = durability_lock.lock().await;
5460        let store = self.run_store.clone();
5461        let durable_agent = agent_id.to_string();
5462        let durable_run = run_id.to_string();
5463        let records = tokio::task::spawn_blocking(move || {
5464            store.get_run_trace_for_checked(&durable_agent, &durable_run)
5465        })
5466        .await
5467        .map_err(|error| format!("recovered cancellation trace task failed: {error}"))?
5468        .map_err(|error| format!("recovered cancellation trace read failed: {error}"))?
5469        .ok_or_else(|| "recovered cancellation durable owner is unavailable".to_string())?;
5470        let started = records
5471            .iter()
5472            .find_map(|record| match record {
5473                car_proto::RunRecord::Started(started) => Some(started),
5474                _ => None,
5475            })
5476            .ok_or_else(|| "recovered cancellation is missing durable RunStarted".to_string())?;
5477        if started.agent_id != agent_id {
5478            return Err("recovered cancellation durable agent mismatch".into());
5479        }
5480        let client_id = started
5481            .client_id
5482            .as_deref()
5483            .filter(|client_id| {
5484                !client_id.is_empty()
5485                    && !client_id.contains('/')
5486                    && !client_id.contains('\\')
5487                    && *client_id != "."
5488                    && *client_id != ".."
5489            })
5490            .ok_or_else(|| {
5491                "recovered cancellation has no authenticated durable journal owner".to_string()
5492            })?
5493            .to_string();
5494        if records
5495            .iter()
5496            .any(|record| matches!(record, car_proto::RunRecord::Ended(_)))
5497        {
5498            return Err(format!("run `{run_id}` is already terminal"));
5499        }
5500        let requested = records
5501            .iter()
5502            .find_map(|record| match record {
5503                car_proto::RunRecord::CancellationRequested(requested) => Some(requested),
5504                _ => None,
5505            })
5506            .ok_or_else(|| format!("run `{run_id}` has no durable cancellation request"))?;
5507        if requested.receipt_version != result.receipt_version
5508            || requested.run_id != result.run_id
5509            || requested.idempotency_key != result.idempotency_key
5510            || requested.reason_digest != result.reason_digest
5511            || requested.principal != result.principal
5512            || requested.action_id != result.action_id
5513            || requested.request_id != result.request_id
5514        {
5515            return Err(format!(
5516                "run `{run_id}` recovered cancellation receipt mismatch"
5517            ));
5518        }
5519        if let Some(existing) = records.iter().find_map(|record| match record {
5520            car_proto::RunRecord::CancellationResult(existing) => Some(existing),
5521            _ => None,
5522        }) {
5523            if existing != result {
5524                return Err(format!(
5525                    "run `{run_id}` has a different durable cancellation receipt"
5526                ));
5527            }
5528        }
5529        {
5530            let runs = self.runs.lock().await;
5531            if let Some(meta) = runs.get(run_id) {
5532                if meta.agent_id != agent_id
5533                    || meta.client_id != client_id
5534                    || !meta.start_committed
5535                    || meta.is_terminal()
5536                    || meta.cancellation_pending.as_ref() != Some(requested)
5537                {
5538                    return Err(format!(
5539                        "run `{run_id}` live state does not authenticate recovered cancellation"
5540                    ));
5541                }
5542                if meta
5543                    .cancellation_receipt
5544                    .as_ref()
5545                    .is_some_and(|existing| existing != result)
5546                {
5547                    return Err(format!(
5548                        "run `{run_id}` has a different live cancellation receipt"
5549                    ));
5550                }
5551            }
5552        }
5553
5554        let store = self.run_store.clone();
5555        let durable_agent = agent_id.to_string();
5556        let durable_result = result.clone();
5557        tokio::task::spawn_blocking(move || {
5558            store.write_cancellation_result(&durable_agent, &durable_result)
5559        })
5560        .await
5561        .map_err(|error| format!("recovered cancellation durability task failed: {error}"))?
5562        .map_err(|error| format!("recovered cancellation durability failed: {error}"))?;
5563
5564        let journal_dir = self.journal_dir.clone();
5565        let journal_failures = self.journal_failures.clone();
5566        let journal_client = client_id.clone();
5567        let journal_result = result.clone();
5568        let journal_was_new = tokio::task::spawn_blocking(move || {
5569            append_recovered_cancellation_result_journal(
5570                journal_dir,
5571                journal_failures,
5572                journal_client,
5573                journal_result,
5574            )
5575        })
5576        .await
5577        .map_err(|error| format!("recovered cancellation journal task failed: {error}"))??;
5578
5579        let mut runs = self.runs.lock().await;
5580        let (cursor, fanout) = match runs.get_mut(run_id) {
5581            Some(meta) => {
5582                let fanout = meta.cancellation_receipt.is_none();
5583                meta.cancellation_receipt = Some(result.clone());
5584                if fanout {
5585                    meta.durability_generation = meta.durability_generation.wrapping_add(1);
5586                }
5587                (meta.turn_cursor(), fanout)
5588            }
5589            None => (
5590                records
5591                    .iter()
5592                    .filter(|record| matches!(record, car_proto::RunRecord::Turn(_)))
5593                    .count(),
5594                journal_was_new,
5595            ),
5596        };
5597        if fanout {
5598            let subscribers = self.run_subscribers.lock().await;
5599            Self::fanout_locked(
5600                &subscribers,
5601                run_id,
5602                agent_id,
5603                car_proto::RunRecord::CancellationResult(result.clone()),
5604                cursor,
5605                car_proto::RunLiveStatus::CancellationPending,
5606            );
5607        }
5608        Ok(())
5609    }
5610
5611    async fn persist_recovered_orphan_terminal_cancellation(
5612        &self,
5613        agent_id: &str,
5614        result: &car_proto::RunCancelResponse,
5615    ) -> Result<(), String> {
5616        if result.principal != "host" {
5617            return Err(
5618                "only the host may terminalize a run without live cancellation control".into(),
5619            );
5620        }
5621        let termination = car_proto::RunTermination::Incomplete;
5622        let completion_digest = run_completion_digest(&termination)?;
5623        if result.terminal_digest.as_deref() != Some(completion_digest.as_str()) {
5624            return Err("recovered orphan cancellation terminal digest mismatch".into());
5625        }
5626        let requested = car_proto::RunCancellationRequested {
5627            receipt_version: result.receipt_version,
5628            run_id: result.run_id.clone(),
5629            idempotency_key: result.idempotency_key.clone(),
5630            reason_digest: result.reason_digest.clone(),
5631            principal: result.principal.clone(),
5632            action_id: result.action_id.clone(),
5633            request_id: result.request_id.clone(),
5634        };
5635        let run_id = result.run_id.as_str();
5636        let durability_lock = self.run_durability_lock(run_id).await;
5637        let _durability_guard = durability_lock.lock().await;
5638
5639        let store = self.run_store.clone();
5640        let durable_agent = agent_id.to_string();
5641        let durable_run = run_id.to_string();
5642        let records = tokio::task::spawn_blocking(move || {
5643            store.get_run_trace_for_checked(&durable_agent, &durable_run)
5644        })
5645        .await
5646        .map_err(|error| format!("recovered cancellation trace task failed: {error}"))?
5647        .map_err(|error| format!("recovered cancellation trace read failed: {error}"))?
5648        .ok_or_else(|| "recovered cancellation durable owner is unavailable".to_string())?;
5649        let started = records
5650            .iter()
5651            .find_map(|record| match record {
5652                car_proto::RunRecord::Started(started) => Some(started.clone()),
5653                _ => None,
5654            })
5655            .ok_or_else(|| "recovered cancellation is missing durable RunStarted".to_string())?;
5656        if started.agent_id != agent_id {
5657            return Err("recovered cancellation durable agent mismatch".into());
5658        }
5659        let client_id = started
5660            .client_id
5661            .as_deref()
5662            .filter(|client_id| {
5663                !client_id.is_empty()
5664                    && !client_id.contains('/')
5665                    && !client_id.contains('\\')
5666                    && *client_id != "."
5667                    && *client_id != ".."
5668            })
5669            .ok_or_else(|| {
5670                "recovered cancellation has no authenticated durable journal owner".to_string()
5671            })?
5672            .to_string();
5673        if let Some(existing) = records.iter().find_map(|record| match record {
5674            car_proto::RunRecord::CancellationRequested(existing) => Some(existing),
5675            _ => None,
5676        }) {
5677            if existing != &requested {
5678                return Err(format!(
5679                    "run `{run_id}` has a different cancellation request"
5680                ));
5681            }
5682        }
5683        if records
5684            .iter()
5685            .any(|record| matches!(record, car_proto::RunRecord::CancellationResult(_)))
5686        {
5687            return Err(format!(
5688                "run `{run_id}` already has a nonterminal cancellation receipt"
5689            ));
5690        }
5691        if let Some(existing) = records.iter().find_map(|record| match record {
5692            car_proto::RunRecord::Ended(existing) => Some(existing),
5693            _ => None,
5694        }) {
5695            if !matches!(existing.termination, car_proto::RunTermination::Incomplete)
5696                || existing.completion_digest.as_deref() != Some(completion_digest.as_str())
5697            {
5698                return Err(format!("run `{run_id}` is already terminal"));
5699            }
5700            return Ok(());
5701        }
5702        {
5703            let runs = self.runs.lock().await;
5704            if let Some(meta) = runs.get(run_id) {
5705                if meta.agent_id != agent_id
5706                    || meta.client_id != client_id
5707                    || !meta.start_committed
5708                    || meta.is_terminal()
5709                    || meta.pending_terminal.is_some()
5710                    || meta
5711                        .cancellation_pending
5712                        .as_ref()
5713                        .is_some_and(|existing| existing != &requested)
5714                    || meta.cancellation_receipt.is_some()
5715                {
5716                    return Err(format!(
5717                        "run `{run_id}` live state does not authenticate recovered cancellation"
5718                    ));
5719                }
5720            }
5721        }
5722
5723        let ended = car_proto::RunEnded {
5724            run_id: run_id.to_string(),
5725            client_id: Some(client_id.clone()),
5726            agent_id: agent_id.to_string(),
5727            termination,
5728            completion_digest: Some(completion_digest),
5729            ended_at: chrono::Utc::now(),
5730        };
5731        let store = self.run_store.clone();
5732        let durable_agent = agent_id.to_string();
5733        let durable_requested = requested.clone();
5734        let durable_ended = ended.clone();
5735        tokio::task::spawn_blocking(move || {
5736            store.write_cancellation_requested(&durable_agent, &durable_requested)?;
5737            store.write_ended(&durable_ended)
5738        })
5739        .await
5740        .map_err(|error| format!("recovered terminal durability task failed: {error}"))?
5741        .map_err(|error| format!("recovered terminal durability failed: {error}"))?;
5742
5743        let journal_dir = self.journal_dir.clone();
5744        let journal_failures = self.journal_failures.clone();
5745        let journal_client = client_id;
5746        let journal_requested = requested.clone();
5747        let journal_ended = ended.clone();
5748        let journal_result = result.clone();
5749        tokio::task::spawn_blocking(move || {
5750            append_recovered_terminal_cancellation_journal(
5751                journal_dir,
5752                journal_failures,
5753                journal_client,
5754                journal_requested,
5755                journal_ended,
5756                journal_result,
5757            )
5758        })
5759        .await
5760        .map_err(|error| format!("recovered cancellation journal task failed: {error}"))??;
5761
5762        let mut runs = self.runs.lock().await;
5763        if let Some(meta) = runs.get_mut(run_id) {
5764            let cursor = meta.turn_cursor();
5765            let request_is_new = meta.cancellation_pending.is_none();
5766            meta.cancellation_pending = Some(requested.clone());
5767            meta.termination = Some(ended.termination.clone());
5768            meta.ended_at = Some(ended.ended_at);
5769            meta.pending_terminal = None;
5770            meta.resume_lease = None;
5771            meta.cancellation_pending = None;
5772            meta.cancellation_receipt = None;
5773            meta.durability_generation = meta.durability_generation.wrapping_add(1);
5774            let subscribers = self.run_subscribers.lock().await;
5775            if request_is_new {
5776                Self::fanout_locked(
5777                    &subscribers,
5778                    run_id,
5779                    agent_id,
5780                    car_proto::RunRecord::CancellationRequested(requested),
5781                    cursor,
5782                    car_proto::RunLiveStatus::CancellationPending,
5783                );
5784            }
5785            Self::fanout_locked(
5786                &subscribers,
5787                run_id,
5788                agent_id,
5789                car_proto::RunRecord::Ended(ended),
5790                cursor,
5791                car_proto::RunLiveStatus::Incomplete,
5792            );
5793        }
5794        drop(runs);
5795        self.clear_terminal_run_turns(run_id).await;
5796        Ok(())
5797    }
5798
5799    /// Mark a run `Incomplete` (R5) — used by disconnect cleanup when a
5800    /// harness drops without `runs.complete`. No-op if the run is
5801    /// already terminal (the common healthy-close case where
5802    /// `runs.complete` won the race). Returns `true` if it actually
5803    /// wrote the `Incomplete` marker.
5804    ///
5805    /// U3: on the transition to `Incomplete`, appends the terminal
5806    /// `RunEnded { Incomplete }` line to disk so an orphaned run reports
5807    /// `Incomplete` (not `InProgress`) on REPLAY after a restart. Disk
5808    /// failures are logged, never fatal.
5809    pub async fn prepare_run_incomplete(&self, run_id: &str) -> Option<car_proto::RunEnded> {
5810        self.prepare_run_incomplete_for_active_owner(run_id, None)
5811            .await
5812    }
5813
5814    async fn prepare_run_incomplete_for_active_owner(
5815        &self,
5816        run_id: &str,
5817        expected_active_client_id: Option<&str>,
5818    ) -> Option<car_proto::RunEnded> {
5819        // If a `runs.complete` transaction already prepared its exact outcome,
5820        // disconnect cleanup must finish that transaction, never overwrite it
5821        // with Incomplete.
5822        let termination = self
5823            .runs
5824            .lock()
5825            .await
5826            .get(run_id)
5827            .and_then(|meta| meta.pending_terminal.as_ref())
5828            .map(|ended| ended.termination.clone())
5829            .unwrap_or(car_proto::RunTermination::Incomplete);
5830        match self
5831            .prepare_run_completion_for_active_owner(run_id, termination, expected_active_client_id)
5832            .await
5833        {
5834            Ok(ended) => Some(ended),
5835            Err(error) => {
5836                tracing::error!(run_id, %error, "failed to prepare durable incomplete terminal");
5837                None
5838            }
5839        }
5840    }
5841
5842    pub async fn mark_run_incomplete(&self, run_id: &str) -> Option<car_proto::RunEnded> {
5843        let ended = self.prepare_run_incomplete(run_id).await?;
5844        if let Err(error) = self.commit_run_completion(&ended).await {
5845            tracing::error!(run_id, %error, "failed to commit incomplete terminal");
5846            return None;
5847        }
5848        Some(ended)
5849    }
5850
5851    /// Free the in-memory per-turn buffer of a TERMINAL run, keeping the
5852    /// lightweight `RunMeta` header resident (agent run tracing, heap
5853    /// hygiene follow-up). Called at the end of `complete_run` /
5854    /// `mark_run_incomplete`, AFTER the disk flush and terminal fanout, so
5855    /// the disk store remains the durable source of truth and any late
5856    /// `runs.subscribe` re-sources the snapshot from disk
5857    /// ([`Self::subscribe_run`]).
5858    ///
5859    /// Guards on `is_terminal()` so a same-id reuse that somehow re-opened
5860    /// the run (it shouldn't — ids are uuids) never has live turns dropped.
5861    /// `Vec::new()` releases the buffer's capacity, not just its length —
5862    /// the per-turn payloads (full prompt + CLI output) are the heavy part.
5863    async fn clear_terminal_run_turns(&self, run_id: &str) {
5864        let mut runs = self.runs.lock().await;
5865        if let Some(meta) = runs.get_mut(run_id) {
5866            if meta.is_terminal() {
5867                meta.turns = Vec::new();
5868            }
5869        }
5870    }
5871
5872    /// Non-acquiring read of a run's current metadata (clone). Used by
5873    /// tests and by the U2 recorder to learn a run's owning `agent_id`.
5874    ///
5875    /// NOTE: this clones the ENTIRE `RunMeta`, including its `turns` buffer
5876    /// (full prompts + CLI output — up to the per-run caps). Hot callers
5877    /// that need only the run's header facts must use [`Self::run_header`]
5878    /// instead; cloning the whole buffer per batch RPC is a ~512 MB
5879    /// worst-case copy at this PR's caps (ADV-2).
5880    pub async fn run_meta(&self, run_id: &str) -> Option<RunMeta> {
5881        self.runs.lock().await.get(run_id).cloned()
5882    }
5883
5884    /// Lightweight, non-cloning read of a run's header facts under the
5885    /// `runs` lock: `(agent_id, terminal, turn_count, trace_corruption)`.
5886    /// The owning agent id and optional error are the only heap allocations;
5887    /// This is what `handle_runs_record_turns` needs — owner for the write
5888    /// authz check, terminality for the distinguishable `run_terminal` drop,
5889    /// and the current turn count for the (fast-path) ceiling pre-check —
5890    /// without the `run_meta` deep clone of every recorded turn's payload.
5891    pub async fn run_header(&self, run_id: &str) -> Option<(String, bool, usize, Option<String>)> {
5892        self.runs.lock().await.get(run_id).map(|m| {
5893            (
5894                m.agent_id.clone(),
5895                m.is_terminal(),
5896                m.turns.len(),
5897                m.trace_corruption.clone(),
5898            )
5899        })
5900    }
5901
5902    /// Lightweight authenticated producer binding for run lifecycle handlers:
5903    /// `(client_id, terminal)`. Never falls back to disk because an on-disk row
5904    /// has no live WebSocket producer to authorize new writes.
5905    pub async fn run_lifecycle_binding(&self, run_id: &str) -> Option<(String, bool)> {
5906        self.runs.lock().await.get(run_id).map(|meta| {
5907            (
5908                meta.active_client_id.clone(),
5909                meta.is_terminal() || meta.trace_corruption.is_some(),
5910            )
5911        })
5912    }
5913
5914    /// `(client_id, terminal, start_committed, completion_pending)` for the
5915    /// authenticated producer path. Proposal writes require the exact owner,
5916    /// a committed start, and no prepared terminal transaction.
5917    pub async fn run_lifecycle_state(&self, run_id: &str) -> Option<(String, bool, bool, bool)> {
5918        self.run_lifecycle_state_with_corruption(run_id).await.map(
5919            |(client_id, terminal, committed, pending, _)| {
5920                (client_id, terminal, committed, pending)
5921            },
5922        )
5923    }
5924
5925    pub async fn run_lifecycle_state_with_corruption(
5926        &self,
5927        run_id: &str,
5928    ) -> Option<(String, bool, bool, bool, Option<String>)> {
5929        self.runs.lock().await.get(run_id).map(|meta| {
5930            (
5931                meta.active_client_id.clone(),
5932                meta.is_terminal(),
5933                meta.start_committed,
5934                meta.pending_terminal.is_some() || meta.cancellation_pending.is_some(),
5935                meta.trace_corruption.clone(),
5936            )
5937        })
5938    }
5939
5940    /// Atomically fence an orphaned live run to a replacement socket that has
5941    /// already authenticated as the run's agent. The immutable durable owner
5942    /// never changes: proposal and terminal artifacts continue to validate
5943    /// against the original `RunStarted` identity.
5944    pub(crate) async fn resume_run(
5945        &self,
5946        run_id: &str,
5947        agent_id: &str,
5948        replacement_client_id: &str,
5949    ) -> Result<RunResumeBinding, String> {
5950        // Serialize the ownership swap with every durable append/finalization
5951        // for this run. A task that already crossed this fence commits before
5952        // resume; one that has only captured the stale socket identity loses
5953        // after resume and cannot write any bytes.
5954        let durability_lock = self.run_durability_lock(run_id).await;
5955        let _durability_guard = durability_lock.lock().await;
5956        // Session creation/removal takes this same short fence around the map
5957        // mutation. Keep both guards through the run-owner compare-and-swap so
5958        // neither a late contender nor a disconnect can invalidate liveness
5959        // between validation and commit.
5960        let _liveness_guard = self.run_resume_liveness.lock().await;
5961        let live_clients = self
5962            .sessions
5963            .lock()
5964            .await
5965            .keys()
5966            .cloned()
5967            .collect::<std::collections::HashSet<_>>();
5968        if !live_clients.contains(replacement_client_id) {
5969            return Err("runs.resume replacement socket is no longer active".into());
5970        }
5971        let mut runs = self.runs.lock().await;
5972        let meta = runs
5973            .get_mut(run_id)
5974            .ok_or_else(|| "run not found or not authorized".to_string())?;
5975        if meta.agent_id != agent_id {
5976            return Err("run not found or not authorized".into());
5977        }
5978        if meta.is_terminal() {
5979            return Err(format!(
5980                "{} run `{run_id}` is already terminal",
5981                car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX
5982            ));
5983        }
5984        if !meta.start_committed {
5985            return Err(format!(
5986                "{} run `{run_id}` start is not durably committed",
5987                car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX
5988            ));
5989        }
5990        if meta.pending_terminal.is_some()
5991            || meta.cancellation_pending.is_some()
5992            || meta.trace_corruption.is_some()
5993        {
5994            return Err(format!(
5995                "{} run `{run_id}` is quarantined and cannot be resumed",
5996                car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX
5997            ));
5998        }
5999
6000        if meta.active_client_id == replacement_client_id {
6001            let resumed_from_client_id =
6002                meta.resume_predecessor_client_id.clone().ok_or_else(|| {
6003                    format!(
6004                        "{} run `{run_id}` is already active on this socket",
6005                        car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX
6006                    )
6007                })?;
6008            return Ok(RunResumeBinding {
6009                run_id: meta.run_id.clone(),
6010                agent_id: meta.agent_id.clone(),
6011                active_client_id: meta.active_client_id.clone(),
6012                resumed_from_client_id,
6013            });
6014        }
6015        if live_clients.contains(&meta.active_client_id) {
6016            return Err(format!(
6017                "{} run `{run_id}` still has a live owner",
6018                car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX
6019            ));
6020        }
6021        let lease_is_current = meta.resume_lease.as_ref().is_some_and(|lease| {
6022            lease.disconnected_client_id == meta.active_client_id
6023                && tokio::time::Instant::now() < lease.expires_at
6024        });
6025        if !lease_is_current {
6026            return Err(format!(
6027                "{} run `{run_id}` resume lease is unavailable or expired",
6028                car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX
6029            ));
6030        }
6031
6032        let resumed_from_client_id = std::mem::replace(
6033            &mut meta.active_client_id,
6034            replacement_client_id.to_string(),
6035        );
6036        meta.resume_predecessor_client_id = Some(resumed_from_client_id.clone());
6037        meta.resume_lease = None;
6038        meta.durability_generation = meta.durability_generation.wrapping_add(1);
6039        Ok(RunResumeBinding {
6040            run_id: meta.run_id.clone(),
6041            agent_id: meta.agent_id.clone(),
6042            active_client_id: meta.active_client_id.clone(),
6043            resumed_from_client_id,
6044        })
6045    }
6046
6047    /// Returns the immutable durable owner and the currently fenced live owner.
6048    pub(crate) async fn run_owner_binding(&self, run_id: &str) -> Option<(String, String)> {
6049        self.runs
6050            .lock()
6051            .await
6052            .get(run_id)
6053            .map(|meta| (meta.client_id.clone(), meta.active_client_id.clone()))
6054    }
6055
6056    /// Append per-turn trace records to a run's in-memory buffer (agent
6057    /// run tracing, U2). The recorder calls this after every
6058    /// `proposal.submit` on a session with a current run, passing the
6059    /// `RunRecord::Turn`s the recorder produced for that proposal.
6060    ///
6061    /// Returns a [`RecordRunTurnsOutcome`]:
6062    /// - `Appended { new_total }` — the batch landed; `new_total` is the
6063    ///   run's new total turn count (the caller computes its next
6064    ///   `start_index` from it).
6065    /// - `RefusedCeiling` — the batch would take the run past
6066    ///   [`RECORD_TURNS_RUN_CEILING`] and was refused WHOLE (the under-lock
6067    ///   runaway backstop, ADV-1). Nothing was appended.
6068    /// - `UnknownOrTerminal` — the `run_id` is unknown (the bracket was
6069    ///   never opened) or already terminal (a turn arriving after
6070    ///   `runs.complete` is dropped — the run is closed). Nothing was
6071    ///   appended.
6072    ///
6073    /// U3 flushes the in-memory buffer to disk; U4 broadcasts it. Both read
6074    /// via [`Self::run_turns`].
6075    ///
6076    /// U3: the same turn records are appended to the run's JSONL file so
6077    /// REPLAY (U5) sees them after a restart. Persistence is the commit point:
6078    /// only after the exact JSONL batch is durable do memory and subscribers
6079    /// advance. The operation runs in a daemon-owned task, so dropping a
6080    /// request waiter cannot strand a persisted batch between disk and memory.
6081    pub async fn record_run_turns(
6082        self: &Arc<Self>,
6083        run_id: &str,
6084        records: Vec<car_proto::RunRecord>,
6085    ) -> RecordRunTurnsOutcome {
6086        self.record_run_turns_for_owner(run_id, None, records).await
6087    }
6088
6089    /// Append client-narrated turns only while `active_client_id` still owns
6090    /// the run. The socket id is daemon-minted and captured by the handler
6091    /// before it detaches the work into a daemon-owned task; a later resume
6092    /// therefore fences delayed work from the predecessor without trusting a
6093    /// caller-supplied credential.
6094    #[doc(hidden)]
6095    pub async fn record_run_turns_for_active_owner(
6096        self: &Arc<Self>,
6097        run_id: &str,
6098        active_client_id: &str,
6099        records: Vec<car_proto::RunRecord>,
6100    ) -> RecordRunTurnsOutcome {
6101        self.record_run_turns_for_owner(run_id, Some(active_client_id.to_string()), records)
6102            .await
6103    }
6104
6105    async fn record_run_turns_for_owner(
6106        self: &Arc<Self>,
6107        run_id: &str,
6108        expected_active_client_id: Option<String>,
6109        records: Vec<car_proto::RunRecord>,
6110    ) -> RecordRunTurnsOutcome {
6111        let state = Arc::clone(self);
6112        let run_id = run_id.to_string();
6113        match tokio::spawn(async move {
6114            state
6115                .record_run_turns_owned(&run_id, expected_active_client_id.as_deref(), records)
6116                .await
6117        })
6118        .await
6119        {
6120            Ok(outcome) => outcome,
6121            Err(error) => RecordRunTurnsOutcome::PersistenceFailed(format!(
6122                "durable turn append task failed: {error}"
6123            )),
6124        }
6125    }
6126
6127    /// Make a proposal's authenticated final trace durable exactly once and
6128    /// reflect a newly-appended batch into the live registry/subscription
6129    /// stream before proposal finalization can acknowledge success.
6130    pub async fn ensure_proposal_run_turns(
6131        self: &Arc<Self>,
6132        pending: &crate::run_store::PendingProposalFinalization,
6133    ) -> Result<(), String> {
6134        let state = Arc::clone(self);
6135        let pending = pending.clone();
6136        tokio::spawn(async move { state.ensure_proposal_run_turns_owned(&pending).await })
6137            .await
6138            .map_err(|error| format!("proposal trace durability task failed: {error}"))?
6139    }
6140
6141    async fn ensure_proposal_run_turns_owned(
6142        &self,
6143        pending: &crate::run_store::PendingProposalFinalization,
6144    ) -> Result<(), String> {
6145        let durability_lock = self.run_durability_lock(&pending.run_id).await;
6146        let _durability_guard = durability_lock.lock().await;
6147        let (agent_id, status, generation) = {
6148            let runs = self.runs.lock().await;
6149            let meta = runs
6150                .get(&pending.run_id)
6151                .ok_or_else(|| format!("active run `{}` is absent", pending.run_id))?;
6152            if let Some(error) = &meta.trace_corruption {
6153                return Err(error.clone());
6154            }
6155            if !meta.accepts_proposals() {
6156                return Err(format!("active run `{}` is not writable", pending.run_id));
6157            }
6158            (
6159                meta.agent_id.clone(),
6160                meta.live_status(),
6161                meta.durability_generation,
6162            )
6163        };
6164        let store = self.run_store.clone();
6165        let durable = pending.clone();
6166        let durable_result =
6167            tokio::task::spawn_blocking(move || store.ensure_proposal_turns(&durable))
6168                .await
6169                .map_err(|error| format!("proposal trace durability task failed: {error}"))?;
6170        let mut runs = self.runs.lock().await;
6171        let ensured = match durable_result {
6172            Ok(ensured) => ensured,
6173            Err(error) if crate::run_store::is_trace_corruption_error(&error) => {
6174                let message = self
6175                    .quarantine_run_trace_locked(&mut runs, &pending.run_id, error.to_string())
6176                    .await;
6177                return Err(message);
6178            }
6179            Err(error) => return Err(format!("proposal trace durability failed: {error}")),
6180        };
6181        let meta = runs.get(&pending.run_id).ok_or_else(|| {
6182            format!(
6183                "active run `{}` disappeared after durability",
6184                pending.run_id
6185            )
6186        })?;
6187        if let Some(error) = &meta.trace_corruption {
6188            return Err(error.clone());
6189        }
6190        if meta.durability_generation != generation || !meta.accepts_proposals() {
6191            return Err(format!(
6192                "active run `{}` changed during proposal trace durability",
6193                pending.run_id
6194            ));
6195        }
6196        let already_live = !ensured.appended
6197            && meta.turns.iter().any(|record| match record {
6198                car_proto::RunRecord::Turn(turn) => {
6199                    turn.proposal_id.as_deref() == Some(pending.final_proposal_id.as_str())
6200                }
6201                _ => false,
6202            });
6203        if ensured.appended || !already_live {
6204            let meta = runs
6205                .get_mut(&pending.run_id)
6206                .expect("run registry remains locked during proposal trace commit");
6207            let base = ensured
6208                .records
6209                .first()
6210                .and_then(|record| match record {
6211                    car_proto::RunRecord::Turn(turn) => Some(turn.index),
6212                    _ => None,
6213                })
6214                .unwrap_or(meta.turns.len());
6215            meta.turns.extend(ensured.records.iter().cloned());
6216            meta.durability_generation = meta.durability_generation.wrapping_add(1);
6217            let subs = self.run_subscribers.lock().await;
6218            for (offset, record) in ensured.records.into_iter().enumerate() {
6219                Self::fanout_locked(
6220                    &subs,
6221                    &pending.run_id,
6222                    &agent_id,
6223                    record,
6224                    base + offset + 1,
6225                    status,
6226                );
6227            }
6228        }
6229        Ok(())
6230    }
6231
6232    async fn record_run_turns_owned(
6233        &self,
6234        run_id: &str,
6235        expected_active_client_id: Option<&str>,
6236        mut records: Vec<car_proto::RunRecord>,
6237    ) -> RecordRunTurnsOutcome {
6238        // Serialize durability only with this run. The global registry lock is
6239        // held for the admission snapshot and memory/fanout commit, never for
6240        // trace scanning, append, flush, or fsync.
6241        let durability_lock = self.run_durability_lock(run_id).await;
6242        let _durability_guard = durability_lock.lock().await;
6243        let (agent_id, base, new_total, status, generation) = {
6244            let mut runs = self.runs.lock().await;
6245            match runs.get_mut(run_id) {
6246                Some(meta) => {
6247                    if let Some(error) = &meta.trace_corruption {
6248                        return RecordRunTurnsOutcome::PersistenceFailed(error.clone());
6249                    }
6250                    if expected_active_client_id
6251                        .is_some_and(|expected| meta.active_client_id != expected)
6252                    {
6253                        return RecordRunTurnsOutcome::UnknownOrTerminal;
6254                    }
6255                    if !meta.accepts_proposals() {
6256                        return RecordRunTurnsOutcome::UnknownOrTerminal;
6257                    }
6258                    // ADV-1: hard ceiling, under the lock. Count only the
6259                    // `Turn` records — Started/Ended are not buffered here,
6260                    // and the ceiling is a turn-count cap. Refuse the whole
6261                    // batch if appending it would cross the ceiling.
6262                    let incoming_turns = records
6263                        .iter()
6264                        .filter(|r| matches!(r, car_proto::RunRecord::Turn(_)))
6265                        .count();
6266                    if meta.turns.len() + incoming_turns > RECORD_TURNS_RUN_CEILING {
6267                        return RecordRunTurnsOutcome::RefusedCeiling;
6268                    } else {
6269                        // Position of the first record about to be appended —
6270                        // its post-append cursor is `base + 1`.
6271                        let base = meta.turns.len();
6272                        // FIX 6: re-stamp each turn's `index` from the LIVE
6273                        // append position under the `runs` lock. The caller
6274                        // computes a provisional `start_index` from a pre-read
6275                        // `run_turn_count` OUTSIDE this lock — a TOCTOU: two
6276                        // concurrent proposals on one connection can read the
6277                        // same `start_index` and bake overlapping indices.
6278                        // Authoritatively assigning the index here, from
6279                        // `base + offset`, makes it contiguous and correct
6280                        // regardless of the racing reads upstream.
6281                        for (offset, record) in records.iter_mut().enumerate() {
6282                            if let car_proto::RunRecord::Turn(turn) = record {
6283                                turn.index = base + offset;
6284                            }
6285                        }
6286                        let agent_id = meta.agent_id.clone();
6287                        let status = meta.live_status();
6288                        (
6289                            agent_id,
6290                            base,
6291                            base + records.len(),
6292                            status,
6293                            meta.durability_generation,
6294                        )
6295                    }
6296                }
6297                None => return RecordRunTurnsOutcome::UnknownOrTerminal,
6298            }
6299        };
6300
6301        let store = self.run_store.clone();
6302        let durable_agent = agent_id.clone();
6303        let durable_run = run_id.to_string();
6304        let durable_records = records.clone();
6305        let append = tokio::task::spawn_blocking(move || {
6306            store.append_turns(&durable_agent, &durable_run, &durable_records)
6307        })
6308        .await;
6309        let mut runs = self.runs.lock().await;
6310        match append {
6311            Ok(Ok(())) => {}
6312            Ok(Err(error)) => {
6313                tracing::warn!(run_id, %error, "run-store: failed to persist turns");
6314                if crate::run_store::is_trace_corruption_error(&error) {
6315                    let message = self
6316                        .quarantine_run_trace_locked(&mut runs, run_id, error.to_string())
6317                        .await;
6318                    return RecordRunTurnsOutcome::PersistenceFailed(message);
6319                }
6320                return RecordRunTurnsOutcome::PersistenceFailed(error.to_string());
6321            }
6322            Err(error) => {
6323                tracing::warn!(run_id, %error, "run-store: durable turn task failed");
6324                return RecordRunTurnsOutcome::PersistenceFailed(error.to_string());
6325            }
6326        }
6327
6328        let meta = runs
6329            .get_mut(run_id)
6330            .expect("run registry entry survives a committed turn append");
6331        if let Some(error) = &meta.trace_corruption {
6332            return RecordRunTurnsOutcome::PersistenceFailed(error.clone());
6333        }
6334        if meta.durability_generation != generation || !meta.accepts_proposals() {
6335            return RecordRunTurnsOutcome::PersistenceFailed(format!(
6336                "run `{run_id}` changed during durable turn append"
6337            ));
6338        }
6339        meta.turns.extend(records.iter().cloned());
6340        meta.durability_generation = meta.durability_generation.wrapping_add(1);
6341        let subs = self.run_subscribers.lock().await;
6342        for (offset, record) in records.into_iter().enumerate() {
6343            Self::fanout_locked(&subs, run_id, &agent_id, record, base + offset + 1, status);
6344        }
6345        RecordRunTurnsOutcome::Appended { new_total }
6346    }
6347
6348    /// Number of turns already recorded for a run — the `start_index` the
6349    /// recorder passes so per-turn `index` stays monotonic across the
6350    /// run's proposals. `0` for an unknown run.
6351    pub async fn run_turn_count(&self, run_id: &str) -> usize {
6352        self.runs
6353            .lock()
6354            .await
6355            .get(run_id)
6356            .map(|m| m.turns.len())
6357            .unwrap_or(0)
6358    }
6359
6360    /// Clone of a run's ordered per-turn trace (agent run tracing, U2).
6361    /// This is the accessor U3 reads to flush turns to disk and U4 reads
6362    /// to broadcast them. Empty Vec for an unknown run or a run with no
6363    /// turns yet.
6364    pub async fn run_turns(&self, run_id: &str) -> Vec<car_proto::RunRecord> {
6365        self.runs
6366            .lock()
6367            .await
6368            .get(run_id)
6369            .map(|m| m.turns.clone())
6370            .unwrap_or_default()
6371    }
6372
6373    /// Atomically snapshot a run's turns AND register a live
6374    /// `runs.trace.event` subscriber for `(run_id, host_client_id)` —
6375    /// the load-bearing invariant #1 (agent run tracing, U4).
6376    ///
6377    /// Holds the `runs` lock across BOTH (a) reading the run's current
6378    /// turn buffer as the snapshot at cursor `C = turns.len()` and (b)
6379    /// inserting the subscriber into `run_subscribers`. Because the
6380    /// lifecycle path (`record_run_turns` / `complete_run` /
6381    /// `mark_run_incomplete`) appends-and-notifies under the SAME lock,
6382    /// the snapshot contains exactly the turns ≤ C and every turn appended
6383    /// after registration is delivered — no turn in the snapshot/register
6384    /// window is dropped (gap) or double-delivered (dup).
6385    ///
6386    /// Spawns the subscriber's drain task (bounded channel → WS) so the
6387    /// producer never writes the socket directly (invariant #2). Returns
6388    /// the snapshot response shape. `None` if the `run_id` is unknown (the
6389    /// handler maps that to a not-found error). A re-subscribe by the same
6390    /// `(run_id, host_client_id)` replaces the prior subscriber (its drain
6391    /// task ends when the old sender drops) and re-snapshots — the R8
6392    /// reconnect path: the fresh snapshot covers any turns emitted during
6393    /// the gap with no dup.
6394    ///
6395    /// Terminal-run disk fallback: a completed/incomplete run has its
6396    /// in-memory `turns` evicted for heap hygiene (see
6397    /// [`Self::clear_terminal_run_turns`]). When the run is terminal AND its
6398    /// in-memory buffer is empty, the snapshot is re-sourced from the disk
6399    /// store ([`crate::run_store::RunStore::get_run_trace`], the same source
6400    /// `runs.get_trace` reads), filtered to `RunRecord::Turn` so the shape
6401    /// matches the live in-memory snapshot. A terminal run takes no further
6402    /// appends, so the disk trail is final and there is no gap/dup concern —
6403    /// the snapshot/register atomicity the `runs` lock provides is only
6404    /// load-bearing for the in-progress path, which is unchanged.
6405    pub(crate) async fn subscribe_run_page(
6406        &self,
6407        run_id: &str,
6408        host_client_id: &str,
6409        channel: Arc<WsChannel>,
6410        cursor: usize,
6411        limit: usize,
6412    ) -> Result<Option<RunSubscribePageResult>, String> {
6413        // Build the subscriber (and its drain task) OUTSIDE the runs lock
6414        // so spawning never happens under the hot lock; registration
6415        // itself is the only thing serialized.
6416        let subscriber =
6417            crate::host::RunTraceSubscriber::spawn(host_client_id.to_string(), channel);
6418
6419        let (agent_id, mut expected_state) = {
6420            let runs = self.runs.lock().await;
6421            let Some(meta) = runs.get(run_id) else {
6422                return Ok(None);
6423            };
6424            if let Some(error) = &meta.trace_corruption {
6425                return Err(error.clone());
6426            }
6427            if meta.is_terminal() {
6428                return Ok(Some(RunSubscribePageResult::Durable {
6429                    agent_id: meta.agent_id.clone(),
6430                    status: meta.live_status(),
6431                }));
6432            }
6433            (
6434                meta.agent_id.clone(),
6435                (meta.turns.len(), meta.pending_terminal.is_some()),
6436            )
6437        };
6438
6439        for attempt in 0..=RUN_SUBSCRIBE_SUMMARY_STATE_RETRY_LIMIT {
6440            // Keep disk I/O off the global run-registry lock. A concurrent
6441            // mutation advances the captured state under that lock; retrying
6442            // the summary read then gives snapshot/registration one coherent
6443            // durable/live boundary.
6444            let store = self.run_store.clone();
6445            let durable_agent = agent_id.clone();
6446            let durable_run = run_id.to_string();
6447            let durable_corruption = tokio::task::spawn_blocking(move || {
6448                store.run_trace_corruption_for(&durable_agent, &durable_run)
6449            })
6450            .await
6451            .map_err(|error| format!("run trace corruption check task failed: {error}"))?
6452            .map_err(|error| format!("run trace corruption check failed: {error}"))?;
6453            let mut runs = self.runs.lock().await;
6454            let Some(meta) = runs.get(run_id) else {
6455                return Ok(None);
6456            };
6457            if let Some(error) = &meta.trace_corruption {
6458                return Err(error.clone());
6459            }
6460            if meta.agent_id != agent_id {
6461                return Err(format!("run `{run_id}` changed ownership during subscribe"));
6462            }
6463            if let Some(corruption) = durable_corruption {
6464                let message = self
6465                    .quarantine_run_trace_locked(
6466                        &mut runs,
6467                        run_id,
6468                        format!("malformed run trace record at line {}", corruption.line),
6469                    )
6470                    .await;
6471                return Err(message);
6472            }
6473            let status = meta.live_status();
6474            if meta.is_terminal() {
6475                return Ok(Some(RunSubscribePageResult::Durable { agent_id, status }));
6476            }
6477            let current_state = (meta.turns.len(), meta.pending_terminal.is_some());
6478            if current_state != expected_state {
6479                if attempt == RUN_SUBSCRIBE_SUMMARY_STATE_RETRY_LIMIT {
6480                    return Err(format!(
6481                        "run `{run_id}` changed repeatedly during durable subscribe validation; retry"
6482                    ));
6483                }
6484                expected_state = current_state;
6485                drop(runs);
6486                continue;
6487            }
6488            let live_cursor = meta.turns.len();
6489            if cursor > live_cursor {
6490                return Err(format!(
6491                    "runs.subscribe cursor {cursor} exceeds live_cursor {live_cursor}"
6492                ));
6493            }
6494            let end = cursor.saturating_add(limit).min(live_cursor);
6495            let turns = meta.turns[cursor..end].to_vec();
6496            let next_cursor = (end < live_cursor).then_some(end);
6497            let subscribed = next_cursor.is_none();
6498            if subscribed {
6499                let mut subs = self.run_subscribers.lock().await;
6500                subs.insert((run_id.to_string(), host_client_id.to_string()), subscriber);
6501            }
6502            drop(runs);
6503
6504            return Ok(Some(RunSubscribePageResult::Ready(
6505                car_proto::RunSubscribeResponse {
6506                    run_id: run_id.to_string(),
6507                    agent_id,
6508                    turns,
6509                    cursor,
6510                    limit,
6511                    next_cursor,
6512                    live_cursor,
6513                    subscribed,
6514                    status,
6515                },
6516            )));
6517        }
6518        unreachable!("bounded subscribe validation loop always returns")
6519    }
6520
6521    /// Remove a live run-trace subscriber for `(run_id, host_client_id)`
6522    /// (agent run tracing, U4). Returns `true` if a subscription existed.
6523    /// Dropping the [`crate::host::RunTraceSubscriber`] drops its channel
6524    /// sender, which ends the drain task.
6525    pub async fn unsubscribe_run(&self, run_id: &str, host_client_id: &str) -> bool {
6526        self.run_subscribers
6527            .lock()
6528            .await
6529            .remove(&(run_id.to_string(), host_client_id.to_string()))
6530            .is_some()
6531    }
6532
6533    /// Drop every live run-trace subscription owned by `host_client_id`
6534    /// (agent run tracing, U4 — R8 cleanup). Called from
6535    /// [`remove_session`](crate::session::ServerState::remove_session) on disconnect so a CarHost that drops doesn't
6536    /// leave dangling drain tasks. Reconnect-durability is client-side:
6537    /// the run stays subscribable while it lives, and the CarHost re-issues
6538    /// `runs.subscribe {run_id}` on its new connection — the server never
6539    /// synthesizes a failure on subscriber drop.
6540    pub async fn drop_run_subscribers_for_client(&self, host_client_id: &str) {
6541        self.run_subscribers
6542            .lock()
6543            .await
6544            .retain(|(_run, client), _| client != host_client_id);
6545    }
6546
6547    /// Disconnect cleanup for agent runs (R5). Called from
6548    /// [`remove_session`] with the `client_id` of the dropping
6549    /// connection. For every run this connection owns that is **not**
6550    /// yet terminal, wait a short grace window then re-check: if a
6551    /// concurrently-dispatched `runs.complete` made it terminal in the
6552    /// meantime, leave it; otherwise mark it `Incomplete`. The grace
6553    /// window is the fix for the serve-mode false-positive where a
6554    /// healthy `runs.complete` is still in flight (in a spawned
6555    /// dispatch task) when the close frame arrives (Risk: run identity
6556    /// races).
6557    async fn sweep_runs_for_disconnect(&self, session: &ClientSession) {
6558        let client_id = session.client_id.as_str();
6559        // Snapshot the still-open runs owned by this connection.
6560        let pending: Vec<(String, Option<tokio::time::Instant>)> = {
6561            let runs = self.runs.lock().await;
6562            runs.values()
6563                .filter(|m| m.active_client_id == client_id && !m.is_terminal())
6564                .map(|m| {
6565                    let resume_deadline = m.resume_lease.as_ref().and_then(|lease| {
6566                        (lease.disconnected_client_id == client_id).then_some(lease.expires_at)
6567                    });
6568                    (m.run_id.clone(), resume_deadline)
6569                })
6570                .collect()
6571        };
6572        if pending.is_empty() {
6573            return;
6574        }
6575        for (run_id, resume_deadline) in pending {
6576            // The lease deadline was installed atomically with removal from
6577            // `sessions`; sleeping to that absolute instant keeps teardown
6578            // work before this point from silently extending the contract.
6579            match resume_deadline {
6580                Some(deadline) => tokio::time::sleep_until(deadline).await,
6581                None => tokio::time::sleep(RUN_COMPLETE_GRACE).await,
6582            }
6583            match self.run_lifecycle_state(&run_id).await {
6584                None | Some((_, true, _, _)) => continue,
6585                Some((owner, false, _, _)) if owner != client_id => {
6586                    // An authenticated replacement claimed this run during the
6587                    // disconnect grace window. The old owner is fenced and
6588                    // must not be allowed to close the replacement's run.
6589                    continue;
6590                }
6591                Some((_, false, true, _)) => {}
6592                Some((_, false, false, _)) => {
6593                    match self
6594                        .reconcile_or_release_unacknowledged_start(session, &run_id)
6595                        .await
6596                    {
6597                        Ok(true) => continue,
6598                        Ok(false) => {}
6599                        Err(error) => {
6600                            tracing::error!(run_id, client_id, %error, "failed to reconcile or release an unacknowledged RunStarted transaction");
6601                            continue;
6602                        }
6603                    }
6604                }
6605            }
6606            match self.run_store.pending_proposal(&run_id) {
6607                Ok(Some(_)) => {
6608                    tracing::warn!(
6609                        run_id,
6610                        client_id,
6611                        "leaving run open because proposal finalization is pending durable replay"
6612                    );
6613                    continue;
6614                }
6615                Err(error) => {
6616                    tracing::error!(run_id, client_id, %error, "leaving run open because proposal finalization state is unreadable and outcome is unknown");
6617                    continue;
6618                }
6619                Ok(None) => {}
6620            }
6621            match self.run_store.execution_marker(&run_id) {
6622                Ok(Some(marker)) => {
6623                    tracing::warn!(
6624                        run_id,
6625                        client_id,
6626                        original_proposal_id = marker.original_proposal_id,
6627                        "leaving run open because proposal execution outcome is unknown"
6628                    );
6629                    continue;
6630                }
6631                Err(error) => {
6632                    tracing::error!(run_id, client_id, %error, "leaving run open because proposal execution marker is unreadable and outcome is unknown");
6633                    continue;
6634                }
6635                Ok(None) => {}
6636            }
6637            if let Some(ended) = self
6638                .prepare_run_incomplete_for_active_owner(&run_id, Some(client_id))
6639                .await
6640            {
6641                let durable_client_id = ended.client_id.as_deref().unwrap_or(client_id);
6642                let append_result = match session
6643                    .append_run_terminal_event_once(&ended, durable_client_id)
6644                    .await
6645                {
6646                    Err(error) if error.is_retry_safe() => {
6647                        tracing::warn!(
6648                            run_id,
6649                            client_id,
6650                            %error,
6651                            "disconnect terminal durability is unknown; retrying the exact journal row once"
6652                        );
6653                        session
6654                            .append_run_terminal_event_once(&ended, durable_client_id)
6655                            .await
6656                    }
6657                    result => result,
6658                };
6659                if let Err(error) = append_result {
6660                    tracing::error!(run_id, client_id, %error, "failed to bind disconnect terminal journal event");
6661                    continue;
6662                }
6663                if let Err(error) = self.commit_run_completion(&ended).await {
6664                    tracing::error!(run_id, client_id, %error, "failed to commit disconnect terminal");
6665                    continue;
6666                }
6667                if let Err(error) = session.clear_run_journal_binding(&run_id).await {
6668                    tracing::error!(run_id, client_id, %error, "failed to clear durable disconnect journal binding");
6669                    continue;
6670                }
6671                let mut current = session.current_run_id.lock().await;
6672                if current.as_deref() == Some(run_id.as_str()) {
6673                    *current = None;
6674                }
6675            }
6676        }
6677    }
6678
6679    /// Fails only when `<home>/.car/policies/` holds a malformed rule file —
6680    /// see [`apply_project_policies`] for why that is fatal rather than a
6681    /// warning.
6682    pub async fn create_session(
6683        &self,
6684        client_id: &str,
6685        channel: Arc<WsChannel>,
6686    ) -> Result<Arc<ClientSession>, String> {
6687        let journal_path = self.journal_dir.join(format!("{}.jsonl", client_id));
6688        let event_log = match self.journal_failures.clone() {
6689            Some(failures) => EventLog::with_journal_failure_injector(journal_path, failures),
6690            None => EventLog::with_journal(journal_path),
6691        };
6692
6693        let negotiated_capabilities =
6694            Arc::new(std::sync::RwLock::new(std::collections::BTreeSet::new()));
6695        let halted = Arc::new(AtomicBool::new(false));
6696        let ws_executor = Arc::new(WsToolExecutor::new(
6697            channel.clone(),
6698            negotiated_capabilities.clone(),
6699            halted.clone(),
6700        ));
6701
6702        // Compose: connector (remote MCP) tools route to the shared
6703        // process-wide executor; everything else falls back to this
6704        // session's WS tool executor. Both views share the same routing
6705        // table, so a connector enabled in any session is callable here.
6706        let executor: Arc<dyn ToolExecutor> =
6707            Arc::new(self.mcp_executor.share_with_fallback(ws_executor));
6708
6709        // Outbound human messaging. Built-in iMessage adapter over the same
6710        // `RealMessageSender` + `~/.car` config store the daemon's
6711        // `messaging.*` handlers use, plus a host fallback so a channel CAR has
6712        // no built-in transport for (Teams, per Parslee-ai/car#885) is
6713        // delivered by the host rather than compiled into the runtime.
6714        //
6715        // NO APPROVAL LOOP. `RealMessageSender` deliberately calls the *un-gated*
6716        // `car_ffi_common::integrations::messages_send` so that the approval
6717        // transport's own prompt does not raise an approval for itself. That
6718        // choice is safe to reuse here, and the recursion this new path might
6719        // look like it opens does not exist, because the two directions never
6720        // meet: `messaging.send` → registry → adapter → `messages_send` is a
6721        // straight line into the JXA bridge with no re-entry into the runtime,
6722        // and an approval raised *for* `messaging.send` is delivered by the
6723        // orchestrator/fanout path, which calls `MessageSender` directly and
6724        // never dispatches a runtime tool. Neither leg can call the other, so
6725        // there is no cycle to bound. Keep it that way: routing approval
6726        // delivery through `messaging.send` WOULD create one.
6727        let outbound = Arc::new(car_messaging::outbound::OutboundRegistry::new());
6728        outbound.register(Arc::new(
6729            car_messaging::outbound::ImessageOutboundAdapter::new(
6730                Arc::new(crate::messaging_orchestrator::RealMessageSender),
6731                crate::messaging_config::MessagingConfigStore::from_home(),
6732            ),
6733        ));
6734        outbound.set_fallback(Arc::new(crate::host_channel::HostChannelAdapter::new(
6735            executor.clone(),
6736        )));
6737
6738        // The trajectory store is the daemon-wide one, so traces from every
6739        // session accumulate in a single history. Without this the executor's
6740        // `persist_trajectory` short-circuits on its `self.trajectory_store
6741        // .as_ref()?` and nothing is ever recorded — which is why the
6742        // per-tool success rates had no data behind them.
6743        let runtime = Runtime::new()
6744            .with_event_log(event_log)
6745            .with_executor(executor)
6746            .with_trajectory_store(self.trajectory_store.clone())
6747            .with_message_sink(outbound);
6748
6749        // Declarative rules from `~/.car/policies/*.toml`. Same `~/.car`
6750        // convention the messaging config and the approval journal use — the
6751        // daemon serves whatever project a client happens to be working in, so
6752        // its policy set is the operator's, not a per-project one. (The
6753        // assistant, which IS rooted in a project, loads `<root>/.car`.)
6754        if let Some(car_dir) = car_home_dir() {
6755            apply_project_policies(&runtime, &car_dir).await?;
6756        }
6757
6758        // The session's permission-tier gate. Built here rather than inline in
6759        // the `ClientSession` literal below because the admission gate
6760        // registered a few lines down has to hold the SAME object the
6761        // `permission.*` RPCs mutate — see the field doc on
6762        // `ClientSession::permission_gate`.
6763        //
6764        // `SandboxEdit` is the default `docs/websocket-protocol.md` publishes,
6765        // and the code had drifted to `ReadOnly` while nothing enforced the
6766        // tier. `RiskClassifier::baseline` maps both `StateWrite` and
6767        // `ToolCall` to `SandboxEdit`, so keeping the drifted `ReadOnly`
6768        // default the moment enforcement lands would escalate essentially
6769        // every action every existing binding client submits. `full_access`
6770        // actions still escalate — mandatory HITL, exactly as documented.
6771        let permission_gate = Arc::new(tokio::sync::RwLock::new(car_policy::PermissionGate::new(
6772            car_policy::PermissionTier::SandboxEdit,
6773        )));
6774        // These handles are shared with the permission admission gate so it
6775        // observes the identity established later by `session.auth`; taking a
6776        // snapshot here would leave every newly-created session unbound.
6777        let authenticated = Arc::new(std::sync::atomic::AtomicBool::new(false));
6778        let agent_id = Arc::new(tokio::sync::Mutex::new(None));
6779        let callback_tool_schema_digests = Arc::new(tokio::sync::RwLock::new(HashMap::new()));
6780
6781        // If the embedder supplied a shared memgine, every session uses it.
6782        // Otherwise each session gets its own — matches pre-extraction behavior.
6783        // Create it before the admission gate so proposal.submit and the
6784        // permission advisory RPCs consult the exact same live skill ceilings.
6785        let memgine = match &self.shared_memgine {
6786            Some(eng) => eng.clone(),
6787            None => Arc::new(Mutex::new(car_memgine::MemgineEngine::new(None))),
6788        };
6789        // A non-shared session engine has no startup constructor hook, so
6790        // rehydrate the authoritative identity profile here too. Shared engines
6791        // were initialized in `try_with_config`; repeating the idempotent upsert
6792        // makes an externally edited identity.json visible to the next session.
6793        match self.identity_store.load() {
6794            Ok(identity) => {
6795                let mut engine = memgine.lock().await;
6796                mirror_identity_into_memgine(&mut engine, &identity);
6797            }
6798            Err(error) => tracing::warn!(
6799                error = %error,
6800                "identity profile could not be mirrored into a new session"
6801            ),
6802        }
6803
6804        // A terminal callback poisons this WebSocket session after the current
6805        // proposal's engine-level abort and rollback. Register the latch as an
6806        // ordinary admission gate so a later proposal gets the same complete
6807        // rejected-result shape as every other pre-execution refusal.
6808        runtime
6809            .register_admission_gate(Arc::new(SessionHaltAdmissionGate {
6810                halted: halted.clone(),
6811            }))
6812            .await;
6813
6814        // Statically verify submitted proposals before any action dispatches.
6815        //
6816        // This is the surface that matters for it: `proposal.submit` accepts
6817        // caller-authored, **multi-action** proposals, and per-action validation
6818        // only rejects an action once execution reaches it — so a bad tool name
6819        // in action 5 is caught after actions 1–4 have already had their side
6820        // effects. The gate refuses the whole proposal up front, so nothing
6821        // partial happens. (On a runtime that submits one action at a time, like
6822        // the assistant loop, it adds nothing the validator wasn't already
6823        // doing; see `car_engine::verify_gate` for what blocks and why.)
6824        runtime
6825            .register_admission_gate(Arc::new(car_engine::StaticVerificationGate::new(
6826                runtime.tools.clone(),
6827            )))
6828            .await;
6829
6830        // Out-of-process supervision (proposal item 5). Registered
6831        // unconditionally: with no supervisor subscribed the gate short-circuits
6832        // to Allow on an empty-map read, so leaving it on costs nothing and
6833        // means a supervisor can attach to a session that was already running
6834        // rather than only to sessions created after it connected.
6835        runtime
6836            .register_admission_gate(Arc::new(crate::supervision::SupervisionGate::new(
6837                self.supervision.clone(),
6838            )))
6839            .await;
6840
6841        // Enforce the session's granted permission tier before any action
6842        // dispatches (Parslee-ai/car#890).
6843        //
6844        // Everything this needs already existed — the per-session tier gate,
6845        // the durable daemon-wide approval ledger, `permission.approve` — but
6846        // nothing consulted any of it on the way to the executor, so an action
6847        // the daemon's own `permission.evaluate` called `needs_approval`
6848        // executed anyway and its state write persisted. This is the missing
6849        // call site: it asks the SAME gate the same question `permission.
6850        // evaluate` asks, against the SAME shared ledger, so the advisory
6851        // answer and the enforced one cannot diverge. An escalation blocks the
6852        // whole proposal until an operator approves the named fingerprint;
6853        // approving it makes the next submit come back `Allow` with no
6854        // escalation raised at all.
6855        runtime
6856            .register_admission_gate(Arc::new(
6857                crate::permission_gate::PermissionAdmissionGate::new(
6858                    permission_gate.clone(),
6859                    self.approval_ledger.clone(),
6860                )
6861                .with_authenticated_agent(authenticated.clone(), agent_id.clone())
6862                .with_callback_tools(
6863                    callback_tool_schema_digests.clone(),
6864                    runtime.registry.clone(),
6865                )
6866                .with_skill_memgine(memgine.clone())
6867                .with_event_log(runtime.log.clone()),
6868            ))
6869            .await;
6870
6871        let session = Arc::new(ClientSession {
6872            client_id: client_id.to_string(),
6873            runtime: Arc::new(runtime),
6874            channel,
6875            host: self.host.clone(),
6876            memgine,
6877            browser: car_ffi_common::browser::BrowserSessionSlot::new(),
6878            // When auth is disabled (no token installed), every
6879            // session is "authenticated" by default — preserves the
6880            // pre-#32 behaviour. When auth is enabled, the value is
6881            // ignored on creation; the dispatcher's gate checks
6882            // `ServerState::auth_token.is_some()` to decide whether
6883            // to enforce.
6884            authenticated,
6885            negotiated_protocol_version: std::sync::atomic::AtomicU32::new(0),
6886            negotiated_capabilities,
6887            inference_control: Arc::new(crate::inference_control::InferenceRegistry::default()),
6888            is_host: std::sync::atomic::AtomicBool::new(false),
6889            agent_id,
6890            agent_method_allowlist: Arc::new(std::sync::RwLock::new(None)),
6891            callback_tool_schema_digests,
6892            memory_namespace: tokio::sync::Mutex::new(None),
6893            bound_memgine: tokio::sync::Mutex::new(None),
6894            current_run_id: tokio::sync::Mutex::new(None),
6895            run_lifecycle_guard: Arc::new(tokio::sync::Mutex::new(())),
6896            permission_gate,
6897            halted,
6898            evolution_guard: crate::evolution::CycleGuard::default(),
6899            last_chat_turn: tokio::sync::Mutex::new(None),
6900            chat_inflight: std::sync::atomic::AtomicUsize::new(0),
6901            tenant: tokio::sync::Mutex::new(None),
6902            tool_stream_subscribed: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)),
6903        });
6904
6905        // Seed connector tools already enabled (in any session) into this
6906        // new session's runtime registry so the model sees them at once.
6907        // Only touches the manager if connectors have been initialized —
6908        // embedders that never use connectors pay nothing here.
6909        if self.connectors.get().is_some() {
6910            for entry in self.connectors().enabled_tool_entries().await {
6911                session.runtime.register_tool_entry(entry).await;
6912            }
6913        }
6914
6915        {
6916            let _liveness_guard = self.run_resume_liveness.lock().await;
6917            self.sessions
6918                .lock()
6919                .await
6920                .insert(client_id.to_string(), session.clone());
6921        }
6922
6923        Ok(session)
6924    }
6925
6926    /// Bind a session's runtime to the [`car_engine::McpSubstrate`] backed by an
6927    /// already-connected MCP connector, making that session **coherent**:
6928    /// its commodity built-ins (`read_file`/`write_file`/…/`run_command`)
6929    /// act on the *same* environment as its `mcp_{slug}_*` connector tools,
6930    /// instead of being split between the host/WS client and the remote MCP
6931    /// server (`docs/execution-substrate.md` §3, phase 3).
6932    ///
6933    /// This is **opt-in and explicit** — it is reached only via the
6934    /// `session.bindSubstrate` JSON-RPC method, never inferred from a
6935    /// connector merely being enabled. Sessions that never call it keep the
6936    /// exact historic composition (`share_with_fallback(ws_executor)` over a
6937    /// default `LocalSubstrate`), so GUI / A2A-over-WS / connectors-as-tools
6938    /// / coder behavior is unchanged.
6939    ///
6940    /// On bind we:
6941    /// 1. Wrap the live connector session (shared with the
6942    ///    `mcp_{slug}_*` routes — no second dial) as an [`car_engine::McpSubstrate`] and
6943    ///    `set_substrate` it on the session runtime.
6944    /// 2. `register_agent_basics()` so the model sees portable names
6945    ///    (`read_file`/`run_command`/…) that the engine routes to the bound
6946    ///    substrate.
6947    /// 3. Swap the session executor to a [`SubstrateShadowExecutor`] over the
6948    ///    same `share_with_fallback(ws_executor)` composition, so the bare
6949    ///    substrate-owned names fall through to the substrate while connector
6950    ///    routes and host tool callbacks keep working unchanged (option (a)).
6951    ///
6952    /// Returns the substrate's environment name (the connector slug) on
6953    /// success, or an error string if the connector is not connected.
6954    pub async fn bind_substrate_to_connector(
6955        &self,
6956        session: &Arc<ClientSession>,
6957        slug: &str,
6958    ) -> Result<String, String> {
6959        self.ensure_connectors_loaded().await;
6960
6961        // The connector's live MCP session is the same handle the
6962        // `mcp_{slug}_*` routes dispatch through — wrap it, don't re-dial.
6963        let mcp_session = self
6964            .mcp_executor
6965            .session(slug)
6966            .await
6967            .ok_or_else(|| format!("connector '{slug}' is not connected"))?;
6968
6969        let substrate: Arc<dyn car_engine::Substrate> =
6970            Arc::new(car_engine::McpSubstrate::new(mcp_session, slug.to_string()));
6971        let name = substrate.name().to_string();
6972
6973        // 1. Bind the substrate; 2. register substrate-backed built-ins.
6974        //
6975        // These three steps (set_substrate → register_agent_basics →
6976        // set_executor below) are not one critical section, so a tool `execute`
6977        // racing on this same session mid-swap could observe a half-applied
6978        // state. That's benign here: a WS connection processes its JSON-RPC
6979        // requests sequentially, and `bindSubstrate` is a client control call
6980        // issued before tool work — so no in-flight built-in overlaps the swap.
6981        session.runtime.set_substrate(substrate).await;
6982        session.runtime.register_agent_basics().await;
6983
6984        // 3. Shadow the substrate-owned names so they fall through to the
6985        //    bound substrate, keeping connector routes + WS host callbacks.
6986        let ws_executor = Arc::new(WsToolExecutor::new(
6987            session.channel.clone(),
6988            session.negotiated_capabilities.clone(),
6989            session.halted.clone(),
6990        ));
6991        let composed: Arc<dyn ToolExecutor> =
6992            Arc::new(self.mcp_executor.share_with_fallback(ws_executor));
6993        let shadowed: Arc<dyn ToolExecutor> = Arc::new(SubstrateShadowExecutor::new(composed));
6994        session.runtime.set_executor(shadowed).await;
6995
6996        Ok(name)
6997    }
6998
6999    /// Remove a per-client session from the registry on disconnect.
7000    /// Returns the removed session if present so callers can drop any
7001    /// remaining strong refs (e.g. drain pending tool callbacks). Fix
7002    /// for MULTI-4 / WS-3 — without this, `state.sessions` retains
7003    /// `Arc<ClientSession>` for every connection that ever existed.
7004    pub async fn remove_session(&self, client_id: &str) -> Option<Arc<ClientSession>> {
7005        // Cancel any live detached tool invocations on this session's
7006        // runtime (linus review D3): after disconnect the per-session
7007        // registry is unreachable, so a still-running tool would be
7008        // uncancellable until daemon restart. Same teardown discipline
7009        // as run subscribers / chat sessions below.
7010        if let Some(session) = self.sessions.lock().await.get(client_id).cloned() {
7011            let n = session.runtime.tool_handles.cancel_all().await;
7012            if n > 0 {
7013                tracing::info!(
7014                    client_id,
7015                    cancelled = n,
7016                    "cancelled detached tools at session teardown"
7017                );
7018            }
7019        }
7020
7021        let removed = {
7022            let _liveness_guard = self.run_resume_liveness.lock().await;
7023            let removed = self.sessions.lock().await.remove(client_id);
7024            let resume_negotiated = removed.as_ref().is_some_and(|session| {
7025                session
7026                    .negotiated_capabilities
7027                    .read()
7028                    .map(|caps| caps.contains(car_proto::RUNS_RESUME_CAPABILITY))
7029                    .unwrap_or(false)
7030            });
7031            if resume_negotiated {
7032                let expires_at = tokio::time::Instant::now() + self.run_resume_lease;
7033                let mut runs = self.runs.lock().await;
7034                for meta in runs
7035                    .values_mut()
7036                    .filter(|meta| meta.active_client_id == client_id && !meta.is_terminal())
7037                {
7038                    meta.resume_lease = Some(RunResumeLease {
7039                        disconnected_client_id: client_id.to_string(),
7040                        expires_at,
7041                    });
7042                }
7043            }
7044            removed
7045        };
7046        if let Some(session) = &removed {
7047            // #169: drop the agent_id → client_id binding so a
7048            // disconnected lifecycle agent can reconnect (or its
7049            // supervisor-respawned replacement can take the slot)
7050            // without colliding with the stale claim.
7051            let bound = session.agent_id.lock().await.clone();
7052            if let Some(id) = bound {
7053                let mut attached = self.attached_agents.lock().await;
7054                if attached.get(&id).map(String::as_str) == Some(client_id) {
7055                    attached.remove(&id);
7056                }
7057            }
7058            // Drop the peer-messaging guard with the binding. The guard holds a
7059            // rate budget and a dedupe window keyed by agent *name*, so a
7060            // supervisor-respawned replacement taking the same name would
7061            // otherwise inherit the dead process's history — arriving
7062            // pre-throttled, or silently deduped against messages it never saw.
7063            let bound_for_guards = session.agent_id.lock().await.clone();
7064            if let Some(id) = bound_for_guards {
7065                self.peer_guards.lock().await.remove(&id);
7066            }
7067            // Drop any in-flight `agents.chat` sessions bound to this
7068            // client — either side disconnecting orphans the stream,
7069            // and a respawned agent's stray `agent.chat.event`
7070            // notifications must not race into a stale routing entry.
7071            // See `docs/proposals/agent-chat-surface.md`.
7072            let bound_agent = session.agent_id.lock().await.clone();
7073            let mut chats = self.chat_sessions.lock().await;
7074            chats.retain(|_, s| {
7075                if s.host_client_id == client_id {
7076                    return false;
7077                }
7078                if let Some(agent_id) = &bound_agent {
7079                    if &s.agent_id == agent_id {
7080                        return false;
7081                    }
7082                }
7083                true
7084            });
7085            // Drop the chat lock before the run sweep, which awaits the
7086            // grace window and re-locks `runs` — keeping locks disjoint
7087            // avoids holding `chat_sessions` across the sleep.
7088            drop(chats);
7089            // Same for in-process A2A chat collectors owned by this client.
7090            // Dropping the entry drops the sole sender, so the collecting task's
7091            // `recv()` returns `None` and it falls back immediately instead of
7092            // waiting out its 180s cap. Done after `drop(chats)` so we never
7093            // hold `chat_sessions` while locking `chat_collectors` (the
7094            // interceptor locks them collectors→sessions; staying disjoint here
7095            // avoids inverting that order).
7096            self.chat_collectors
7097                .lock()
7098                .await
7099                .retain(|_, c| c.host_client_id != client_id);
7100            // Agent run tracing (U4): drop this connection's live
7101            // run-trace subscriptions so its drain tasks end and the
7102            // registry doesn't accumulate stale `(run_id, client)` entries
7103            // across reconnects. Reconnect-durability is client-side: the
7104            // run stays subscribable; CarHost re-subscribes by `run_id` on
7105            // its new connection (R8). This is exempt from the chat
7106            // drain-and-synthesize-error path — a dropped trace subscriber
7107            // never marks the underlying run failed.
7108            self.drop_run_subscribers_for_client(client_id).await;
7109            // Browser-drawer subscriptions are per-connection too. Dropping
7110            // them also starts the control grace period on any view this
7111            // connection was DRIVING, so an agent is never parked forever
7112            // behind a controller whose app just quit (see
7113            // `browser_view::CONTROL_GRACE`).
7114            self.browser_views
7115                .drop_subscriptions_for_client(client_id)
7116                .await;
7117            // Only a host-client removal can change host connectivity. This
7118            // used to fan a redundant reverse call to every producer for every
7119            // agent/CLI disconnect too. The session is already out of
7120            // `sessions`, so a real host removal reads the post-removal truth.
7121            if session.is_host.load(std::sync::atomic::Ordering::Acquire) {
7122                let host_connected = self.any_host_connected().await;
7123                self.browser_views
7124                    .broadcast_host_connected(host_connected)
7125                    .await;
7126            }
7127            // And if this connection was a supervised agent PUBLISHING a
7128            // browser, that browser went with the process. Its views stay
7129            // registered reporting an empty browser, so the drawer sees the
7130            // change and a restarted process can replace them in place (see
7131            // `browser_relay::RelayProducer::note_disconnected`).
7132            self.browser_views
7133                .note_producer_disconnected(client_id)
7134                .await;
7135            // Coder event subscriptions are per-connection too; the session
7136            // itself keeps running (reconnect + `coder.subscribe { from_seq }`
7137            // replays what was missed).
7138            // Board watchers (`coder.watch`) and discussion streams are
7139            // per-connection too, and are dropped on the same boundary.
7140            crate::coder::rpc::drop_subscriptions_for_client(self, client_id).await;
7141            crate::coder::discuss::drop_subscriptions_for_client(self, client_id).await;
7142            // Agent run tracing (R5): any run this connection owns that
7143            // has no terminal record yet is swept to `Incomplete` after
7144            // a short grace window — long enough for an in-flight
7145            // `runs.complete` to land first so a healthy close is never
7146            // mislabeled.
7147            let _run_guard = session.run_lifecycle_guard.lock().await;
7148            self.sweep_runs_for_disconnect(session).await;
7149        }
7150        removed
7151    }
7152}
7153
7154#[cfg(test)]
7155mod durable_task_tests {
7156    use super::*;
7157
7158    #[tokio::test]
7159    async fn dropping_response_waiter_does_not_cancel_operation_or_release_its_guard() {
7160        let temp = tempfile::tempdir().unwrap();
7161        let state = Arc::new(ServerState::with_config(ServerStateConfig::new(
7162            temp.path().to_path_buf(),
7163        )));
7164        let overlap_guard = Arc::new(tokio::sync::Mutex::new(()));
7165        let operation_guard = overlap_guard.clone();
7166        let (started_tx, started_rx) = tokio::sync::oneshot::channel();
7167        let (release_tx, release_rx) = tokio::sync::oneshot::channel();
7168
7169        let response = state
7170            .spawn_durable_operation("auth.test", async move {
7171                let _held = operation_guard.lock().await;
7172                let _ = started_tx.send(());
7173                let _ = release_rx.await;
7174                Ok::<_, String>(serde_json::json!({ "ok": true }))
7175            })
7176            .await;
7177        started_rx.await.unwrap();
7178        drop(response);
7179
7180        assert!(
7181            overlap_guard.try_lock().is_err(),
7182            "dropping only the connection waiter must not release the operation guard"
7183        );
7184        release_tx.send(()).unwrap();
7185        let joined = tokio::time::timeout(
7186            std::time::Duration::from_secs(1),
7187            state.durable_tasks.lock().await.join_next(),
7188        )
7189        .await
7190        .expect("daemon-owned operation should finish")
7191        .expect("task should exist")
7192        .expect("task should join");
7193        assert_eq!(joined, "auth.test");
7194        assert!(
7195            overlap_guard.try_lock().is_ok(),
7196            "the guard releases only after the daemon-owned operation finishes"
7197        );
7198    }
7199}
7200
7201#[cfg(test)]
7202mod tool_timeout_tests {
7203    use super::*;
7204
7205    #[test]
7206    fn honors_action_timeout_over_default() {
7207        // The action's budget bounds the wait (+grace so the executor's own
7208        // deadline reaps first) — no 60s ceiling, no env read.
7209        assert_eq!(
7210            tool_callback_timeout(Some(180_000)),
7211            std::time::Duration::from_millis(180_000 + TOOL_TIMEOUT_GRACE_MS)
7212        );
7213        // A budget far above the old 60s ceiling is honored (the #259 bug),
7214        // and the wait always exceeds the budget (so the executor wins).
7215        assert!(tool_callback_timeout(Some(600_000)) > std::time::Duration::from_secs(600));
7216        assert!(tool_callback_timeout(Some(180_000)) >= std::time::Duration::from_millis(180_000));
7217    }
7218
7219    #[test]
7220    fn default_is_not_the_old_60s() {
7221        // With no action budget and no env override, the fallback is the
7222        // raised default — not the hardcoded 60s that reaped real tools.
7223        // (Asserted only when CAR_TOOL_TIMEOUT is unset, which it is in CI.)
7224        if std::env::var_os("CAR_TOOL_TIMEOUT").is_none() {
7225            assert_eq!(
7226                tool_callback_timeout(None),
7227                std::time::Duration::from_millis(DEFAULT_TOOL_TIMEOUT_MS)
7228            );
7229            assert!(
7230                DEFAULT_TOOL_TIMEOUT_MS > 60_000,
7231                "default must exceed the old 60s"
7232            );
7233        }
7234    }
7235}
7236
7237#[cfg(test)]
7238mod observer_mode_tests {
7239    use super::*;
7240
7241    fn journal_dir() -> PathBuf {
7242        let target = std::env::var_os("CARGO_TARGET_DIR")
7243            .map(std::path::PathBuf::from)
7244            .unwrap_or_else(|| {
7245                std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
7246                    .join("..")
7247                    .join("..")
7248                    .join("target")
7249            });
7250        std::fs::create_dir_all(&target).ok();
7251        let target = std::fs::canonicalize(&target).unwrap_or(target);
7252        let tmp = tempfile::TempDir::new_in(&target).unwrap();
7253        let p = tmp.path().to_path_buf();
7254        std::mem::forget(tmp); // keep the dir alive for the test
7255        p
7256    }
7257
7258    #[test]
7259    fn supervisor_returns_observer_error_when_marker_set() {
7260        // Closes Parslee-ai/car-releases#44: the second car-server on
7261        // a host installs the observer marker after `with_paths`
7262        // returns AlreadyRunning. Subsequent `state.supervisor()`
7263        // calls must return a clear "observe-only" error mentioning
7264        // the manifest path — they must NOT retry user_default()
7265        // (which would re-acquire the lock and likely also fail).
7266        let state = ServerState::standalone(journal_dir());
7267        let fake_manifest = PathBuf::from("/tmp/fake-manifest-for-test.json");
7268        state
7269            .install_observer_manifest(fake_manifest.clone())
7270            .expect("install_observer_manifest succeeds on fresh state");
7271        assert_eq!(state.observer_manifest_path(), Some(&fake_manifest));
7272
7273        let err = state.supervisor().map(|_| ()).unwrap_err();
7274        assert!(
7275            err.contains("observe-only"),
7276            "error must mention observe-only mode: {err}"
7277        );
7278        assert!(
7279            err.contains("fake-manifest-for-test.json"),
7280            "error must surface the manifest path so operators know which daemon owns it: {err}"
7281        );
7282    }
7283
7284    #[test]
7285    fn install_observer_manifest_is_idempotent_per_path_collision() {
7286        let state = ServerState::standalone(journal_dir());
7287        let p = PathBuf::from("/tmp/manifest-a.json");
7288        let q = PathBuf::from("/tmp/manifest-b.json");
7289        state.install_observer_manifest(p.clone()).unwrap();
7290        // OnceLock::set returns the value back on collision.
7291        let err = state.install_observer_manifest(q.clone()).unwrap_err();
7292        assert_eq!(err, q);
7293        assert_eq!(state.observer_manifest_path(), Some(&p));
7294    }
7295
7296    #[test]
7297    fn supervisor_if_installed_does_not_lazy_init() {
7298        // The Heisenberg-subscribe guard: `host.subscribe`'s
7299        // identity path must use the non-acquiring read so a
7300        // purely observational client can't cause the daemon to
7301        // claim `<manifest>.lock` as a side effect of asking
7302        // about it. Fresh state has no supervisor installed.
7303        let state = ServerState::standalone(journal_dir());
7304        assert!(state.supervisor_if_installed().is_none());
7305        // observer_manifest_path should remain unset too — no
7306        // implicit init.
7307        assert!(state.observer_manifest_path().is_none());
7308    }
7309}
7310
7311#[cfg(test)]
7312mod substrate_binding_tests {
7313    use super::*;
7314    use car_engine::{McpSession, McpToolInfo};
7315    use car_ir::ActionProposal;
7316    use serde_json::json;
7317    use std::sync::Mutex as StdMutex;
7318
7319    fn journal_dir() -> PathBuf {
7320        let target = std::env::var_os("CARGO_TARGET_DIR")
7321            .map(std::path::PathBuf::from)
7322            .unwrap_or_else(|| {
7323                std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
7324                    .join("..")
7325                    .join("..")
7326                    .join("target")
7327            });
7328        std::fs::create_dir_all(&target).ok();
7329        let target = std::fs::canonicalize(&target).unwrap_or(target);
7330        let tmp = tempfile::TempDir::new_in(&target).unwrap();
7331        let p = tmp.path().to_path_buf();
7332        std::mem::forget(tmp);
7333        p
7334    }
7335
7336    /// A minimal in-memory MCP session standing in for a connected `vm`
7337    /// connector. `read_text`/`write_text`/`run_command` behave like the
7338    /// `vm` bridge so we can assert a substrate-bound session's built-ins
7339    /// land here rather than on the WS client (which has no responder in a
7340    /// test and would hang/fail).
7341    struct FakeVmSession {
7342        name: String,
7343        files: Arc<StdMutex<std::collections::HashMap<String, String>>>,
7344    }
7345
7346    #[async_trait::async_trait]
7347    impl McpSession for FakeVmSession {
7348        async fn list_tools(&mut self) -> Result<Vec<McpToolInfo>, String> {
7349            Ok(vec![])
7350        }
7351        async fn call_tool(
7352            &mut self,
7353            name: &str,
7354            args: serde_json::Value,
7355        ) -> Result<serde_json::Value, String> {
7356            match name {
7357                "write_text" => {
7358                    let p = args["path"].as_str().unwrap().to_string();
7359                    let c = args["content"].as_str().unwrap().to_string();
7360                    self.files.lock().unwrap().insert(p, c);
7361                    Ok(json!("ok"))
7362                }
7363                "read_text" => {
7364                    let p = args["path"].as_str().unwrap();
7365                    let c = self
7366                        .files
7367                        .lock()
7368                        .unwrap()
7369                        .get(p)
7370                        .cloned()
7371                        .ok_or_else(|| "not found".to_string())?;
7372                    Ok(json!(c))
7373                }
7374                "run_command" => {
7375                    let command = args["command"].as_str().unwrap_or_default();
7376                    if let Some(path) = command
7377                        .split_once("[ -e '")
7378                        .map(|(_, path)| path)
7379                        .and_then(|path| path.split_once("' ]").map(|(path, _)| path))
7380                    {
7381                        let exists = self.files.lock().unwrap().contains_key(path);
7382                        return Ok(json!({
7383                            "stdout": "",
7384                            "stderr": "",
7385                            "exit_code": if exists { 0 } else { 1 }
7386                        }));
7387                    }
7388                    Ok(json!({
7389                        "stdout": "from-vm",
7390                        "stderr": "",
7391                        "exit_code": 0
7392                    }))
7393                }
7394                other => Err(format!("unknown tool {other}")),
7395            }
7396        }
7397        fn name(&self) -> &str {
7398            &self.name
7399        }
7400    }
7401
7402    fn read_file_proposal(path: &str) -> ActionProposal {
7403        serde_json::from_value(json!({
7404            "source": "test",
7405            "actions": [{
7406                "id": "r0",
7407                "type": "tool_call",
7408                "tool": "read_file",
7409                "parameters": { "path": path },
7410                "dependencies": [],
7411            }],
7412        }))
7413        .expect("proposal deserializes")
7414    }
7415
7416    /// The outbound message sink is attached to every session runtime, so
7417    /// `messaging.send` is a real, executable tool rather than a schema the
7418    /// runtime would refuse. `with_message_sink` registers the sink and the
7419    /// schema together, so seeing the schema proves the sink is there.
7420    #[tokio::test]
7421    async fn session_runtime_has_the_messaging_send_tool() {
7422        let state = Arc::new(ServerState::standalone(journal_dir()));
7423        let session = state
7424            .create_session("c-messaging", Arc::new(WsChannel::test_stub()))
7425            .await
7426            .unwrap();
7427
7428        assert!(
7429            session
7430                .runtime
7431                .tools
7432                .read()
7433                .await
7434                .contains_key("messaging.send"),
7435            "every session must be able to execute messaging.send"
7436        );
7437    }
7438
7439    /// Default (no bind): the session runtime keeps a `LocalSubstrate`.
7440    /// This is the backward-compat invariant — every existing consumer
7441    /// (GUI, A2A-over-WS, connectors-as-tools, coder) sees no change.
7442    #[tokio::test]
7443    async fn default_session_substrate_is_local() {
7444        let state = Arc::new(ServerState::standalone(journal_dir()));
7445        let channel = Arc::new(WsChannel::test_stub());
7446        let session = state.create_session("c-default", channel).await.unwrap();
7447
7448        let sub = session.runtime.substrate().await;
7449        assert_eq!(
7450            sub.name(),
7451            "local",
7452            "an un-bound session must keep the default LocalSubstrate"
7453        );
7454        assert!(sub.is_local(), "default substrate must be the host");
7455    }
7456
7457    /// Binding fails cleanly when the named connector is not connected —
7458    /// no panic, a descriptive error, and the substrate stays Local.
7459    #[tokio::test]
7460    async fn bind_unknown_connector_errors_and_keeps_local() {
7461        let state = Arc::new(ServerState::standalone(journal_dir()));
7462        let channel = Arc::new(WsChannel::test_stub());
7463        let session = state.create_session("c-missing", channel).await.unwrap();
7464
7465        let err = state
7466            .bind_substrate_to_connector(&session, "nope")
7467            .await
7468            .unwrap_err();
7469        assert!(err.contains("not connected"), "got: {err}");
7470        assert_eq!(session.runtime.substrate().await.name(), "local");
7471    }
7472
7473    /// A connector-driven session that binds its substrate routes the bare
7474    /// commodity built-ins (`read_file`/`write_file`) to that connector's
7475    /// environment via the engine fall-through, NOT to the WS client. We
7476    /// prove it end-to-end: write+read through the runtime land on the fake
7477    /// VM session's file map.
7478    #[tokio::test]
7479    async fn bound_session_routes_builtins_to_substrate() {
7480        let state = Arc::new(ServerState::standalone(journal_dir()));
7481        let channel = Arc::new(WsChannel::test_stub());
7482        let session = state.create_session("c-vm", channel).await.unwrap();
7483
7484        // Register a connected "vm" connector session + its routes the way
7485        // the connector manager does on dial.
7486        let files = Arc::new(StdMutex::new(std::collections::HashMap::new()));
7487        let fake: Arc<Mutex<dyn McpSession>> = Arc::new(Mutex::new(FakeVmSession {
7488            name: "vm".into(),
7489            files: files.clone(),
7490        }));
7491        state.mcp_executor.add_session("vm", fake).await;
7492
7493        // Bind the session to the vm connector's substrate.
7494        let bound = state
7495            .bind_substrate_to_connector(&session, "vm")
7496            .await
7497            .expect("bind succeeds for a connected connector");
7498        assert_eq!(bound, "vm");
7499        assert_eq!(session.runtime.substrate().await.name(), "vm");
7500
7501        // write_file through the runtime → must hit the fake VM session,
7502        // not the WS client (which would hang/fail with no responder).
7503        let write = serde_json::from_value::<ActionProposal>(json!({
7504            "source": "test",
7505            "actions": [{
7506                "id": "w0",
7507                "type": "tool_call",
7508                "tool": "write_file",
7509                "parameters": { "path": "/vm/a.txt", "content": "vm-bytes" },
7510                "dependencies": [],
7511            }],
7512        }))
7513        .unwrap();
7514        let wres = session.runtime.execute(&write).await;
7515        assert!(
7516            wres.results[0].error.is_none(),
7517            "write_file via substrate failed: {:?}",
7518            wres.results[0].error
7519        );
7520        assert_eq!(
7521            files.lock().unwrap().get("/vm/a.txt").map(String::as_str),
7522            Some("vm-bytes"),
7523            "write_file must land on the bound VM substrate"
7524        );
7525
7526        // read_file back must return the VM's content.
7527        let rres = session
7528            .runtime
7529            .execute(&read_file_proposal("/vm/a.txt"))
7530            .await;
7531        assert!(
7532            rres.results[0].error.is_none(),
7533            "read_file via substrate failed: {:?}",
7534            rres.results[0].error
7535        );
7536        let out = rres.results[0].output.clone().unwrap_or(Value::Null);
7537        let content = out.get("content").and_then(|v| v.as_str()).unwrap_or("");
7538        // read_file output is line-numbered `cat -n` style (H1/F4-remainder,
7539        // audit 2026-07-06); the VM's single-line content becomes "     1\t<line>".
7540        assert_eq!(
7541            content, "     1\tvm-bytes",
7542            "read_file must come from the VM substrate"
7543        );
7544    }
7545
7546    /// The shadow executor: substrate-owned built-in names return the
7547    /// `"unknown tool"` prefix (so the engine falls through to the bound
7548    /// substrate), while any other tool name is delegated to the inner
7549    /// executor unchanged (connector `mcp_*` routes + host WS callbacks).
7550    #[tokio::test]
7551    async fn shadow_executor_only_shadows_builtin_names() {
7552        struct Inner;
7553        #[async_trait::async_trait]
7554        impl ToolExecutor for Inner {
7555            async fn execute(&self, tool: &str, _p: &Value) -> Result<Value, String> {
7556                Ok(json!({ "delegated": tool }))
7557            }
7558        }
7559        let shadow = SubstrateShadowExecutor::new(Arc::new(Inner));
7560
7561        for name in SUBSTRATE_OWNED_TOOLS {
7562            let err = shadow.execute(name, &json!({})).await.unwrap_err();
7563            assert!(
7564                err.starts_with("unknown tool"),
7565                "{name} must be shadowed so the engine falls through: {err}"
7566            );
7567        }
7568
7569        // A connector tool name and a pure tool pass through untouched.
7570        for passthrough in ["mcp_vm_run_command", "calculate", "browser.run"] {
7571            let out = shadow.execute(passthrough, &json!({})).await.unwrap();
7572            assert_eq!(out["delegated"], json!(passthrough));
7573        }
7574    }
7575}
7576
7577/// `.car/policies/*.toml` reaching the runtime at all — the whole point of
7578/// wiring `load_project_policies` to a real call site. Exercised against a
7579/// temp directory rather than through `create_session`, which would read the
7580/// developer's real `$HOME`.
7581#[cfg(test)]
7582mod project_policy_loading_tests {
7583    use super::*;
7584
7585    fn runtime() -> Runtime {
7586        Runtime::new()
7587    }
7588
7589    #[tokio::test]
7590    async fn a_missing_policies_directory_is_silent() {
7591        let dir = tempfile::tempdir().unwrap();
7592        // No `.car` at all, let alone `.car/policies`.
7593        apply_project_policies(&runtime(), &dir.path().join(".car"))
7594            .await
7595            .expect("a project with no rules must start normally");
7596    }
7597
7598    #[tokio::test]
7599    async fn an_empty_policies_directory_is_silent() {
7600        let dir = tempfile::tempdir().unwrap();
7601        let car = dir.path().join(".car");
7602        std::fs::create_dir_all(car.join("policies")).unwrap();
7603        apply_project_policies(&runtime(), &car).await.unwrap();
7604    }
7605
7606    #[tokio::test]
7607    async fn a_well_formed_rule_file_loads_and_takes_effect() {
7608        let dir = tempfile::tempdir().unwrap();
7609        let car = dir.path().join(".car");
7610        std::fs::create_dir_all(car.join("policies")).unwrap();
7611        // Exactly the shape `car_policy::rules`' module docs advertise: a
7612        // top-level list for `deny_tool`, a table array for the per-parameter
7613        // rules.
7614        std::fs::write(
7615            car.join("policies").join("messaging.toml"),
7616            "deny_tool = [\"messaging.send\"]\n\n\
7617             [[deny_tool_param]]\n\
7618             tool = \"messaging.send\"\n\
7619             param = \"channel\"\n\
7620             equals = \"slack\"\n",
7621        )
7622        .unwrap();
7623
7624        let rt = runtime();
7625        apply_project_policies(&rt, &car).await.unwrap();
7626
7627        // Registered on the runtime's engine, not merely parsed.
7628        let names: Vec<String> = rt
7629            .list_policies(None)
7630            .await
7631            .unwrap()
7632            .into_iter()
7633            .map(|(name, _)| name)
7634            .collect();
7635        assert!(
7636            names.len() >= 2,
7637            "both declarative rules must reach the policy engine, got {names:?}"
7638        );
7639    }
7640
7641    #[tokio::test]
7642    async fn a_malformed_rule_file_fails_loudly_and_names_the_file() {
7643        let dir = tempfile::tempdir().unwrap();
7644        let car = dir.path().join(".car");
7645        std::fs::create_dir_all(car.join("policies")).unwrap();
7646        std::fs::write(
7647            car.join("policies").join("broken.toml"),
7648            "[[deny_tool]\ntool = \"shell\"",
7649        )
7650        .unwrap();
7651
7652        let err = apply_project_policies(&runtime(), &car)
7653            .await
7654            .expect_err("a malformed policy file must NOT be swallowed");
7655        assert!(
7656            err.contains("broken.toml"),
7657            "the operator has to know which file to fix: {err}"
7658        );
7659        assert!(err.contains("refusing to start"), "{err}");
7660    }
7661}
7662
7663#[cfg(test)]
7664mod org_scope_wiring_tests {
7665    use super::*;
7666
7667    fn granter_hex_for(secret: &[u8], user: &str) -> String {
7668        let master = car_sync::StretchedMaster::from_issued_high_entropy(secret, user);
7669        let vk = car_sync::ed25519_verifying(&car_sync::derive_ed25519_identity(&master, user));
7670        vk.to_bytes().iter().map(|b| format!("{b:02x}")).collect()
7671    }
7672    fn granter_hex() -> String {
7673        granter_hex_for(b"granter-login", "acc_granter")
7674    }
7675
7676    #[test]
7677    fn unset_env_is_the_personal_only_path() {
7678        // The real-tenants-OFF invariant: no value → Ok(None) → base provider.
7679        assert!(ServerState::parse_org_scope_config(None).unwrap().is_none());
7680        assert!(ServerState::parse_org_scope_config(Some(String::new()))
7681            .unwrap()
7682            .is_none());
7683    }
7684
7685    #[test]
7686    fn wellformed_env_parses_org_and_granter() {
7687        let hex = granter_hex();
7688        let (org, granters) = ServerState::parse_org_scope_config(Some(format!("orgtest:{hex}")))
7689            .unwrap()
7690            .expect("well-formed opt-in parses");
7691        assert_eq!(org, "orgtest");
7692        assert_eq!(granters.len(), 1);
7693        let expected_master =
7694            car_sync::StretchedMaster::from_issued_high_entropy(b"granter-login", "acc_granter");
7695        let expected = car_sync::ed25519_verifying(&car_sync::derive_ed25519_identity(
7696            &expected_master,
7697            "acc_granter",
7698        ));
7699        assert_eq!(granters[0].to_bytes(), expected.to_bytes());
7700    }
7701
7702    #[test]
7703    fn multiple_granters_parse_in_order() {
7704        // The trusted set is plural: an org can designate several granters.
7705        let g1 = granter_hex_for(b"g1", "acc_g1");
7706        let g2 = granter_hex_for(b"g2", "acc_g2");
7707        let (org, granters) =
7708            ServerState::parse_org_scope_config(Some(format!("orgtest:{g1},{g2}")))
7709                .unwrap()
7710                .expect("well-formed multi-granter opt-in parses");
7711        assert_eq!(org, "orgtest");
7712        assert_eq!(granters.len(), 2);
7713        assert_ne!(granters[0].to_bytes(), granters[1].to_bytes());
7714        // Surrounding whitespace on a granter entry is tolerated.
7715        let spaced = ServerState::parse_org_scope_config(Some(format!("orgtest:{g1} , {g2}")))
7716            .unwrap()
7717            .unwrap();
7718        assert_eq!(spaced.1.len(), 2);
7719    }
7720
7721    #[test]
7722    fn one_bad_granter_in_the_list_fails_the_whole_value() {
7723        // A typo in ANY granter is a loud failure — never silently drop it and
7724        // proceed with a partial trusted set.
7725        let good = granter_hex();
7726        assert!(
7727            ServerState::parse_org_scope_config(Some(format!("orgtest:{good},not-hex"))).is_err(),
7728            "a malformed granter among valid ones must be a hard error"
7729        );
7730        // An org id with no granters at all is rejected.
7731        assert!(ServerState::parse_org_scope_config(Some("orgtest:".into())).is_err());
7732        assert!(ServerState::parse_org_scope_config(Some("orgtest: , ".into())).is_err());
7733    }
7734
7735    #[test]
7736    fn malformed_env_is_a_hard_error_never_silent() {
7737        // A typo must NOT silently fall through to a wrong/unauthenticated state.
7738        let hex = granter_hex();
7739        let bads = [
7740            "no-colon-here".to_string(),   // missing ':'
7741            ":deadbeef".to_string(),       // empty org
7742            "orgtest:not-hex".to_string(), // bad hex
7743            "orgtest:aabb".to_string(),    // wrong length
7744            format!("bad org:{hex}"),      // non-canonical org (space)
7745            format!("org/evil:{hex}"),     // non-canonical org (delimiter)
7746        ];
7747        for bad in &bads {
7748            assert!(
7749                ServerState::parse_org_scope_config(Some(bad.clone())).is_err(),
7750                "malformed value {bad:?} must be a hard error"
7751            );
7752        }
7753        // NOTE: an all-zero (identity/small-order) key PARSES — from_bytes accepts
7754        // the encoding; verify_strict rejects it at verification, so such a granter
7755        // simply grants nothing (fail-closed), which is correct, not a parse error.
7756    }
7757
7758    #[test]
7759    fn scope_routing_sends_only_matching_org_to_the_org_subsystem() {
7760        use car_sync::Scope;
7761        let acme = Scope::Shared { org: "acme".into() };
7762        let globex = Scope::Shared {
7763            org: "globex".into(),
7764        };
7765
7766        // org-scope OFF (no holder): EVERYTHING routes to the personal subsystem —
7767        // the byte-identical invariant.
7768        assert_eq!(route_for_scope(&Scope::Personal, None), SyncRoute::User);
7769        assert_eq!(route_for_scope(&acme, None), SyncRoute::User);
7770
7771        // opted into "acme": only Scope::Shared{acme} routes to the org subsystem.
7772        assert_eq!(route_for_scope(&acme, Some("acme")), SyncRoute::Org);
7773        assert_eq!(
7774            route_for_scope(&Scope::Personal, Some("acme")),
7775            SyncRoute::User
7776        );
7777        assert_eq!(
7778            route_for_scope(&globex, Some("acme")),
7779            SyncRoute::User,
7780            "a different org must NOT go to acme's delivery subsystem"
7781        );
7782    }
7783}