Skip to main content

trustee_api/
state.rs

1//! Shared server state: per-user multi-session registry, broadcast channels, and auth state.
2//!
3//! ## Multi-Session Per User (MSU)
4//!
5//! Each authenticated user gets their own [`UserSessions`] containing N independent
6//! [`UserSessionEntry`] instances (default max 4). Each entry has:
7//! - An independent `Session` (workflow state, output, etc.)
8//! - A dedicated broadcast channel for WebSocket fan-out
9//! - Creation and last-active timestamps
10//!
11//! Sessions are keyed by user identity (`sub` claim from JWT, or `dev:email` for
12//! dev mode). Unauthenticated deployments use a single `"default"` key, preserving
13//! backward compatibility with single-user CLI operation.
14
15use std::sync::Arc;
16
17use dashmap::DashMap;
18use tokio::sync::{broadcast, mpsc, Mutex};
19use trustee_core::session::Session;
20use trustee_core::types::TuiMessage;
21
22use crate::auth::AuthState;
23
24// ---------------------------------------------------------------------------
25// 16F console dispatch: user-key pinning (nghr ec3e0622 companion)
26// ---------------------------------------------------------------------------
27
28tokio::task_local! {
29    /// When set (by the `/xagent/{agent}` dispatch prelude), ALL
30    /// user-key-derived resolution — home dir, session bucket, project id,
31    /// secrets merge, MCP loader cache — pins to THIS key: the THQ profile
32    /// binding id (the stable identity anchor), NOT the token's claims.
33    /// Old-world 16E agents carry their Kanidm sub here, making the override
34    /// a bit-identical no-op for them. The ec3e0622 guarantee: token claims
35    /// can never re-home a principal.
36    static USER_KEY_OVERRIDE: String;
37}
38
39/// The effective namespace key for `user_key` under the active dispatch
40/// scope (see [`USER_KEY_OVERRIDE`]).
41pub(crate) fn effective_user_key(user_key: &str) -> String {
42    USER_KEY_OVERRIDE
43        .try_with(|v| v.clone())
44        .unwrap_or_else(|_| user_key.to_string())
45}
46
47/// Wrap a dispatched handler so every user-key-derived resolution inside it
48/// pins to the binding's key (see [`USER_KEY_OVERRIDE`]).
49pub(crate) async fn in_dispatch_scope<F: std::future::Future>(user_key: &str, fut: F) -> F::Output {
50    USER_KEY_OVERRIDE.scope(user_key.to_string(), fut).await
51}
52
53/// Deterministic re-link of pre-rotation homes (nghr ec3e0622 acceptance 3).
54///
55/// `users/home_links.json`: `{ "links": [ { "from_hash": "<data dir>",
56/// "to_hash": "<current claims-derived dir>" } ] }` — the data lives under
57/// `from_hash`; the claims-derived `to_hash` home is the (empty) one the
58/// rotation created. Applied at startup, idempotent:
59/// - `to` missing → rename `from` → `to` (config + sessions restored);
60/// - `to` exists but has ZERO entries → remove it, then rename;
61/// - `to` exists with content → LOUD skip (manual reconciliation required).
62pub fn apply_home_links() -> Vec<String> {
63    let Some(home) = dirs::home_dir() else {
64        return Vec::new();
65    };
66    let users = home.join(".trustee").join("users");
67    let map_path = users.join("home_links.json");
68    let Ok(text) = std::fs::read_to_string(&map_path) else {
69        return Vec::new();
70    };
71    let Ok(doc) = serde_json::from_str::<serde_json::Value>(&text) else {
72        tracing::warn!(
73            "home_links.json is not valid JSON — ignored: {}",
74            map_path.display()
75        );
76        return Vec::new();
77    };
78    let mut report = Vec::new();
79    if let Some(links) = doc.get("links").and_then(|v| v.as_array()) {
80        for l in links {
81            let (Some(from), Some(to)) = (
82                l.get("from_hash").and_then(|v| v.as_str()),
83                l.get("to_hash").and_then(|v| v.as_str()),
84            ) else {
85                continue;
86            };
87            let from_dir = users.join(from);
88            let to_dir = users.join(to);
89            if !from_dir.exists() {
90                continue; // already applied (or never existed)
91            }
92            if to_dir.exists() {
93                let empty = std::fs::read_dir(&to_dir)
94                    .map(|it| it.count() == 0)
95                    .unwrap_or(false);
96                if !empty {
97                    let msg = format!(
98                        "home link {from} → {to}: target exists with content — NOT touched, reconcile manually"
99                    );
100                    tracing::warn!("home_links: {msg}");
101                    report.push(msg);
102                    continue;
103                }
104                let _ = std::fs::remove_dir(&to_dir);
105            }
106            match std::fs::rename(&from_dir, &to_dir) {
107                Ok(()) => {
108                    let msg =
109                        format!("home link applied: {from} → {to} (config/sessions restored)");
110                    tracing::info!("home_links: {msg}");
111                    report.push(msg);
112                }
113                Err(e) => {
114                    let msg = format!("home link {from} → {to} FAILED: {e}");
115                    tracing::warn!("home_links: {msg}");
116                    report.push(msg);
117                }
118            }
119        }
120    }
121    report
122}
123
124// ---------------------------------------------------------------------------
125// Multi-session types
126// ---------------------------------------------------------------------------
127
128/// A single session with its own broadcast channel.
129pub struct UserSessionEntry {
130    /// The agent session, protected by a mutex.
131    pub session: Arc<Mutex<Session>>,
132    /// Broadcast sender for this session's WebSocket fan-out.
133    pub ws_tx: broadcast::Sender<String>,
134    /// When this session was created.
135    pub created_at: chrono::DateTime<chrono::Utc>,
136    /// Last time a command was submitted or state changed.
137    /// Updated on every /sessions/{id}/command and /sessions/{id}/cancel call.
138    pub last_active: Arc<Mutex<chrono::DateTime<chrono::Utc>>>,
139    /// The drain task's abort handle (a3e1f3b6, owner report 2026-11-14): the
140    /// drain task holds the ORIGINAL ws_tx sender + the session Arc — without
141    /// aborting it on destroy the broadcast channel never closes (the arc
142    /// cycle: the drain task keeps the session alive, the session keeps
143    /// workflow_tx alive, workflow_rx never closes) and the connected
144    /// clients' WS streams stay open (no onclose — the console's 'Session
145    /// ended' detection never fires) while the destroyed session's memory
146    /// leaks until process exit.
147    pub drain_abort: Option<tokio::task::AbortHandle>,
148}
149
150/// All sessions belonging to one authenticated user.
151pub struct UserSessions {
152    /// session_id → session entry
153    pub sessions: DashMap<String, UserSessionEntry>,
154    /// Shared token store for all this user's sessions (MCP credential isolation).
155    pub token_store: Arc<pep::MemoryTokenStore>,
156}
157
158/// Summary of an active session for listing (serializable for API responses).
159#[derive(Debug, serde::Serialize)]
160pub struct SessionListItem {
161    pub session_id: String,
162    pub session_name: Option<String>,
163    pub workflow_state: String,
164    pub created_at: String,
165    pub last_active: String,
166    /// Number of handoff rotations this session has performed (Bug 5). The
167    /// in-memory registry key never changes across a rotation — only the
168    /// checkpoint-chain name does — so a card whose name changed without this
169    /// hint looks like it silently renamed. 0 = never rotated.
170    pub handoff_count: u32,
171}
172
173/// Errors from multi-session operations.
174#[derive(Debug)]
175pub enum SessionError {
176    /// User has reached max_sessions_per_user limit.
177    MaxSessionsReached(usize),
178    /// Session ID not found for this user.
179    NotFound(String),
180    /// Session is not Idle (cannot destroy/overwrite a running session).
181    NotIdle(String),
182}
183
184impl std::fmt::Display for SessionError {
185    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
186        match self {
187            SessionError::MaxSessionsReached(n) => {
188                write!(f, "Maximum {} sessions per user reached", n)
189            }
190            SessionError::NotFound(id) => write!(f, "Session {} not found", id),
191            SessionError::NotIdle(state) => write!(f, "Session is not idle (state: {})", state),
192        }
193    }
194}
195
196impl std::error::Error for SessionError {}
197
198/// Top-level registry: user_key → user's session collection.
199pub type SessionRegistry = Arc<DashMap<String, UserSessions>>;
200
201// ---------------------------------------------------------------------------
202// ServerState
203// ---------------------------------------------------------------------------
204
205/// Cached per-user MCP tool loader (16C).
206///
207/// One entry per user hash. `loader: None` means the user's effective
208/// config has MCP disabled — agents run MCP-less via abk's
209/// `McpSource::Prebuilt(None)` (semantically identical to
210/// `[mcp] enabled = false` today, but with zero config re-parsing).
211pub struct McpLoaderEntry {
212    /// Built loader; `None` = MCP disabled for this user.
213    pub loader: Option<std::sync::Arc<abk::agent::McpToolLoader>>,
214    /// Content fingerprint of the effective `[mcp]` config (SHA-256, first
215    /// 8 bytes as u64). Content, never mtime — overlays are rewritten in place.
216    pub fingerprint: u64,
217    pub built_at: chrono::DateTime<chrono::Utc>,
218    /// Set when the last build failed; the message surfaces to the user's
219    /// next dispatch (fail loud). Other users are never affected.
220    pub degraded: Option<String>,
221    /// When `degraded` was set. Rebuilds are held back for
222    /// [`MCP_BUILD_RETRY_BACKOFF`] — a poison entry never sticks forever.
223    pub failed_at: Option<chrono::DateTime<chrono::Utc>>,
224}
225
226/// Minimum delay before retrying a failed MCP loader build (16C).
227pub const MCP_BUILD_RETRY_BACKOFF: std::time::Duration = std::time::Duration::from_secs(30);
228
229/// Maximum age of a cached MCP toolset manifest (42131deb).
230///
231/// The 16C fingerprint covers the LOCAL config only — a served-surface
232/// change behind a byte-identical config (bridge/spec deploy) leaves the
233/// fingerprint matching while the cached tool list goes stale: phantom
234/// tools for every fresh session until a restart. Past this age the next
235/// fresh-session dispatch REBUILDS the loader (fresh connect re-lists the
236/// server's tools), bounding the phantom window to ≤ 10 minutes with no
237/// operator action.
238pub const MCP_TOOLSET_MAX_AGE: std::time::Duration = std::time::Duration::from_secs(600);
239
240/// Why a cached MCP loader entry was (or wasn't) reused (42131deb).
241#[derive(Debug, Clone, PartialEq, Eq)]
242pub(crate) enum LoaderFreshness {
243    /// Fingerprint matches and the entry is young — reuse.
244    Fresh,
245    /// Effective config content changed — rebuild (existing 16C behavior).
246    ConfigChanged,
247    /// Entry older than [`MCP_TOOLSET_MAX_AGE`] — the SERVED surface may
248    /// have changed under an unchanged local config; rebuild so the next
249    /// fresh session re-lists the server's tools.
250    AgedOut,
251    /// Degraded entry still inside the retry backoff — fail loud.
252    Backoff(std::time::Duration),
253}
254
255/// Pure staleness decision for a cache entry — unit-testable; the async
256/// cache path consumes it verbatim (b597c9e9 decide-fn pattern).
257pub(crate) fn loader_freshness(
258    entry_fingerprint: u64,
259    entry_built_at: chrono::DateTime<chrono::Utc>,
260    entry_degraded: Option<&str>,
261    entry_failed_at: Option<chrono::DateTime<chrono::Utc>>,
262    wanted_fingerprint: u64,
263    now: chrono::DateTime<chrono::Utc>,
264) -> LoaderFreshness {
265    // Degraded entries first: a poison entry never sticks forever, but it
266    // is held back for the retry window regardless of age/changes.
267    if entry_degraded.is_some() {
268        if let Some(failed_at) = entry_failed_at {
269            let backoff = chrono::Duration::from_std(MCP_BUILD_RETRY_BACKOFF)
270                .unwrap_or_else(|_| chrono::Duration::seconds(30));
271            if now < failed_at + backoff {
272                return LoaderFreshness::Backoff(
273                    (failed_at + backoff - now).to_std().unwrap_or_default(),
274                );
275            }
276        }
277    }
278    if wanted_fingerprint != entry_fingerprint {
279        return LoaderFreshness::ConfigChanged;
280    }
281    let max_age = chrono::Duration::from_std(MCP_TOOLSET_MAX_AGE)
282        .unwrap_or_else(|_| chrono::Duration::seconds(600));
283    if now > entry_built_at + max_age {
284        LoaderFreshness::AgedOut
285    } else {
286        LoaderFreshness::Fresh
287    }
288}
289
290/// 16F: per-agent dispatch target — THQ agent_name → the agent-user that
291/// THQ-dispatched sessions must run AS.
292///
293/// Populated once at boot by the 16E discovery scan
294/// (`thq_dispatch::populate`); restart-only lifecycle, exactly like the
295/// enrollment lane itself.
296#[derive(Debug, Clone)]
297pub struct ThqDispatchEntry {
298    /// The agent-user's stable key = its Kanidm `sub` (16E sub-pin). This is
299    /// the identity impersonated on the inner dispatch: session bucket,
300    /// per-user home, MCP loader cache, and Cedar principal all resolve
301    /// through it.
302    pub user_key: String,
303    /// The agent's Kanidm service token from its per-user `.env` — exchanged
304    /// for a short-lived `role=agent` Bearer at dispatch time. `None` = the
305    /// agent has no provisioned credential (dispatch fails loud, 502).
306    pub service_token: Option<String>,
307    /// The issuer the token's own credential declares (`[mcp.credentials.*]
308    /// .issuer_url` from the agent's overlay). Kanidm accepts a token
309    /// exchange only on the origin the token was minted for — the issuer
310    /// MUST travel with the credential that owns the token (verified live
311    /// 2026-08-30: same token, 200 on its own vhost, 400 elsewhere).
312    pub issuer_url: Option<String>,
313}
314
315/// 16F: the service-account issuer declared by the shared config's first
316/// `[mcp.credentials.*]` entry with `type = "service-account"` (see
317/// `ServerState::service_issuer` for why this exists).
318pub fn service_issuers_from_config(config_toml: &str) -> Vec<String> {
319    let Ok(v) = config_toml.parse::<toml::Value>() else {
320        return Vec::new();
321    };
322    let Some(creds) = v
323        .get("mcp")
324        .and_then(|m| m.get("credentials"))
325        .and_then(|c| c.as_table())
326    else {
327        return Vec::new();
328    };
329    let mut out = Vec::new();
330    for (_name, cred) in creds {
331        if cred.get("type").and_then(|t| t.as_str()) == Some("service-account") {
332            if let Some(issuer) = cred.get("issuer_url").and_then(|i| i.as_str()) {
333                if !out.iter().any(|e: &String| e == issuer) {
334                    out.push(issuer.to_string());
335                }
336            }
337        }
338    }
339    out
340}
341
342/// Shared state accessible by all axum handlers.
343#[derive(Clone)]
344pub struct ServerState {
345    /// Per-user multi-session registry (MSU).
346    pub sessions: SessionRegistry,
347    /// Broadcast sender for backward compat — delegates to the default user's channel.
348    pub ws_tx: broadcast::Sender<String>,
349    /// Auth state (None = auth disabled, all endpoints open).
350    pub auth: Option<Arc<AuthState>>,
351    /// Shared config TOML (all users share the same agent config).
352    pub config_toml: Option<String>,
353    /// Shared secrets (injected into every per-user session).
354    pub secrets: Option<std::collections::HashMap<String, String>>,
355    /// Shared build info (injected into every per-user session).
356    pub build_info: Option<trustee_core::types::BuildInfo>,
357    /// Global concurrency limiter — limits the number of simultaneous workflows
358    /// across all users. Default: 8 concurrent workflows.
359    pub workflow_semaphore: Arc<tokio::sync::Semaphore>,
360    /// Maximum number of concurrent sessions per user. Default: 4.
361    pub max_sessions_per_user: usize,
362    /// Whether per-user config overlays may override the `[llm]` section.
363    /// Default: `false` — overlays are limited to `[mcp]`.
364    pub allow_llm_overlay: bool,
365    /// Per-user McpToolLoader cache (16C), keyed by user hash (never the raw key).
366    pub mcp_loaders: Arc<DashMap<String, McpLoaderEntry>>,
367    /// Single-flight build locks per user hash (16C).
368    mcp_build_locks: Arc<DashMap<String, Arc<tokio::sync::Mutex<()>>>>,
369    /// 16F: THQ per-agent dispatch table (agent_name → target), boot-populated.
370    pub thq_dispatch: Arc<DashMap<String, ThqDispatchEntry>>,
371    /// 16F: cached impersonation Bearers per agent user_key
372    /// `(token, expires_at)` — keyed on the exchanged JWT's REAL `exp` claim
373    /// (payload-parsed, never the `expires_in` estimate), proactively
374    /// refreshed at T-120s before expiry (±15s per-process jitter), and
375    /// invalidated + re-minted by the budget-of-one 401 retry in the
376    /// xagent dispatch wrappers (199c4801/9618b41c; elimination spec
377    /// fb82b866 response c35042bb leg L2).
378    pub agent_dispatch_tokens: Arc<DashMap<String, (String, std::time::Instant)>>,
379}
380
381impl ServerState {
382    /// Create new shared state from a default session, broadcast sender, and optional auth.
383    pub fn new(
384        session: Session,
385        ws_tx: broadcast::Sender<String>,
386        auth: Option<Arc<AuthState>>,
387    ) -> Self {
388        // nghr ec3e0622 acceptance 3: deterministic re-link of pre-rotation
389        // homes, applied once at startup (idempotent; every action logged).
390        apply_home_links();
391
392        let sessions = Arc::new(DashMap::new());
393
394        // Store the default user's UserSessions with an initial session
395        let token_store = Arc::new(pep::MemoryTokenStore::new());
396        let (ws_tx_entry, _) = broadcast::channel::<String>(256);
397
398        let now = chrono::Utc::now();
399        let initial_entry = UserSessionEntry {
400            session: Arc::new(Mutex::new(session)),
401            ws_tx: ws_tx_entry,
402            created_at: now,
403            last_active: Arc::new(Mutex::new(now)),
404            drain_abort: None,
405        };
406
407        let user_sessions = UserSessions {
408            sessions: DashMap::new(),
409            token_store,
410        };
411        user_sessions
412            .sessions
413            .insert("default".to_string(), initial_entry);
414
415        sessions.insert("default".to_string(), user_sessions);
416
417        Self {
418            sessions,
419            ws_tx,
420            auth,
421            config_toml: None,
422            secrets: None,
423            build_info: None,
424            workflow_semaphore: Arc::new(tokio::sync::Semaphore::new(8)),
425            max_sessions_per_user: 4,
426            allow_llm_overlay: false,
427            mcp_loaders: Arc::new(DashMap::new()),
428            mcp_build_locks: Arc::new(DashMap::new()),
429            thq_dispatch: Arc::new(DashMap::new()),
430            agent_dispatch_tokens: Arc::new(DashMap::new()),
431        }
432    }
433
434    pub fn with_config_toml(mut self, config_toml: String) -> Self {
435        // fffe9141 F3: loud boot WARN when [llm.provider].model is
436        // missing/empty — the silent fictional fallback is gone (abk 0.22.8
437        // F5), so a missing model explodes at the first LLM call with the
438        // honest 'no model configured' error. THQ-runtime instances
439        // legitimately boot without serving LLM sessions — warn, never
440        // refuse.
441        let model = config_toml
442            .parse::<toml::Value>()
443            .ok()
444            .and_then(|v| {
445                v.get("llm")
446                    .and_then(|l| l.get("provider"))
447                    .and_then(|p| p.get("model"))
448                    .and_then(|m| m.as_str())
449                    .map(|s| s.trim().to_string())
450            })
451            .unwrap_or_default();
452        if model.is_empty() {
453            tracing::warn!(
454                "[llm.provider].model is missing or empty — LLM calls will fail with 'no model configured: set [llm.provider].model or OPENAI_DEFAULT_MODEL' until set (THQ-runtime instances that never serve LLM sessions may ignore this)"
455            );
456        }
457        self.config_toml = Some(config_toml);
458        self
459    }
460
461    /// 16F: the issuer URL Kanidm accepts SERVICE-ACCOUNT token exchange on.
462    ///
463    /// Kanidm binds exchange to the OAuth2 client's configured origin: the
464    /// `[oidc]` issuer host serves logins/validation fine but REJECTS token
465    /// exchange with `invalid_request`, while the `idp` vhost — the issuer
466    /// every working `[mcp.credentials.*]` service-account entry already
467    /// uses — accepts it (verified live 2026-08-30: same token, 200 vs 400).
468    /// The exchange therefore takes its issuer from the shared config's
469    /// first `service-account` credential; callers fall back to the auth
470    /// issuer when absent.
471    pub fn service_issuers(&self) -> Vec<String> {
472        service_issuers_from_config(self.config_toml.as_deref().unwrap_or(""))
473    }
474
475    pub fn with_secrets(mut self, secrets: std::collections::HashMap<String, String>) -> Self {
476        self.secrets = Some(secrets);
477        self
478    }
479
480    pub fn with_build_info(mut self, build_info: trustee_core::types::BuildInfo) -> Self {
481        self.build_info = Some(build_info);
482        self
483    }
484
485    pub fn with_max_concurrent_workflows(mut self, max: usize) -> Self {
486        self.workflow_semaphore = Arc::new(tokio::sync::Semaphore::new(max));
487        self
488    }
489
490    /// Set the max sessions per user.
491    pub fn with_max_sessions_per_user(mut self, max: usize) -> Self {
492        self.max_sessions_per_user = max;
493        self
494    }
495
496    /// Allow per-user config overlays to override the `[llm]` section.
497    /// Default: `false` (overlays limited to `[mcp]`).
498    pub fn with_allow_llm_overlay(mut self, allow: bool) -> Self {
499        self.allow_llm_overlay = allow;
500        self
501    }
502
503    // -----------------------------------------------------------------------
504    // MSU: Multi-session methods
505    // -----------------------------------------------------------------------
506
507    /// Create a new session for a user. Returns the session_id.
508    ///
509    /// Creates a fresh `Session::new()`, copies shared config, sets per-user
510    /// isolation, creates a broadcast channel, spawns a drain task, and inserts
511    /// into the user's session DashMap.
512    ///
513    /// The session identity is PINNED here (c2f9508, restore-reattach fix):
514    /// `session.session_id` = the registry key, so the disk checkpoint chain id
515    /// and the live session id coincide and `get_or_restore_session_by_any_id`
516    /// can re-attach a client by the id it already holds after an engine
517    /// restart. The abk first-command auto-derive still fires for TUI/desktop
518    /// sessions (no registry key there). Resumed sessions overwrite the pin
519    /// with the resumed chain id (routes::resume_session) — chain semantics
520    /// preserved.
521    ///
522    /// There is NO "active session" concept anymore — the legacy
523    /// active-pointer surface was retired (2026-10-27, owner order); every
524    /// consumer targets a session id via the session-scoped routes.
525    pub async fn create_session(
526        &self,
527        user_key: &str,
528        session_name: Option<String>,
529        identity: Option<String>,
530    ) -> Result<String, SessionError> {
531        // Get or create the user's UserSessions entry
532        let user_sessions = self
533            .sessions
534            .entry(user_key.to_string())
535            .or_insert_with(|| UserSessions {
536                sessions: DashMap::new(),
537                token_store: Arc::new(pep::MemoryTokenStore::new()),
538            });
539
540        // Check session limit
541        if user_sessions.sessions.len() >= self.max_sessions_per_user {
542            return Err(SessionError::MaxSessionsReached(self.max_sessions_per_user));
543        }
544
545        // Create new Session
546        let (mut session, workflow_rx) = Session::new();
547
548        // Copy shared config
549        if let Some(ref config_toml) = self.config_toml {
550            session.config_toml = Some(config_toml.clone());
551            session.parse_auto_handoff_config();
552            if let Ok(table) = config_toml.parse::<toml::Value>() {
553                if let Some(name) = table
554                    .get("agent")
555                    .and_then(|a| a.get("name"))
556                    .and_then(|n| n.as_str())
557                {
558                    session.agent_name = name.to_string();
559                }
560            }
561        }
562
563        session.secrets = self.secrets.clone();
564        session.build_info = self.build_info.clone();
565
566        // Per-user isolation
567        self.apply_user_isolation(&mut session, user_key);
568
569        // Apply session_name if provided
570        session.session_name = session_name;
571
572        // Apply agent identity if provided
573        session.identity = identity;
574
575        // Create broadcast channel
576        let (ws_tx_entry, _) = broadcast::channel::<String>(256);
577
578        // Generate session_id
579        let session_id = format!(
580            "session_{}_{}",
581            chrono::Utc::now().format("%Y_%m_%d_%H_%M"),
582            &uuid::Uuid::new_v4().to_string()[..8]
583        );
584
585        // PIN the session identity (Hamrah 2026-10-26, restore-reattach fix):
586        // the registry key IS the disk chain id. Without this, the first
587        // command's abk run auto-generates a DIFFERENT session_id for the
588        // on-disk checkpoint chain, and get_or_restore_session_by_any_id can
589        // never find the chain by the live id after an engine restart.
590        session.session_id = Some(session_id.clone());
591
592        let now = chrono::Utc::now();
593
594        // Insert into user's sessions DashMap
595        user_sessions.sessions.insert(
596            session_id.clone(),
597            UserSessionEntry {
598                session: Arc::new(Mutex::new(session)),
599                ws_tx: ws_tx_entry.clone(),
600                created_at: now,
601                last_active: Arc::new(Mutex::new(now)),
602                drain_abort: None,
603            },
604        );
605
606        // Spawn drain task
607        let session_arc = user_sessions
608            .sessions
609            .get(&session_id)
610            .map(|e| e.session.clone());
611        if let Some(session_arc) = session_arc {
612            let handle =
613                self.spawn_user_drain_task(session_id.clone(), session_arc, ws_tx_entry, workflow_rx);
614            // Store the abort handle (a3e1f3b6) so destroy_session can close
615            // the channel — the drain task holds the ORIGINAL sender + the
616            // session Arc.
617            if let Some(mut e) = user_sessions.sessions.get_mut(&session_id) {
618                e.drain_abort = Some(handle.abort_handle());
619            }
620        }
621
622        Ok(session_id)
623    }
624
625    /// Get a specific session by user_key + session_id.
626    /// Updates last_active on the session entry.
627    pub async fn get_session(
628        &self,
629        user_key: &str,
630        session_id: &str,
631    ) -> Option<(Arc<Mutex<Session>>, broadcast::Sender<String>)> {
632        let user_sessions = self.sessions.get(user_key)?;
633        let entry = user_sessions.sessions.get(session_id)?;
634
635        // Update last_active
636        let now = chrono::Utc::now();
637        *entry.last_active.lock().await = now;
638
639        Some((entry.session.clone(), entry.ws_tx.clone()))
640    }
641
642    /// Get a session by EITHER its live MSU registry key OR its
643    /// checkpoint/session identity (`session.session_id`).
644    ///
645    /// The web frontend tracks `currentSessionId` from the `ResumeInfo` WS
646    /// message, which carries the auto-derived checkpoint id
647    /// (`session_YYYY_MM_DD_HH_MM_uuid8`) — NOT the live MSU registry key
648    /// (`"default"` or the key from `create_session()`). External clients
649    /// like Torpi/THQ pass the live registry key. This resolver accepts both:
650    ///
651    /// 1. Try registry-key lookup first (precise, used by Torpi/THQ).
652    /// 2. Fall back to scanning the user's live sessions for one whose
653    ///    `session.session_id` matches the requested id (used by the
654    ///    embedded web UI after a command or resume).
655    ///
656    /// Returns `(live_registry_key, session_arc, ws_tx)`, or `None` if not
657    /// found. The live key is returned so callers that need to set it as
658    /// active (or otherwise reference the registry) use the real key.
659    pub async fn get_session_by_any_id(
660        &self,
661        user_key: &str,
662        id: &str,
663    ) -> Option<(String, Arc<Mutex<Session>>, broadcast::Sender<String>)> {
664        // Fast path: registry key match.
665        let user_sessions = self.sessions.get(user_key)?;
666        if let Some(entry) = user_sessions.sessions.get(id) {
667            // Update last_active
668            let now = chrono::Utc::now();
669            *entry.last_active.lock().await = now;
670            return Some((id.to_string(), entry.session.clone(), entry.ws_tx.clone()));
671        }
672
673        // Slow path: scan live sessions for a matching session.session_id.
674        for entry in user_sessions.sessions.iter() {
675            let session = entry.session.lock().await;
676            if session.session_id.as_deref() == Some(id) {
677                let key = entry.key().clone();
678                let ws_tx = entry.ws_tx.clone();
679                drop(session);
680                // Update last_active
681                let now = chrono::Utc::now();
682                *entry.last_active.lock().await = now;
683                return Some((key, entry.session.clone(), ws_tx));
684            }
685        }
686
687        None
688    }
689
690    /// Tombstone directory for destroyed-but-on-disk session chains.
691    ///
692    /// Destroy removes the live session but deliberately keeps checkpoints
693    /// ("checkpoint data is preserved"). Restoring must NOT resurrect a
694    /// chain the user explicitly terminated, so destroy writes a marker
695    /// here and [`Self::restore_session_from_checkpoint`] refuses to
696    /// hydrate tombstoned chains. Markers are tiny and intentionally
697    /// permanent.
698    fn tombstone_path(&self, user_key: &str, chain_id: &str) -> Option<std::path::PathBuf> {
699        if chain_id.is_empty() || !chain_id.starts_with("session_") {
700            return None;
701        }
702        let user_key = effective_user_key(user_key);
703        let hash = trustee_core::user_hash(&user_key);
704        dirs::home_dir().map(|home| {
705            home.join(".trustee")
706                .join("users")
707                .join(&hash)
708                .join("session_tombstones")
709                .join(chain_id)
710        })
711    }
712
713    /// Write a destroy tombstone for a checkpoint chain (best effort).
714    ///
715    /// Called from [`Self::destroy_session`] when the destroyed live
716    /// session was attached to an on-disk chain.
717    fn write_tombstone(&self, user_key: &str, chain_id: &str) {
718        if let Some(path) = self.tombstone_path(user_key, chain_id) {
719            if let Some(parent) = path.parent() {
720                let _ = std::fs::create_dir_all(parent);
721            }
722            let _ = std::fs::write(&path, chrono::Utc::now().to_rfc3339());
723        }
724    }
725
726    /// Read the destroy-tombstone stamp for a chain, if one exists.
727    ///
728    /// 7df3cae2 (tombstone-shadow class): a destroyed chain whose
729    /// checkpoints sit intact on disk must be DISCLOSED, never shadowed by
730    /// a lying generic absence — readers use this stamp to answer
731    /// tombstoned-but-non-empty loudly. `None` = no marker (chain either
732    /// alive/resumable or genuinely absent).
733    pub fn chain_tombstone_stamp(&self, user_key: &str, chain_id: &str) -> Option<String> {
734        let path = self.tombstone_path(user_key, chain_id)?;
735        let text = std::fs::read_to_string(&path).ok()?;
736        let trimmed = text.trim();
737        if trimmed.is_empty() {
738            None
739        } else {
740            Some(trimmed.to_string())
741        }
742    }
743
744    /// Look up a live session by any id; on a miss, transparently restore
745    /// the checkpoint chain from disk.
746    ///
747    /// Restart resilience: trustee-api holds live sessions in memory only,
748    /// so a process restart orphans every open console. When a client asks
749    /// for a chain id that is not in memory but EXISTS on disk
750    /// (checkpoints present, not tombstoned by destroy), a live session is
751    /// rebuilt with `resume_info` pointing at the latest checkpoint and
752    /// registered under the chain id. The next command continues the
753    /// conversation; the console's alive-check and WS reconnect succeed.
754    ///
755    /// Returns `(live_registry_key, session_arc, ws_tx)` — the key is the
756    /// chain id for restored sessions.
757    pub async fn get_or_restore_session_by_any_id(
758        &self,
759        user_key: &str,
760        id: &str,
761    ) -> Option<(String, Arc<Mutex<Session>>, broadcast::Sender<String>)> {
762        if let Some(found) = self.get_session_by_any_id(user_key, id).await {
763            return Some(found);
764        }
765        self.restore_session_from_checkpoint(user_key, id).await
766    }
767
768    /// Rebuild a live session from an on-disk checkpoint chain.
769    ///
770    /// Persona note: dispatched (THQ/xagent) sessions carry `identity:
771    /// None` by design — their persona is the per-user overlay config
772    /// ([lifecycle].system_template), which `apply_user_isolation` +
773    /// `get_user_config_and_home` re-apply here exactly as at creation.
774    /// Explicit per-request identity overrides (rare) are not persisted
775    /// and therefore not restored.
776    async fn restore_session_from_checkpoint(
777        &self,
778        user_key: &str,
779        chain_id: &str,
780    ) -> Option<(String, Arc<Mutex<Session>>, broadcast::Sender<String>)> {
781        // Never resurrect an explicitly destroyed chain.
782        if let Some(path) = self.tombstone_path(user_key, chain_id) {
783            if path.exists() {
784                return None;
785            }
786        }
787
788        // Restore needs the caller's resolved config (persona) and home.
789        let (config_toml, home_dir) = self.get_user_config_and_home(user_key);
790        let config_toml = config_toml?;
791
792        // The chain must exist on disk with at least one checkpoint;
793        // otherwise this id is genuinely unknown → 404 semantics.
794        //
795        // Resume-handle resolution (Ravand 2026-10-27): a resumed session's
796        // live id is a fresh registry key — its disk dir carries a
797        // `resumed_from` constant naming the SOURCE chain (recorded at
798        // resume time by write_resume_pointer). Follow it so
799        // restore-by-held-id works after an engine restart: the SOURCE
800        // chain is restored under the HELD id, and the next command
801        // continues the source conversation.
802        let info = match trustee_core::sessions::create_resume_info(
803            &config_toml,
804            chain_id,
805            home_dir.as_deref(),
806        )
807        .await
808        .ok()
809        .flatten()
810        {
811            Some(info) => info,
812            None => {
813                let source = trustee_core::sessions::resolve_resume_handle(
814                    &config_toml,
815                    chain_id,
816                    home_dir.as_deref(),
817                )
818                .await
819                .ok()
820                .flatten()?;
821                trustee_core::sessions::create_resume_info(
822                    &config_toml,
823                    &source,
824                    home_dir.as_deref(),
825                )
826                .await
827                .ok()
828                .flatten()?
829            }
830        };
831
832        // Single-flight per chain id: racing lookups dedupe on the DashMap
833        // entry — the first insert wins, the loser adopts the winner.
834        let user_sessions = self
835            .sessions
836            .entry(user_key.to_string())
837            .or_insert_with(|| UserSessions {
838                sessions: DashMap::new(),
839                token_store: Arc::new(pep::MemoryTokenStore::new()),
840            });
841
842        if let Some(existing) = user_sessions.sessions.get(chain_id) {
843            let now = chrono::Utc::now();
844            *existing.last_active.lock().await = now;
845            return Some((
846                chain_id.to_string(),
847                existing.session.clone(),
848                existing.ws_tx.clone(),
849            ));
850        }
851
852        // Respect the per-user cap: evict the least-recently-active Idle
853        // live session to make room (restores are user-initiated
854        // reattaches, so refusing them would recreate the lost-session
855        // bug for heavy users). If nothing is evictable, refuse.
856        if user_sessions.sessions.len() >= self.max_sessions_per_user {
857            let mut evict: Option<(String, chrono::DateTime<chrono::Utc>)> = None;
858            for entry in user_sessions.sessions.iter() {
859                let is_idle = entry.session.lock().await.workflow_state
860                    == trustee_core::types::WorkflowState::Idle;
861                if !is_idle {
862                    continue;
863                }
864                let la = entry.last_active.lock().await;
865                if evict.as_ref().is_none_or(|(_, t)| *la < *t) {
866                    evict = Some((entry.key().clone(), *la));
867                }
868            }
869            if let Some((victim, _)) = evict {
870                // Abort the victim's drain task (a3e1f3b6) — same arc cycle
871                // as destroy: without the abort its ws_tx + session Arc keep
872                // the channel open and the memory alive.
873                if let Some(mut v) = user_sessions.sessions.get_mut(&victim) {
874                    if let Some(handle) = v.drain_abort.take() {
875                        handle.abort();
876                    }
877                }
878                user_sessions.sessions.remove(&victim);
879            } else {
880                return None;
881            }
882        }
883
884        // Build the session exactly like create_session, then attach the
885        // on-disk chain identity and resume pointer.
886        let (mut session, workflow_rx) = Session::new();
887        session.config_toml = Some(config_toml.clone());
888        session.parse_auto_handoff_config();
889        if let Ok(table) = config_toml.parse::<toml::Value>() {
890            if let Some(name) = table
891                .get("agent")
892                .and_then(|a| a.get("name"))
893                .and_then(|n| n.as_str())
894            {
895                session.agent_name = name.to_string();
896            }
897        }
898        session.secrets = self.secrets.clone();
899        session.build_info = self.build_info.clone();
900        self.apply_user_isolation(&mut session, user_key);
901        session.session_id = Some(chain_id.to_string());
902        session.resume_info = Some(info);
903        session.session_name = None; // display name re-derives from metadata
904
905        let (ws_tx_entry, _) = broadcast::channel::<String>(256);
906        let now = chrono::Utc::now();
907
908        user_sessions.sessions.insert(
909            chain_id.to_string(),
910            UserSessionEntry {
911                session: Arc::new(Mutex::new(session)),
912                ws_tx: ws_tx_entry.clone(),
913                created_at: now,
914                last_active: Arc::new(Mutex::new(now)),
915                drain_abort: None,
916            },
917        );
918
919        // Drain task: forward workflow output into the WS broadcast channel
920        // (same contract as create_session — without this the restored
921        // session would run silently).
922        let session_arc = user_sessions
923            .sessions
924            .get(chain_id)
925            .map(|e| e.session.clone())
926            .ok_or_else(|| SessionError::NotFound(chain_id.to_string()))
927            .ok()?;
928        let handle = self.spawn_user_drain_task(
929            chain_id.to_string(),
930            session_arc.clone(),
931            ws_tx_entry.clone(),
932            workflow_rx,
933        );
934        // Store the abort handle (a3e1f3b6) so destroy_session/eviction can
935        // close the channel — the drain task holds a ws_tx sender + the
936        // session Arc.
937        if let Some(mut e) = user_sessions.sessions.get_mut(chain_id) {
938            e.drain_abort = Some(handle.abort_handle());
939        }
940
941        Some((chain_id.to_string(), session_arc, ws_tx_entry))
942    }
943
944    /// List all active sessions for a user, sorted by last_active desc.
945    pub async fn list_sessions(&self, user_key: &str) -> Vec<SessionListItem> {
946        let Some(user_sessions) = self.sessions.get(user_key) else {
947            return Vec::new();
948        };
949
950        let mut items = Vec::new();
951        for entry in user_sessions.sessions.iter() {
952            let session = entry.session.lock().await;
953            let workflow_state = match session.workflow_state {
954                trustee_core::types::WorkflowState::Idle => "Idle",
955                trustee_core::types::WorkflowState::Running => "Running",
956                trustee_core::types::WorkflowState::Cancelling => "Cancelling",
957            };
958            let last_active = entry.last_active.lock().await;
959            items.push(SessionListItem {
960                session_id: entry.key().clone(),
961                session_name: session.session_name.clone(),
962                workflow_state: workflow_state.to_string(),
963                created_at: entry.created_at.to_rfc3339(),
964                last_active: last_active.to_rfc3339(),
965                handoff_count: session.handoff_count,
966            });
967        }
968        drop(user_sessions);
969
970        // Sort by last_active descending
971        items.sort_by(|a, b| b.last_active.cmp(&a.last_active));
972        items
973    }
974
975    /// Destroy a session. The session must be Idle.
976    pub async fn destroy_session(
977        &self,
978        user_key: &str,
979        session_id: &str,
980    ) -> Result<(), SessionError> {
981        let user_sessions = self
982            .sessions
983            .get(user_key)
984            .ok_or_else(|| SessionError::NotFound(session_id.to_string()))?;
985
986        // Check workflow state before removing; capture the on-disk chain
987        // id (if any) so destroy can tombstone it against later restores.
988        let chain_id_for_tombstone = {
989            let entry = user_sessions
990                .sessions
991                .get(session_id)
992                .ok_or_else(|| SessionError::NotFound(session_id.to_string()))?;
993            let session = entry.session.lock().await;
994            if session.workflow_state != trustee_core::types::WorkflowState::Idle {
995                let state_str = match session.workflow_state {
996                    trustee_core::types::WorkflowState::Running => "Running",
997                    trustee_core::types::WorkflowState::Cancelling => "Cancelling",
998                    _ => "Unknown",
999                };
1000                return Err(SessionError::NotIdle(state_str.to_string()));
1001            }
1002            session.session_id.clone()
1003        };
1004
1005        // Abort the drain task FIRST (a3e1f3b6): it holds the ORIGINAL ws_tx
1006        // sender + the session Arc — without the abort the broadcast channel
1007        // never closes (the arc cycle: the drain task keeps the session
1008        // alive, the session keeps workflow_tx alive, workflow_rx never
1009        // closes) and the connected clients' WS streams stay open (no
1010        // onclose — the console's 'Session ended' detection never fires)
1011        // while the destroyed session's memory leaks. Destroy only happens
1012        // on an Idle session (the check above) so no workflow is in flight —
1013        // the abort is safe.
1014        if let Some(mut entry) = user_sessions.sessions.get_mut(session_id) {
1015            if let Some(handle) = entry.drain_abort.take() {
1016                handle.abort();
1017            }
1018        }
1019
1020        // Remove from DashMap
1021        user_sessions.sessions.remove(session_id);
1022
1023        // If the destroyed live session was attached to an on-disk chain,
1024        // tombstone it so a later get_or_restore lookup does not resurrect
1025        // a session the user explicitly terminated.
1026        if let Some(chain_id) = chain_id_for_tombstone {
1027            self.write_tombstone(user_key, &chain_id);
1028        }
1029
1030        Ok(())
1031    }
1032
1033    // -----------------------------------------------------------------------
1034    // Read-only helpers (no session creation side effects)
1035    // -----------------------------------------------------------------------
1036
1037    /// Resolve a user's home_dir without creating an in-memory session.
1038    ///
1039    /// This is the read-only equivalent of the isolation logic in
1040    /// `apply_user_isolation`. Used by endpoints that only need to read
1041    /// checkpoint data from disk (history, session list, session detail)
1042    /// and must NOT create ghost sessions as a side effect.
1043    pub fn get_user_home_dir(&self, user_key: &str) -> Option<std::path::PathBuf> {
1044        // TRUSTEE_CLASSIC_HOME=1 (Android single-user loopback, Hamrah
1045        // 2026-10-26): use the CLASSIC home — the same one the command path
1046        // and the CLI write to — so the session list/resume/history see the
1047        // real sessions instead of an empty users/{hash}/ namespace.
1048        //
1049        // LEAK GUARD (Ravand 2026-10-27 review): under this gate EVERY user
1050        // shares the classic home — per-user checkpoint isolation is OFF.
1051        // That is correct ONLY for the single-user loopback. If the env is
1052        // set on a multi-user server by accident, all users would see each
1053        // other's checkpoint chains — say so LOUD, once, at first use.
1054        if std::env::var("TRUSTEE_CLASSIC_HOME").as_deref() == Ok("1") {
1055            static GATE_WARNED: std::sync::Once = std::sync::Once::new();
1056            GATE_WARNED.call_once(|| {
1057                tracing::warn!(
1058                    "TRUSTEE_CLASSIC_HOME=1 — ALL users share the classic home \
1059                     (~/.trustee); per-user checkpoint isolation is OFF. \
1060                     Single-user loopback ONLY — unset this env on multi-user servers."
1061                );
1062            });
1063            return dirs::home_dir().map(|home| home.join(".trustee"));
1064        }
1065        let user_key = effective_user_key(user_key);
1066        let hash = trustee_core::user_hash(&user_key);
1067        dirs::home_dir().map(|home| home.join(".trustee").join("users").join(&hash))
1068    }
1069
1070    /// Resolve config_toml and home_dir without creating an in-memory session.
1071    ///
1072    /// Returns `(config_toml, home_dir)`. If config is not loaded,
1073    /// config_toml will be None.
1074    pub fn get_user_config_and_home(
1075        &self,
1076        user_key: &str,
1077    ) -> (Option<String>, Option<std::path::PathBuf>) {
1078        (self.config_toml.clone(), self.get_user_home_dir(user_key))
1079    }
1080
1081    // -----------------------------------------------------------------------
1082    // Private helpers
1083    // -----------------------------------------------------------------------
1084
1085    /// Apply per-user isolation: SHA-256 hash → home_dir + project_id.
1086    ///
1087    /// Hashing goes through the single consolidated [`trustee_core::user_hash`]
1088    /// so the web path and the CLI path can never drift apart.
1089    fn apply_user_isolation(&self, session: &mut Session, user_key: &str) {
1090        // TRUSTEE_CLASSIC_HOME=1 (Android single-user loopback): the classic
1091        // home — the per-user isolation only applies in multi-user web mode.
1092        if std::env::var("TRUSTEE_CLASSIC_HOME").as_deref() == Ok("1") {
1093            if let Some(home) = dirs::home_dir() {
1094                session.home_dir = Some(home.join(".trustee"));
1095            }
1096            return;
1097        }
1098        let user_key = effective_user_key(user_key);
1099        let user_hash = trustee_core::user_hash(&user_key);
1100
1101        // Set per-user home directory for checkpoint isolation
1102        let user_home = if let Some(home) = dirs::home_dir() {
1103            let user_home = home.join(".trustee").join("users").join(&user_hash);
1104            session.home_dir = Some(user_home.clone());
1105            Some(user_home)
1106        } else {
1107            None
1108        };
1109
1110        // 92a36073 boot-surface: a MATERIALIZED namespace without per-user
1111        // credentials is the shadowing scenario — say so NOW, not in
1112        // forensics after a global token answered for it.
1113        if let Some(ref user_home) = user_home {
1114            if materialized_user_missing_env(user_home) {
1115                tracing::warn!(
1116                    "user namespace {} is materialized but has NO per-user .env — \
1117                     ${{…}} references will fall through to process env \
1118                     (global ~/.trustee/.env) and execute under the GLOBAL principal",
1119                    &user_hash[..8.min(user_hash.len())]
1120                );
1121            }
1122        }
1123
1124        session.project_id = Some(format!("web{}", &user_hash[..16]));
1125
1126        // ── Per-user .env (Task 2) ──────────────────────────────────────
1127        //
1128        // Load per-user secrets from ~/.trustee/users/{hash}/.env
1129        // These are merged on top of shared secrets (per-user wins).
1130        // They are NEVER set as process env vars — used only for ${VAR}
1131        // substitution in the config TOML below.
1132        let shared_secrets = session.secrets.clone().unwrap_or_default();
1133        let mut merged_secrets = shared_secrets.clone();
1134
1135        if let Some(ref user_home) = user_home {
1136            if let Ok(merged) = self.load_user_secrets(user_home, &merged_secrets) {
1137                merged_secrets = merged;
1138            }
1139        }
1140
1141        // ── Per-user config overlay (Task 3) ────────────────────────────
1142        if let Some(ref user_home) = user_home {
1143            if let Some(merged) = self.merge_user_config(user_home, session.config_toml.as_deref())
1144            {
1145                session.config_toml = Some(merged);
1146                tracing::debug!("Merged per-user config into session");
1147            }
1148        }
1149
1150        // ── ${VAR} substitution (Task 4) ────────────────────────────────
1151        if let Some(ref mut config_toml) = session.config_toml {
1152            substitute_env_vars(
1153                config_toml,
1154                &merged_secrets,
1155                &format!("user {}", &user_hash[..8.min(user_hash.len())]),
1156            );
1157        }
1158
1159        // ── Strip per-user secrets (Task 5) ────────────────────────────
1160        session.secrets = Some(shared_secrets);
1161    }
1162
1163    /// Load secrets from a user's ~/.trustee/users/{hash}/.env and merge
1164    /// on top of `base` (user wins). Returns the merged map.
1165    fn load_user_secrets(
1166        &self,
1167        user_home: &std::path::Path,
1168        base: &std::collections::HashMap<String, String>,
1169    ) -> std::io::Result<std::collections::HashMap<String, String>> {
1170        let user_env_path = user_home.join(".env");
1171        if !user_env_path.exists() {
1172            return Ok(base.clone());
1173        }
1174        let content = std::fs::read_to_string(&user_env_path)?;
1175        let mut merged = base.clone();
1176        for line in content.lines() {
1177            let line = line.trim();
1178            if line.is_empty() || line.starts_with('#') {
1179                continue;
1180            }
1181            if let Some((key, value)) = line.split_once('=') {
1182                let key = key.trim().to_string();
1183                let value = value
1184                    .trim()
1185                    .trim_matches('"')
1186                    .trim_matches('\'')
1187                    .to_string();
1188                merged.insert(key, value);
1189            }
1190        }
1191        tracing::debug!("Loaded per-user secrets from {}", user_env_path.display());
1192        Ok(merged)
1193    }
1194
1195    /// Deep-merge a user's per-user config overlay on top of the shared
1196    /// config. Returns the merged TOML string, or None if no overlay exists.
1197    ///
1198    /// Overlay allowlist (task 16B): only allowlisted top-level sections of
1199    /// the user overlay are merged; anything else is dropped loudly.
1200    /// - `[mcp]` is always allowed — per-user MCP tool sets are the point of
1201    ///   the overlay convention.
1202    /// - `[llm]` is allowed only when the instance opted in via the
1203    ///   `[users].allow_llm_overlay` knob (default **false**).
1204    ///
1205    /// Everything else (`[server]`, `[auth]`, `[storage]`, `[web]`, …) is
1206    /// boot-time, instance-level config and must not be rewritable through a
1207    /// per-user file. This is predictability hardening, not a security
1208    /// boundary: those sections were never re-read from session config.
1209    fn merge_user_config(
1210        &self,
1211        user_home: &std::path::Path,
1212        shared_config: Option<&str>,
1213    ) -> Option<String> {
1214        let user_config_path = user_home.join("config").join("trustee.toml");
1215        if !user_config_path.exists() {
1216            return None;
1217        }
1218        let user_config_toml = std::fs::read_to_string(&user_config_path).ok()?;
1219        let shared = shared_config.unwrap_or("").parse::<toml::Value>().ok()?;
1220        let overlay = user_config_toml.parse::<toml::Value>().ok()?;
1221
1222        let allowed = |section: &str| {
1223            section == "mcp"
1224                // 16E: [thq] is per-user identity config (read from the overlay
1225                // file at boot); keeping it in the merged session config is
1226                // inert but avoids a false "dropping" warn on every dispatch.
1227                || section == "thq"
1228                // xagent personas: an agent's [lifecycle].system_template IS
1229                // her identity — user-owned by design. The WASM extension
1230                // switch ([lifecycle].enabled) is stripped below: persona is
1231                // user-owned, extension control is not.
1232                || section == "lifecycle"
1233                || (self.allow_llm_overlay && section == "llm")
1234        };
1235        // Mask the user in logs: user_home's dir name IS the hash — never
1236        // log the raw key (keys may be emails). Log a short hash prefix.
1237        let dir_name = user_home
1238            .file_name()
1239            .and_then(|n| n.to_str())
1240            .unwrap_or("<unknown>");
1241        let masked_user = dir_name.get(..8).unwrap_or(dir_name);
1242
1243        let mut filtered_overlay = toml::map::Map::new();
1244        if let Some(table) = overlay.as_table() {
1245            for (section, value) in table {
1246                if section == "lifecycle" {
1247                    let mut v = value.clone();
1248                    if let Some(t) = v.as_table_mut() {
1249                        if t.remove("enabled").is_some() {
1250                            tracing::info!(
1251                                "user config overlay: [lifecycle].enabled is not user-settable — stripped for user {}",
1252                                masked_user
1253                            );
1254                        }
1255                    }
1256                    filtered_overlay.insert(section.clone(), v);
1257                } else if allowed(section) {
1258                    filtered_overlay.insert(section.clone(), value.clone());
1259                } else {
1260                    tracing::warn!(
1261                        "user config overlay: dropping non-allowlisted section [{}] for user {}",
1262                        section,
1263                        masked_user
1264                    );
1265                }
1266            }
1267        }
1268
1269        if filtered_overlay.is_empty() {
1270            // Nothing survived the allowlist — no-op, keep shared config as-is.
1271            return None;
1272        }
1273        let overlay = toml::Value::Table(filtered_overlay);
1274
1275        let mut shared = shared;
1276        deep_merge_toml(&mut shared, &overlay);
1277        let merged = toml::to_string(&shared).ok()?;
1278        tracing::debug!("Merged per-user config from {}", user_config_path.display());
1279        Some(merged)
1280    }
1281
1282    /// Resolve the fully-merged config TOML for a user WITHOUT creating a session.
1283    ///
1284    /// This is the read-only equivalent of the config resolution in
1285    /// `apply_user_isolation`: shared config + per-user overlay + ${VAR}
1286    /// substitution. Used by endpoints that need to inspect config (e.g.
1287    /// listing available LLM models) without spawning a ghost session.
1288    ///
1289    /// Returns None if no shared config is loaded.
1290    pub fn resolve_user_config(&self, user_key: &str) -> Option<String> {
1291        let config_toml = self.config_toml.clone()?;
1292
1293        // Resolve user home dir (same hash scheme as apply_user_isolation)
1294        let user_home = self.get_user_home_dir(user_key)?;
1295
1296        // Start from shared secrets; merge per-user .env on top
1297        let mut merged_secrets = self.secrets.clone().unwrap_or_default();
1298        if let Ok(merged) = self.load_user_secrets(&user_home, &merged_secrets) {
1299            merged_secrets = merged;
1300        }
1301
1302        // Merge per-user config overlay
1303        let mut resolved = config_toml;
1304        if let Some(merged) = self.merge_user_config(&user_home, Some(&resolved)) {
1305            resolved = merged;
1306        }
1307
1308        // Substitute ${VAR} from merged secrets
1309        substitute_env_vars(
1310            &mut resolved,
1311            &merged_secrets,
1312            &format!(
1313                "resolve_user_config user {}",
1314                &trustee_core::user_hash(user_key)[..8]
1315            ),
1316        );
1317
1318        Some(resolved)
1319    }
1320
1321    /// Get or build the per-user MCP tool loader (16C).
1322    ///
1323    /// Cache semantics:
1324    /// - fingerprint = content hash of the effective `[mcp]` section
1325    ///   (shared + allowlist-filtered overlay + ${VAR} substitution, i.e.
1326    ///   exactly what `resolve_user_config` produces) — stale on ANY change.
1327    /// - hit + match → `Arc` clone, zero network I/O.
1328    /// - miss/stale → single-flight build (one build per user at a time;
1329    ///   late arrivals re-check and reuse the winner's entry).
1330    /// - `Ok(None)` = MCP disabled for this user → agent runs MCP-less via
1331    ///   abk `McpSource::Prebuilt(None)` (same semantics as
1332    ///   `[mcp] enabled = false`, no per-task re-evaluation).
1333    /// - build failure → cached degraded entry with
1334    ///   [`MCP_BUILD_RETRY_BACKOFF`]; the error is returned so THIS user's
1335    ///   dispatch fails loud while other users are unaffected.
1336    pub async fn get_or_build_mcp_loader(
1337        &self,
1338        user_key: &str,
1339        token_store: &Arc<pep::MemoryTokenStore>,
1340    ) -> Result<Option<std::sync::Arc<abk::agent::McpToolLoader>>, String> {
1341        let user_key = effective_user_key(user_key);
1342        let user_hash = trustee_core::user_hash(&user_key);
1343
1344        // Effective per-user config — the fingerprint source of truth.
1345        let resolved = self.resolve_user_config(&user_key);
1346        let fingerprint = fingerprint_mcp_section(resolved.as_deref());
1347
1348        // Fast path: non-degraded entry that is fresh AND content-current.
1349        if let Some(entry) = self.mcp_loaders.get(&user_hash) {
1350            match loader_freshness(
1351                entry.fingerprint,
1352                entry.built_at,
1353                entry.degraded.as_deref(),
1354                entry.failed_at,
1355                fingerprint,
1356                chrono::Utc::now(),
1357            ) {
1358                LoaderFreshness::Fresh => return Ok(entry.loader.clone()),
1359                LoaderFreshness::ConfigChanged => {
1360                    // fall through to the rebuild path below
1361                }
1362                LoaderFreshness::AgedOut => {
1363                    // 42131deb: the served surface may have changed behind
1364                    // a byte-identical config — rebuild (re-lists tools).
1365                    tracing::info!(
1366                        "MCP toolset for user {} is {}s old (max {}s) — rebuilding \
1367                         to re-list the served surface (phantom-tool guard)",
1368                        &user_hash[..8.min(user_hash.len())],
1369                        (chrono::Utc::now() - entry.built_at).num_seconds(),
1370                        MCP_TOOLSET_MAX_AGE.as_secs()
1371                    );
1372                }
1373                LoaderFreshness::Backoff(remaining) => {
1374                    let err = entry.degraded.clone().unwrap_or_else(|| {
1375                        "previous MCP loader build failed; retry after backoff".to_string()
1376                    });
1377                    let _ = remaining;
1378                    return Err(err);
1379                }
1380            }
1381        }
1382
1383        // Single-flight: one build per user at a time.
1384        let lock = self
1385            .mcp_build_locks
1386            .entry(user_hash.clone())
1387            .or_insert_with(|| Arc::new(tokio::sync::Mutex::new(())))
1388            .clone();
1389        let _guard = lock.lock().await;
1390
1391        // Double-check: another task may have built while we waited.
1392        if let Some(entry) = self.mcp_loaders.get(&user_hash) {
1393            if loader_freshness(
1394                entry.fingerprint,
1395                entry.built_at,
1396                entry.degraded.as_deref(),
1397                entry.failed_at,
1398                fingerprint,
1399                chrono::Utc::now(),
1400            ) == LoaderFreshness::Fresh
1401            {
1402                return Ok(entry.loader.clone());
1403            }
1404        }
1405
1406        match self
1407            .build_mcp_loader(&user_hash, resolved.as_deref(), fingerprint, token_store)
1408            .await
1409        {
1410            Ok(entry) => {
1411                self.mcp_loaders.insert(user_hash.clone(), entry);
1412                Ok(self.mcp_loaders.get(&user_hash).unwrap().loader.clone())
1413            }
1414            Err(err) => {
1415                tracing::warn!(
1416                    "MCP loader build FAILED for user {}; dispatch fails loud, retry after {:?}",
1417                    &user_hash[..8.min(user_hash.len())],
1418                    MCP_BUILD_RETRY_BACKOFF
1419                );
1420                self.mcp_loaders.insert(
1421                    user_hash,
1422                    McpLoaderEntry {
1423                        loader: None,
1424                        fingerprint,
1425                        built_at: chrono::Utc::now(),
1426                        degraded: Some(err.clone()),
1427                        failed_at: Some(chrono::Utc::now()),
1428                    },
1429                );
1430                Err(err)
1431            }
1432        }
1433    }
1434
1435    /// Build a loader entry from the effective config. No caching here —
1436    /// the caller owns insertion and degraded handling.
1437    async fn build_mcp_loader(
1438        &self,
1439        user_hash: &str,
1440        resolved: Option<&str>,
1441        fingerprint: u64,
1442        token_store: &Arc<pep::MemoryTokenStore>,
1443    ) -> Result<McpLoaderEntry, String> {
1444        let mcp_config: Option<abk::config::McpConfig> = match resolved {
1445            Some(toml_str) => {
1446                let value = toml_str
1447                    .parse::<toml::Value>()
1448                    .map_err(|e| format!("config parse failed: {}", e))?;
1449                match value.get("mcp") {
1450                    Some(section) => {
1451                        // 0.19.6 (incident fc94aea9, T3): a leftover ${…} in
1452                        // the effective [mcp] section means the credential
1453                        // reference is UNRESOLVED — the literal placeholder
1454                        // must never reach an agent (it once rode to
1455                        // production auth as the Bearer). Fail the build
1456                        // loud; the degraded path handles retry/backoff.
1457                        let mut leftovers: Vec<String> = Vec::new();
1458                        collect_unresolved_placeholders(section, &mut leftovers);
1459                        if !leftovers.is_empty() {
1460                            return Err(format!(
1461                                "unresolved {} placeholder(s) in [mcp]: {} — supply the \
1462                                 exact-case key in the profile secrets",
1463                                leftovers.len(),
1464                                leftovers.join(", ")
1465                            ));
1466                        }
1467                        use serde::Deserialize as _;
1468                        Some(
1469                            abk::config::McpConfig::deserialize(section.clone())
1470                                .map_err(|e| format!("invalid [mcp] config: {}", e))?,
1471                        )
1472                    }
1473                    None => None,
1474                }
1475            }
1476            None => None,
1477        };
1478
1479        let loader = match mcp_config {
1480            Some(cfg) if cfg.enabled => {
1481                let built = abk::agent::McpToolLoader::with_token_store(
1482                    &cfg,
1483                    Some(token_store.clone() as std::sync::Arc<dyn pep::token_store::TokenStore>),
1484                )
1485                .await
1486                .map_err(|e| format!("MCP loader build failed: {}", e))?;
1487
1488                // THE parity-evidence log line (16C acceptance + migration task):
1489                // one INFO line per build; a task loop that reuses the cache
1490                // shows exactly one line per user per fingerprint.
1491                let servers: Vec<String> = built
1492                    .server_statuses
1493                    .iter()
1494                    .map(|s| {
1495                        if s.connected {
1496                            format!("{}(up,{}tools)", s.name, s.tool_count)
1497                        } else {
1498                            format!("{}(DOWN)", s.name)
1499                        }
1500                    })
1501                    .collect();
1502                tracing::info!(
1503                    "MCP loader built for user {}: servers=[{}] total_tools={}",
1504                    &user_hash[..8.min(user_hash.len())],
1505                    servers.join(", "),
1506                    built.tool_count
1507                );
1508                Some(std::sync::Arc::new(built))
1509            }
1510            _ => None,
1511        };
1512
1513        Ok(McpLoaderEntry {
1514            loader,
1515            fingerprint,
1516            built_at: chrono::Utc::now(),
1517            degraded: None,
1518            failed_at: None,
1519        })
1520    }
1521
1522    /// Spawn a background drain task for a specific session's workflow receiver.
1523    fn spawn_user_drain_task(
1524        &self,
1525        session_id: String,
1526        session: Arc<Mutex<Session>>,
1527        ws_tx: broadcast::Sender<String>,
1528        mut workflow_rx: mpsc::UnboundedReceiver<TuiMessage>,
1529    ) -> tokio::task::JoinHandle<()> {
1530        // Bug 6: broadcast StateChanged only on actual transitions. Without
1531        // this, every WS message was accompanied by a duplicate StateChanged,
1532        // doubling traffic and re-triggering frontend updateState on hot
1533        // StreamDelta/ReasoningDelta bursts. New subscribers learn the current
1534        // state from the WS snapshot, so the initial None is safe.
1535        let mut last_broadcast_state: Option<String> = None;
1536        tokio::spawn(async move {
1537            while let Some(msg) = workflow_rx.recv().await {
1538                {
1539                    let mut session = session.lock().await;
1540                    session.handle_workflow_message(msg.clone());
1541
1542                    let state_str = match session.workflow_state {
1543                        trustee_core::types::WorkflowState::Idle => "Idle",
1544                        trustee_core::types::WorkflowState::Running => "Running",
1545                        trustee_core::types::WorkflowState::Cancelling => "Cancelling",
1546                    };
1547                    if last_broadcast_state.as_deref() != Some(state_str) {
1548                        last_broadcast_state = Some(state_str.to_string());
1549                        let state_msg = serde_json::json!({
1550                            "type": "StateChanged",
1551                            "state": state_str
1552                        });
1553                        let _ = ws_tx.send(state_msg.to_string());
1554                    }
1555                }
1556
1557                let json = serde_json::to_string(&SerializableMessage(&msg)).unwrap_or_default();
1558                let _ = ws_tx.send(json);
1559            }
1560            tracing::debug!("Drain task ended for session: {}", session_id);
1561        })
1562    }
1563
1564    /// Spawn the default user's drain task (backward compatibility).
1565    /// Called during server startup for the initial session.
1566    pub fn spawn_drain_task(self, mut workflow_rx: mpsc::UnboundedReceiver<TuiMessage>) {
1567        // Get the default user's first session
1568        let default_user = self
1569            .sessions
1570            .get("default")
1571            .expect("default user must exist");
1572        let first_entry = default_user
1573            .sessions
1574            .iter()
1575            .next()
1576            .expect("default user must have at least one session");
1577        let session = first_entry.session.clone();
1578        let ws_tx = first_entry.ws_tx.clone();
1579        let session_id = first_entry.key().clone();
1580        drop(first_entry);
1581        drop(default_user);
1582
1583        // Bug 6: transition-only StateChanged broadcasts (see
1584        // spawn_user_drain_task). The task starts in Running state because
1585        // spawn_drain_task is only called after a command began executing.
1586        let mut last_broadcast_state = Some("Running".to_string());
1587        let session_id_for_handle = session_id.clone();
1588        let handle = tokio::spawn(async move {
1589            while let Some(msg) = workflow_rx.recv().await {
1590                {
1591                    let mut session = session.lock().await;
1592                    session.handle_workflow_message(msg.clone());
1593
1594                    let state_str = match session.workflow_state {
1595                        trustee_core::types::WorkflowState::Idle => "Idle",
1596                        trustee_core::types::WorkflowState::Running => "Running",
1597                        trustee_core::types::WorkflowState::Cancelling => "Cancelling",
1598                    };
1599                    if last_broadcast_state.as_deref() != Some(state_str) {
1600                        last_broadcast_state = Some(state_str.to_string());
1601                        let state_msg = serde_json::json!({
1602                            "type": "StateChanged",
1603                            "state": state_str
1604                        });
1605                        let _ = ws_tx.send(state_msg.to_string());
1606                    }
1607                }
1608
1609                let json = serde_json::to_string(&SerializableMessage(&msg)).unwrap_or_default();
1610                let _ = ws_tx.send(json);
1611            }
1612            tracing::debug!("Drain task ended for session: {}", session_id);
1613        });
1614        // Store the abort handle (a3e1f3b6) so destroy_session can close the
1615        // channel — the startup drain task holds a ws_tx clone + the session
1616        // Arc (the same arc cycle as spawn_user_drain_task).
1617        if let Some(default_user) = self.sessions.get("default") {
1618            if let Some(mut e) = default_user.sessions.get_mut(&session_id_for_handle) {
1619                e.drain_abort = Some(handle.abort_handle());
1620            }
1621        }
1622    }
1623
1624    /// Resolve the user key from request headers.
1625    pub async fn resolve_user_key(&self, headers: &axum::http::HeaderMap) -> String {
1626        let Some(ref auth) = self.auth else {
1627            return "default".to_string();
1628        };
1629
1630        // Try Bearer header first
1631        if let Some(token) = headers
1632            .get(axum::http::header::AUTHORIZATION)
1633            .and_then(|v| v.to_str().ok())
1634            .and_then(|v| v.strip_prefix("Bearer "))
1635            .map(|s| s.to_string())
1636        {
1637            if token.starts_with("dev:") {
1638                let parts: Vec<&str> = token.splitn(4, ':').collect();
1639                if parts.len() >= 4 {
1640                    return format!("dev:{}", parts[1]);
1641                }
1642            }
1643            if let Ok(claims) = auth.validate_token(&token).await {
1644                return claims.sub;
1645            }
1646        }
1647
1648        // Try cookie
1649        let cookie_session_id = headers
1650            .get(axum::http::header::COOKIE)
1651            .and_then(|v| v.to_str().ok())
1652            .and_then(|cookies| {
1653                cookies.split(';').map(|c| c.trim()).find_map(|c| {
1654                    c.strip_prefix(&format!("{}=", auth.config.cookie_name))
1655                        .map(|s| s.to_string())
1656                })
1657            });
1658
1659        if let Some(session_id) = cookie_session_id {
1660            if session_id.starts_with("dev:") {
1661                let parts: Vec<&str> = session_id.splitn(4, ':').collect();
1662                if parts.len() >= 4 {
1663                    return format!("dev:{}", parts[1]);
1664                }
1665            }
1666
1667            if let Ok(access_token) = auth.session_manager.get_token(&session_id).await {
1668                if let Ok(claims) = auth.validate_token(&access_token).await {
1669                    return claims.sub;
1670                }
1671            }
1672        }
1673
1674        "default".to_string()
1675    }
1676}
1677
1678// ---------------------------------------------------------------------------
1679// SerializableMessage (unchanged)
1680// ---------------------------------------------------------------------------
1681
1682/// Wrapper to serialize `TuiMessage` as JSON with a `type` discriminator.
1683struct SerializableMessage<'a>(&'a TuiMessage);
1684
1685impl<'a> serde::Serialize for SerializableMessage<'a> {
1686    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
1687    where
1688        S: serde::Serializer,
1689    {
1690        use serde::ser::SerializeStruct;
1691
1692        match self.0 {
1693            TuiMessage::OutputLine(line) => {
1694                let mut s = serializer.serialize_struct("msg", 2)?;
1695                s.serialize_field("type", "OutputLine")?;
1696                s.serialize_field("line", line)?;
1697                s.end()
1698            }
1699            TuiMessage::StreamDelta(delta) => {
1700                let mut s = serializer.serialize_struct("msg", 2)?;
1701                s.serialize_field("type", "StreamDelta")?;
1702                s.serialize_field("delta", delta)?;
1703                s.end()
1704            }
1705            TuiMessage::ReasoningDelta(delta) => {
1706                let mut s = serializer.serialize_struct("msg", 2)?;
1707                s.serialize_field("type", "ReasoningDelta")?;
1708                s.serialize_field("delta", delta)?;
1709                s.end()
1710            }
1711            TuiMessage::WorkflowCompleted => {
1712                let mut s = serializer.serialize_struct("msg", 2)?;
1713                s.serialize_field("type", "WorkflowCompleted")?;
1714                s.serialize_field("state", "Idle")?;
1715                s.end()
1716            }
1717            TuiMessage::WorkflowError(err) => {
1718                let mut s = serializer.serialize_struct("msg", 2)?;
1719                s.serialize_field("type", "WorkflowError")?;
1720                s.serialize_field("error", err)?;
1721                s.end()
1722            }
1723            TuiMessage::ResumeInfo(info) => match info {
1724                Some(ri) => {
1725                    let mut s = serializer.serialize_struct("msg", 5)?;
1726                    s.serialize_field("type", "ResumeInfo")?;
1727                    s.serialize_field("state", "Idle")?;
1728                    s.serialize_field("session_id", &ri.session_id)?;
1729                    s.serialize_field("checkpoint_id", &ri.checkpoint_id)?;
1730                    s.serialize_field("iteration", &ri.iteration)?;
1731                    s.end()
1732                }
1733                None => {
1734                    let mut s = serializer.serialize_struct("msg", 2)?;
1735                    s.serialize_field("type", "ResumeInfo")?;
1736                    s.serialize_field("state", "Idle")?;
1737                    s.end()
1738                }
1739            },
1740            TuiMessage::TodoUpdate(content) => {
1741                let mut s = serializer.serialize_struct("msg", 2)?;
1742                s.serialize_field("type", "TodoUpdate")?;
1743                s.serialize_field("content", content)?;
1744                s.end()
1745            }
1746            TuiMessage::WorkflowCancelled => {
1747                let mut s = serializer.serialize_struct("msg", 2)?;
1748                s.serialize_field("type", "WorkflowCancelled")?;
1749                s.serialize_field("state", "Idle")?;
1750                s.end()
1751            }
1752            TuiMessage::HandoffReady(briefing) => {
1753                let mut s = serializer.serialize_struct("msg", 3)?;
1754                s.serialize_field("type", "HandoffReady")?;
1755                s.serialize_field("state", "Idle")?;
1756                s.serialize_field("briefing", briefing)?;
1757                s.end()
1758            }
1759            TuiMessage::SessionRotated { old, new } => {
1760                let mut s = serializer.serialize_struct("msg", 3)?;
1761                s.serialize_field("type", "SessionRotated")?;
1762                s.serialize_field("old", old)?;
1763                s.serialize_field("new", new)?;
1764                s.end()
1765            }
1766            TuiMessage::HandoffFailed => {
1767                let mut s = serializer.serialize_struct("msg", 2)?;
1768                s.serialize_field("type", "HandoffFailed")?;
1769                s.serialize_field("state", "Idle")?;
1770                s.end()
1771            }
1772            TuiMessage::ToolPending { tool_name, hint } => {
1773                let mut s = serializer.serialize_struct("msg", 3)?;
1774                s.serialize_field("type", "ToolPending")?;
1775                s.serialize_field("tool_name", tool_name)?;
1776                s.serialize_field("hint", hint)?;
1777                s.end()
1778            }
1779            TuiMessage::ToolDone {
1780                tool_name,
1781                success,
1782                hint,
1783            } => {
1784                let mut s = serializer.serialize_struct("msg", 4)?;
1785                s.serialize_field("type", "ToolDone")?;
1786                s.serialize_field("tool_name", tool_name)?;
1787                s.serialize_field("success", success)?;
1788                s.serialize_field("hint", hint)?;
1789                s.end()
1790            }
1791            TuiMessage::ContextTokensUpdated(count) => {
1792                let mut s = serializer.serialize_struct("msg", 2)?;
1793                s.serialize_field("type", "ContextTokensUpdated")?;
1794                s.serialize_field("count", count)?;
1795                s.end()
1796            }
1797            TuiMessage::McpServerStatus {
1798                name,
1799                connected,
1800                tool_count,
1801                error,
1802            } => {
1803                let mut s = serializer.serialize_struct("msg", 5)?;
1804                s.serialize_field("type", "McpServerStatus")?;
1805                s.serialize_field("name", name)?;
1806                s.serialize_field("connected", connected)?;
1807                s.serialize_field("tool_count", tool_count)?;
1808                s.serialize_field("error", error)?;
1809                s.end()
1810            }
1811            TuiMessage::SessionTitleUpdated(title) => {
1812                let mut s = serializer.serialize_struct("msg", 2)?;
1813                s.serialize_field("type", "SessionTitleUpdated")?;
1814                s.serialize_field("title", title)?;
1815                s.end()
1816            }
1817        }
1818    }
1819}
1820
1821// ---------------------------------------------------------------------------
1822// Per-user config helpers
1823// ---------------------------------------------------------------------------
1824
1825/// Deep-merge a TOML overlay on top of a base value (in-place).
1826///
1827/// - Tables: recursively merge key-by-key (overlay wins on conflict).
1828/// - Arrays: overlay replaces base entirely (no merging).
1829/// - Scalars: overlay replaces base.
1830/// - If a key exists in overlay but not base, it's added.
1831fn deep_merge_toml(base: &mut toml::Value, overlay: &toml::Value) {
1832    match (base, overlay) {
1833        (toml::Value::Table(base_table), toml::Value::Table(overlay_table)) => {
1834            for (key, overlay_val) in overlay_table {
1835                match base_table.get_mut(key) {
1836                    Some(base_val) => {
1837                        // Both exist — recurse if both are tables, else replace
1838                        deep_merge_toml(base_val, overlay_val);
1839                    }
1840                    None => {
1841                        // Key only in overlay — insert
1842                        base_table.insert(key.clone(), overlay_val.clone());
1843                    }
1844                }
1845            }
1846        }
1847        // Non-table: overlay replaces base
1848        (base, overlay) => {
1849            *base = overlay.clone();
1850        }
1851    }
1852}
1853
1854/// Replace `${VAR_NAME}` references in a string with values from a secrets map.
1855///
1856/// Falls back to process environment if the variable is not in the map.
1857/// Variables not found in either are left as-is.
1858/// Walk a TOML value tree; collect every string that still contains a
1859/// `${…}` placeholder (post-substitution leftovers — unresolved references).
1860fn collect_unresolved_placeholders(value: &toml::Value, out: &mut Vec<String>) {
1861    match value {
1862        toml::Value::String(s) => {
1863            if let Some(start) = s.find("${") {
1864                if let Some(end_rel) = s[start + 2..].find('}') {
1865                    let name = &s[start + 2..start + 2 + end_rel];
1866                    let rendered = format!("${{{name}}}");
1867                    if !out.contains(&rendered) {
1868                        out.push(rendered);
1869                    }
1870                }
1871            }
1872        }
1873        toml::Value::Table(t) => {
1874            for v in t.values() {
1875                collect_unresolved_placeholders(v, out);
1876            }
1877        }
1878        toml::Value::Array(a) => {
1879            for v in a {
1880                collect_unresolved_placeholders(v, out);
1881            }
1882        }
1883        _ => {}
1884    }
1885}
1886
1887/// Where a `${VAR}` answer came from (92a36073 credential provenance).
1888#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1889pub(crate) enum SecretSource {
1890    /// The secrets map (shared secrets + per-user `.env`, user wins) — the
1891    /// attributed, per-identity answer.
1892    SecretsMap,
1893    /// The PROCESS environment — for the trustee service that is the global
1894    /// `~/.trustee/.env` loaded at boot. When this answers for a per-user
1895    /// namespace, the per-user credential did NOT answer and a global
1896    /// credential did: actions execute under the global principal. Never
1897    /// silent — every occurrence is logged at ERROR with the variable named.
1898    ProcessEnv,
1899    /// Found nowhere; the literal `${…}` remains (the fc94aea9 guard).
1900    Unresolved,
1901}
1902
1903/// One resolved (or unresolved) variable from [`substitute_env_vars`].
1904#[derive(Debug, Clone, PartialEq, Eq)]
1905pub(crate) struct EnvVarResolution {
1906    pub name: String,
1907    pub source: SecretSource,
1908}
1909
1910fn substitute_env_vars(
1911    s: &mut String,
1912    secrets: &std::collections::HashMap<String, String>,
1913    ctx: &str,
1914) -> Vec<EnvVarResolution> {
1915    // Simple state machine: scan for ${, read until }, replace.
1916    let mut result = String::with_capacity(s.len());
1917    let bytes = s.as_bytes();
1918    let mut i = 0;
1919    // One entry per DISTINCT variable, first sighting wins (a var used
1920    // twice resolves from the same source both times).
1921    let mut report: Vec<EnvVarResolution> = Vec::new();
1922
1923    while i < bytes.len() {
1924        if i + 1 < bytes.len() && bytes[i] == b'$' && bytes[i + 1] == b'{' {
1925            // Find closing }
1926            if let Some(end) = s[i + 2..].find('}') {
1927                let var_name = &s[i + 2..i + 2 + end];
1928                // Look up in per-user secrets first, then process env
1929                let (replacement, source) = if let Some(value) = secrets.get(var_name) {
1930                    (value.clone(), SecretSource::SecretsMap)
1931                } else if let Ok(value) = std::env::var(var_name) {
1932                    (value, SecretSource::ProcessEnv)
1933                } else {
1934                    // Not found — leave as-is BUT NOT SILENTLY (incident
1935                    // fc94aea9, T3): an unresolved ${VAR} once rode to
1936                    // production authentication as the literal credential.
1937                    // Every leftover is named, every time.
1938                    (s[i..i + 2 + end + 1].to_string(), SecretSource::Unresolved)
1939                };
1940                if !report.iter().any(|r| r.name == var_name) {
1941                    report.push(EnvVarResolution {
1942                        name: var_name.to_string(),
1943                        source,
1944                    });
1945                }
1946                result.push_str(&replacement);
1947                i = i + 2 + end + 1;
1948            } else {
1949                // No closing } — copy as-is
1950                result.push('$');
1951                i += 1;
1952            }
1953        } else {
1954            result.push(bytes[i] as char);
1955            i += 1;
1956        }
1957    }
1958
1959    // 92a36073 — never-silent credential provenance. A process-env answer
1960    // for a per-user namespace is the global-token shadowing hazard: the
1961    // global principal is about to execute what the per-user identity
1962    // should have. ERROR level, EVERY occurrence (no dedup across calls).
1963    for r in &report {
1964        match r.source {
1965            SecretSource::ProcessEnv => tracing::error!(
1966                "CREDENTIAL PROVENANCE [{}]: ${{{}}} resolved from PROCESS ENV \
1967                 (global ~/.trustee/.env at boot) — no per-user/profile secret \
1968                 answered; calls using it execute under the GLOBAL principal. \
1969                 Supply the exact-case key in the per-user .env / profile \
1970                 secrets to restore attribution.",
1971                ctx,
1972                r.name
1973            ),
1974            SecretSource::Unresolved => {
1975                // Compose the leftover list for the existing loud line.
1976            }
1977            SecretSource::SecretsMap => tracing::debug!(
1978                "CREDENTIAL PROVENANCE [{}]: ${{{}}} resolved from profile/user secrets",
1979                ctx,
1980                r.name
1981            ),
1982        }
1983    }
1984    let unresolved: Vec<&str> = report
1985        .iter()
1986        .filter(|r| r.source == SecretSource::Unresolved)
1987        .map(|r| r.name.as_str())
1988        .collect();
1989    if !unresolved.is_empty() {
1990        tracing::warn!(
1991            "ENV SUBSTITUTION LEFT {} UNRESOLVED [{}]: {} — the literal ${{…}} remains in the \
1992             effective config; supply the key in the profile secrets (exact-case KEY=VALUE) \
1993             or the environment",
1994            unresolved.len(),
1995            ctx,
1996            unresolved
1997                .iter()
1998                .map(|v| format!("${{{v}}}"))
1999                .collect::<Vec<_>>()
2000                .join(", ")
2001        );
2002    }
2003
2004    *s = result;
2005    report
2006}
2007
2008/// 92a36073 boot-surface check: is a MATERIALIZED namespace missing its
2009/// per-user credentials? A directory without a `.env` is exactly the
2010/// shadowing scenario (rotated/empty home + global token downstream) —
2011/// surface it at boot, not in forensics. Pure so it stays unit-tested.
2012pub(crate) fn materialized_user_missing_env(user_home: &std::path::Path) -> bool {
2013    user_home.is_dir() && !user_home.join(".env").exists()
2014}
2015
2016// ---------------------------------------------------------------------------
2017// Tests (task 16B: overlay allowlist + consolidated user hash)
2018// ---------------------------------------------------------------------------
2019
2020/// Fingerprint the effective `[mcp]` section of a resolved config (16C).
2021///
2022/// Content hash (SHA-256, first 8 bytes as u64) — never mtime: overlays are
2023/// rewritten in place. Configs without `[mcp]` fingerprint to a stable
2024/// constant, so the disabled state caches too.
2025fn fingerprint_mcp_section(resolved: Option<&str>) -> u64 {
2026    use sha2::{Digest, Sha256};
2027    let section = resolved
2028        .and_then(|s| s.parse::<toml::Value>().ok())
2029        .and_then(|v| v.get("mcp").cloned());
2030    let bytes = match section {
2031        Some(v) => v.to_string().into_bytes(),
2032        None => b"<no-mcp>".to_vec(),
2033    };
2034    let digest = Sha256::digest(&bytes);
2035    u64::from_be_bytes(digest[..8].try_into().expect("sha256 digest >= 8 bytes"))
2036}
2037
2038#[cfg(test)]
2039mod tests {
2040    use super::*;
2041
2042    /// Default-knob ServerState for overlay tests.
2043    fn test_state() -> ServerState {
2044        let (session, _rx) = Session::new();
2045        let (ws_tx, _ws_rx) = tokio::sync::broadcast::channel::<String>(16);
2046        ServerState::new(session, ws_tx, None)
2047    }
2048
2049    #[tokio::test]
2050    async fn dispatch_scope_pins_user_key_resolution() {
2051        // THE 16F pin (ec3e0622 companion): inside the dispatch scope, EVERY
2052        // user-key-derived resolution lands in the binding's home — the
2053        // caller's own key (and its claims) become irrelevant.
2054        let state = test_state();
2055        let caller = "some-human-sub";
2056        let plain_home = state.get_user_home_dir(caller).unwrap();
2057        let pinned_key = "prof-1234";
2058        let pinned_home = in_dispatch_scope(pinned_key, async {
2059            state.get_user_home_dir(caller).unwrap()
2060        })
2061        .await;
2062        let users_root = dirs::home_dir().unwrap().join(".trustee").join("users");
2063        assert_eq!(plain_home, users_root.join(trustee_core::user_hash(caller)));
2064        assert_eq!(
2065            pinned_home,
2066            users_root.join(trustee_core::user_hash(pinned_key)),
2067            "dispatch scope must pin resolution to the binding key"
2068        );
2069        assert_ne!(plain_home, pinned_home);
2070    }
2071
2072    /// Unique temp dir with a `config/` subdir (std-only; no tempfile dep).
2073    fn temp_user_home(tag: &str) -> std::path::PathBuf {
2074        static COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
2075        let n = COUNTER.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
2076        let dir = std::env::temp_dir().join(format!(
2077            "trustee-state-test-{}-{}-{}",
2078            tag,
2079            std::process::id(),
2080            n
2081        ));
2082        std::fs::create_dir_all(dir.join("config")).expect("create temp user home");
2083        dir
2084    }
2085
2086    fn parse(toml_str: &str) -> toml::Value {
2087        toml_str.parse::<toml::Value>().expect("valid test TOML")
2088    }
2089
2090    /// (a) Overlay with [mcp] + [server] + [auth]: [mcp] applied (user wins),
2091    /// shared [server] preserved untouched, overlay [auth] dropped entirely.
2092    #[test]
2093    fn overlay_allowlist_drops_non_allowlisted_sections() {
2094        let state = test_state();
2095        let home = temp_user_home("allowlist");
2096        std::fs::write(
2097            home.join("config").join("trustee.toml"),
2098            "[server]\nport = 1\n\n[auth]\nmode = \"kanidm\"\n\n[mcp]\nmode = \"user\"\n",
2099        )
2100        .expect("write overlay");
2101
2102        let shared = "[server]\nport = 8080\n\n[mcp]\nmode = \"shared\"\n";
2103        let merged = state
2104            .merge_user_config(&home, Some(shared))
2105            .expect("overlay has allowlisted content");
2106
2107        let merged_val = parse(&merged);
2108        let expected = parse("[server]\nport = 8080\n\n[mcp]\nmode = \"user\"\n");
2109        assert_eq!(
2110            merged_val, expected,
2111            "merged config must be shared + [mcp] overlay only"
2112        );
2113        assert!(
2114            merged_val.get("auth").is_none(),
2115            "overlay [auth] must be dropped"
2116        );
2117    }
2118
2119    /// (b) No overlay file → merge is a no-op (None), shared config untouched.
2120    #[test]
2121    fn no_overlay_file_returns_none() {
2122        let state = test_state();
2123        let home = temp_user_home("empty");
2124        assert!(state
2125            .merge_user_config(&home, Some("[server]\nport = 8080\n"))
2126            .is_none());
2127    }
2128
2129    /// Overlay whose sections are ALL non-allowlisted → no-op (None).
2130    #[test]
2131    fn overlay_with_no_allowlisted_sections_is_noop() {
2132        let state = test_state();
2133        let home = temp_user_home("all-dropped");
2134        std::fs::write(
2135            home.join("config").join("trustee.toml"),
2136            "[server]\nport = 1\n\n[storage]\npath = \"/tmp/x\"\n",
2137        )
2138        .expect("write overlay");
2139        assert!(state
2140            .merge_user_config(&home, Some("[server]\nport = 8080\n"))
2141            .is_none());
2142    }
2143
2144    /// (c) [llm] dropped when allow_llm_overlay=false (default), applied when
2145    /// the instance opted in via with_allow_llm_overlay(true).
2146    #[test]
2147    fn llm_overlay_dropped_by_default_and_kept_when_enabled() {
2148        let shared = "[llm]\nprovider = \"openai\"\n\n[mcp]\nmode = \"shared\"\n";
2149        let overlay = "[llm]\nprovider = \"anthropic\"\n";
2150
2151        // Default: knob false → [llm]-only overlay is a no-op.
2152        let state = test_state();
2153        assert!(!state.allow_llm_overlay);
2154        let home = temp_user_home("llm-off");
2155        std::fs::write(home.join("config").join("trustee.toml"), overlay).expect("write overlay");
2156        assert!(state.merge_user_config(&home, Some(shared)).is_none());
2157
2158        // Opted in: [llm] kept, user value wins; shared [mcp] untouched.
2159        let state = test_state().with_allow_llm_overlay(true);
2160        assert!(state.allow_llm_overlay);
2161        let home = temp_user_home("llm-on");
2162        std::fs::write(home.join("config").join("trustee.toml"), overlay).expect("write overlay");
2163        let merged = state
2164            .merge_user_config(&home, Some(shared))
2165            .expect("[llm] overlay applies when opted in");
2166        let merged_val = parse(&merged);
2167        assert_eq!(merged_val["llm"]["provider"].as_str(), Some("anthropic"));
2168        assert_eq!(merged_val["mcp"]["mode"].as_str(), Some("shared"));
2169    }
2170
2171    /// (d) Web-path home dir resolves through the consolidated
2172    /// trustee_core::user_hash (determinism/known-vectors covered in
2173    /// trustee-core's own tests).
2174    #[test]
2175    fn user_home_dir_uses_consolidated_hash() {
2176        let state = test_state();
2177        if let Some(home) = state.get_user_home_dir("farzan@example.com") {
2178            assert_eq!(
2179                home.file_name().and_then(|n| n.to_str()),
2180                Some(trustee_core::user_hash("farzan@example.com")).as_deref()
2181            );
2182            let users_root = dirs::home_dir().unwrap().join(".trustee").join("users");
2183            assert_eq!(home.parent(), Some(&users_root).map(|p| p.as_path()));
2184        }
2185        // dirs::home_dir() unavailable in sandbox → covered by core tests.
2186    }
2187
2188    /// xagent personas: [lifecycle] survives the overlay allowlist (persona
2189    /// is user-owned) but the WASM extension switch does not.
2190    #[test]
2191    fn lifecycle_overlay_survives_allowlist_enabled_stripped() {
2192        let state = test_state();
2193        let home = temp_user_home("lifecycle-persona");
2194        std::fs::write(
2195            home.join("config").join("trustee.toml"),
2196            "[lifecycle]\nsystem_template = \"You are Saman.\"\nenabled = true\n",
2197        )
2198        .expect("write overlay");
2199
2200        let merged = state
2201            .merge_user_config(
2202                &home,
2203                Some("[lifecycle]\nsystem_template = \"shared default\"\n"),
2204            )
2205            .expect("lifecycle overlay is allowlisted");
2206
2207        let v = merged.parse::<toml::Value>().expect("valid TOML");
2208        assert_eq!(
2209            v["lifecycle"]["system_template"].as_str(),
2210            Some("You are Saman."),
2211            "agent persona wins over shared default"
2212        );
2213        assert_eq!(
2214            v["lifecycle"].get("enabled"),
2215            None,
2216            "WASM extension switch is not user-settable"
2217        );
2218    }
2219
2220    /// 16D integration guard: dev-agent principals (user_key `agent-<name>`)
2221    /// get their OWN session bucket and home-dir namespace — one agent can
2222    /// never see another's sessions, and both go through the same
2223    /// user_hash isolation path as humans.
2224    #[tokio::test]
2225    async fn agent_principals_get_isolated_session_buckets() {
2226        let state = test_state();
2227        let key_a = "agent-farzan";
2228        let key_b = "agent-paydar";
2229
2230        let sid_a = state
2231            .create_session(key_a, None, None)
2232            .await
2233            .expect("create session");
2234        let _sid_b = state
2235            .create_session(key_b, None, None)
2236            .await
2237            .expect("create session");
2238
2239        // Each agent resolves its own session; B cannot see A's by id.
2240        assert!(
2241            state.get_session_by_any_id(key_a, &sid_a).await.is_some(),
2242            "owner bucket resolves its own session"
2243        );
2244        assert!(
2245            state.get_session_by_any_id(key_b, &sid_a).await.is_none(),
2246            "cross-agent session access must be 404/None"
2247        );
2248
2249        // Same isolation path as humans: home dir = users_root/user_hash(key).
2250        if let Some(home) = state.get_user_home_dir(key_a) {
2251            assert_eq!(
2252                home.file_name().and_then(|n| n.to_str()),
2253                Some(trustee_core::user_hash(key_a)).as_deref()
2254            );
2255        }
2256    }
2257}
2258
2259// ---------------------------------------------------------------------------
2260// 16C — per-user McpToolLoader cache
2261// ---------------------------------------------------------------------------
2262
2263#[cfg(test)]
2264mod mcp_loader_cache_tests {
2265    use super::*;
2266
2267    fn state_with_shared(shared: &str) -> ServerState {
2268        let (session, _rx) = Session::new();
2269        let (ws_tx, _ws_rx) = tokio::sync::broadcast::channel::<String>(16);
2270        let mut state = ServerState::new(session, ws_tx, None);
2271        state.config_toml = Some(shared.to_string());
2272        state
2273    }
2274
2275    /// Deterministic throwaway user: real user-home path under
2276    /// ~/.trustee/users/{hash} (that IS the resolution path under test),
2277    /// with config/ subdir; caller cleans up.
2278    struct TempUser {
2279        key: String,
2280        home: std::path::PathBuf,
2281    }
2282
2283    impl TempUser {
2284        fn new(tag: &str) -> Self {
2285            let key = format!("16c-{tag}-{}@test.invalid", std::process::id());
2286            let home = dirs::home_dir()
2287                .expect("HOME available in test env")
2288                .join(".trustee")
2289                .join("users")
2290                .join(trustee_core::user_hash(&key));
2291            std::fs::create_dir_all(home.join("config")).expect("create user home");
2292            Self { key, home }
2293        }
2294
2295        fn write_overlay(&self, toml_str: &str) {
2296            std::fs::write(self.home.join("config").join("trustee.toml"), toml_str)
2297                .expect("write overlay");
2298        }
2299    }
2300
2301    impl Drop for TempUser {
2302        fn drop(&mut self) {
2303            let _ = std::fs::remove_dir_all(&self.home);
2304        }
2305    }
2306
2307    #[tokio::test]
2308    async fn no_mcp_config_caches_disabled_marker() {
2309        let state = state_with_shared("[server]\nport = 8080\n");
2310        let user = TempUser::new("nomcp");
2311        let ts = Arc::new(pep::MemoryTokenStore::new());
2312
2313        let first = state.get_or_build_mcp_loader(&user.key, &ts).await.unwrap();
2314        assert!(first.is_none(), "no [mcp] anywhere → disabled marker");
2315        assert_eq!(state.mcp_loaders.len(), 1, "exactly one cache entry");
2316
2317        // Second call is a fingerprint hit — same disabled marker, still one entry.
2318        let second = state.get_or_build_mcp_loader(&user.key, &ts).await.unwrap();
2319        assert!(second.is_none());
2320        assert_eq!(state.mcp_loaders.len(), 1);
2321    }
2322
2323    #[tokio::test]
2324    async fn concurrent_cold_builds_single_flight() {
2325        let state = Arc::new(state_with_shared("[server]\nport = 8080\n"));
2326        let user = TempUser::new("singleflight");
2327        let ts = Arc::new(pep::MemoryTokenStore::new());
2328
2329        let mut handles = Vec::new();
2330        for _ in 0..5 {
2331            let state = state.clone();
2332            let key = user.key.clone();
2333            let ts = ts.clone();
2334            handles.push(tokio::spawn(async move {
2335                state.get_or_build_mcp_loader(&key, &ts).await
2336            }));
2337        }
2338        for h in handles {
2339            h.await.unwrap().expect("all five succeed");
2340        }
2341        assert_eq!(state.mcp_loaders.len(), 1, "single-flight → one entry");
2342    }
2343
2344    #[tokio::test]
2345    async fn fingerprint_change_triggers_rebuild() {
2346        let state = state_with_shared("[server]\nport = 8080\n");
2347        let user = TempUser::new("fpchange");
2348        let ts = Arc::new(pep::MemoryTokenStore::new());
2349
2350        // v1: an enabled MCP server pointing at a blackhole port — abk keeps
2351        // the loader with a DOWN status (connect refused is fast on loopback).
2352        user.write_overlay("[mcp]\nenabled = true\n\n[[mcp.servers]]\nname = \"v1\"\nurl = \"http://127.0.0.1:9/sse\"\n");
2353        let v1 = state.get_or_build_mcp_loader(&user.key, &ts).await.unwrap();
2354        assert!(v1.is_some(), "enabled [mcp] → real loader");
2355        let fp1 = state
2356            .mcp_loaders
2357            .get(&trustee_core::user_hash(&user.key))
2358            .unwrap()
2359            .fingerprint;
2360
2361        // v2: different server URL → different content fingerprint → rebuild.
2362        std::thread::sleep(std::time::Duration::from_millis(5));
2363        user.write_overlay("[mcp]\nenabled = true\n\n[[mcp.servers]]\nname = \"v2\"\nurl = \"http://127.0.0.1:9/other\"\n");
2364        let v2 = state.get_or_build_mcp_loader(&user.key, &ts).await.unwrap();
2365        assert!(v2.is_some());
2366        let entry = state
2367            .mcp_loaders
2368            .get(&trustee_core::user_hash(&user.key))
2369            .unwrap();
2370        assert_ne!(
2371            entry.fingerprint, fp1,
2372            "fingerprint must change with content"
2373        );
2374        assert!(entry.degraded.is_none());
2375
2376        // The old Arc stays valid for in-flight sessions (no panic, no revoke).
2377        let _still_usable = v1.as_ref().unwrap().tool_count;
2378    }
2379
2380    #[tokio::test]
2381    async fn degraded_entry_fails_loud_within_backoff_and_isolates_users() {
2382        let state = state_with_shared("[server]\nport = 8080\n");
2383        let bad = TempUser::new("degraded-bad");
2384        let good = TempUser::new("degraded-good");
2385        let ts = Arc::new(pep::MemoryTokenStore::new());
2386
2387        // [mcp] present but not a table → McpConfig deserialize fails → Err.
2388        bad.write_overlay("[mcp]\nenabled = \"not-a-bool\"\n");
2389        let err = match state.get_or_build_mcp_loader(&bad.key, &ts).await {
2390            Ok(_) => panic!("invalid [mcp] must fail loud"),
2391            Err(e) => e,
2392        };
2393        assert!(
2394            err.contains("invalid [mcp]"),
2395            "surfaces the parse error: {err}"
2396        );
2397
2398        let entry = state
2399            .mcp_loaders
2400            .get(&trustee_core::user_hash(&bad.key))
2401            .unwrap();
2402        assert!(entry.degraded.is_some(), "poison entry recorded");
2403
2404        // Within backoff: fast-fail again (cached error).
2405        let err2 = match state.get_or_build_mcp_loader(&bad.key, &ts).await {
2406            Ok(_) => panic!("still within backoff"),
2407            Err(e) => e,
2408        };
2409        assert_eq!(err, err2, "same cached error");
2410
2411        // Other users are completely unaffected.
2412        let good_loader = state
2413            .get_or_build_mcp_loader(&good.key, &ts)
2414            .await
2415            .expect("other user unaffected");
2416        assert!(
2417            good_loader.is_none(),
2418            "good user has no [mcp] → disabled marker"
2419        );
2420    }
2421
2422    #[tokio::test]
2423    async fn aged_out_entry_rebuilds_without_restart() {
2424        // THE 42131deb guard: a served-surface change behind a byte-identical
2425        // config must be reflected in the NEXT fresh session's manifest —
2426        // simulated by rewinding built_at past MCP_TOOLSET_MAX_AGE; the next
2427        // dispatch must NOT reuse the stale Arc (it rebuilds + re-lists).
2428        let state = state_with_shared("[server]\nport = 8080\n");
2429        let user = TempUser::new("agedout");
2430        let ts = Arc::new(pep::MemoryTokenStore::new());
2431
2432        user.write_overlay("[mcp]\nenabled = true\n\n[[mcp.servers]]\nname = \"v1\"\nurl = \"http://127.0.0.1:9/sse\"\n");
2433        let v1 = state
2434            .get_or_build_mcp_loader(&user.key, &ts)
2435            .await
2436            .expect("initial build");
2437        let v1 = v1.expect("enabled [mcp] → loader");
2438
2439        // Simulate time: the entry is now older than the max age, config
2440        // byte-identical (fingerprint UNCHANGED — that's the trap).
2441        {
2442            let mut entry = state
2443                .mcp_loaders
2444                .get_mut(&trustee_core::user_hash(&user.key))
2445                .unwrap();
2446            entry.built_at = chrono::Utc::now() - chrono::Duration::seconds(601);
2447        }
2448
2449        let v2 = state
2450            .get_or_build_mcp_loader(&user.key, &ts)
2451            .await
2452            .expect("aged-out dispatch rebuilds");
2453        let v2 = v2.expect("rebuilt loader present");
2454        assert!(
2455            !Arc::ptr_eq(&v1, &v2),
2456            "aged-out entry must NOT be reused — fresh connect re-lists the served surface"
2457        );
2458        let entry = state
2459            .mcp_loaders
2460            .get(&trustee_core::user_hash(&user.key))
2461            .unwrap();
2462        assert!(
2463            chrono::Utc::now() - entry.built_at < chrono::Duration::seconds(5),
2464            "built_at refreshed by the rebuild"
2465        );
2466
2467        // And a young, unchanged entry still hits (no rebuild churn).
2468        let v3 = state
2469            .get_or_build_mcp_loader(&user.key, &ts)
2470            .await
2471            .expect("fresh dispatch");
2472        assert!(Arc::ptr_eq(&v2, &v3.unwrap()), "young entry hits the cache");
2473    }
2474}
2475
2476#[cfg(test)]
2477mod restore_tests {
2478    use super::*;
2479
2480    fn fresh_state() -> ServerState {
2481        let (session, _rx) = Session::new();
2482        let (ws_tx, _ws_rx) = tokio::sync::broadcast::channel::<String>(16);
2483        let mut state = ServerState::new(session, ws_tx, None);
2484        state.config_toml = Some("[agent]\nname = \"trustee\"\n".to_string());
2485        state
2486    }
2487
2488    /// Deterministic throwaway user (same pattern as mcp_loader_cache_tests).
2489    struct TempUser {
2490        key: String,
2491    }
2492
2493    impl TempUser {
2494        fn new(tag: &str) -> Self {
2495            let key = format!("restore-{tag}-{}@test.invalid", std::process::id());
2496            let home = dirs::home_dir()
2497                .expect("HOME available in test env")
2498                .join(".trustee")
2499                .join("users")
2500                .join(trustee_core::user_hash(&key));
2501            std::fs::create_dir_all(home).expect("create user home");
2502            Self { key }
2503        }
2504    }
2505
2506    #[tokio::test]
2507    async fn restore_unknown_chain_returns_none() {
2508        let state = fresh_state();
2509        let user = TempUser::new("unknown");
2510        let out = state
2511            .get_or_restore_session_by_any_id(&user.key, "session_2099_01_01_00_00_deadbeef")
2512            .await;
2513        assert!(out.is_none(), "unknown chain must 404, not restore");
2514    }
2515
2516    #[tokio::test]
2517    async fn destroy_tombstones_chain_id() {
2518        let state = fresh_state();
2519        let user = TempUser::new("tombstone");
2520        let sid = state
2521            .create_session(&user.key, None, None)
2522            .await
2523            .expect("create session");
2524
2525        // Attach the live session to an on-disk-style chain id.
2526        let (session, _ws) = state.get_session(&user.key, &sid).await.unwrap();
2527        {
2528            let mut s = session.lock().await;
2529            s.session_id = Some("session_2026_09_04_10_00_aaaabbbb".to_string());
2530        }
2531
2532        state
2533            .destroy_session(&user.key, &sid)
2534            .await
2535            .expect("destroy");
2536
2537        let path = state
2538            .tombstone_path(&user.key, "session_2026_09_04_10_00_aaaabbbb")
2539            .expect("tombstone path");
2540        assert!(path.exists(), "destroy must write a chain tombstone");
2541
2542        // And a fresh live session with NO chain id must not tombstone anything.
2543        let sid2 = state
2544            .create_session(&user.key, None, None)
2545            .await
2546            .expect("create second session");
2547        state
2548            .destroy_session(&user.key, &sid2)
2549            .await
2550            .expect("destroy 2");
2551        // (No assertion beyond "no panic": session_id was None → no new markers.)
2552    }
2553
2554    #[tokio::test]
2555    async fn destroy_closes_the_broadcast_channel() {
2556        // a3e1f3b6 (the owner's two-bug report, 2026-11-14): destroy must
2557        // CLOSE the session's WS broadcast channel. The drain task held the
2558        // ORIGINAL ws_tx sender + the session Arc (the arc cycle: the drain
2559        // task keeps the session alive, the session keeps workflow_tx alive,
2560        // workflow_rx never closes), so the channel never closed and the
2561        // connected clients' streams stayed open — no onclose, the console's
2562        // 'Session ended' detection never fired, and the destroyed session's
2563        // memory leaked. The abort handle in the entry breaks the cycle:
2564        // after destroy ALL senders are gone and the receivers see Closed.
2565        let state = fresh_state();
2566        let user = TempUser::new("ws-close");
2567        let sid = state
2568            .create_session(&user.key, None, None)
2569            .await
2570            .expect("create session");
2571
2572        let (_, ws_tx) = state.get_session(&user.key, &sid).await.unwrap();
2573        let mut rx = ws_tx.subscribe();
2574        drop(ws_tx); // the caller's clone goes; the entry + the drain task hold theirs
2575
2576        state
2577            .destroy_session(&user.key, &sid)
2578            .await
2579            .expect("destroy");
2580
2581        // The channel is CLOSED: the receiver sees Closed (all senders
2582        // dropped — the entry removed + the drain task aborted). An open
2583        // channel with no messages would block here forever; Closed returns
2584        // immediately.
2585        let outcome = rx.recv().await;
2586        assert!(
2587            matches!(
2588                outcome,
2589                Err(tokio::sync::broadcast::error::RecvError::Closed)
2590            ),
2591            "destroy must close the broadcast channel (the drain task aborted), got {outcome:?}"
2592        );
2593    }
2594
2595    #[tokio::test]
2596    async fn restore_refuses_registry_key_shapes() {
2597        // Registry keys ("default", "web…") are not checkpoint chains; even
2598        // if create_resume_info could not find them anyway, the tombstone
2599        // guard short-circuits non-`session_` ids without touching disk.
2600        let state = fresh_state();
2601        let user = TempUser::new("shapes");
2602        assert!(state.tombstone_path(&user.key, "default").is_none());
2603        let out = state
2604            .get_or_restore_session_by_any_id(&user.key, "default")
2605            .await;
2606        assert!(out.is_none());
2607    }
2608}
2609
2610#[test]
2611fn collect_unresolved_placeholders_walks_the_tree() {
2612    let section: toml::Value = toml::from_str(
2613        r#"
2614[credentials.farzan_service_account]
2615service_token = "${FARZAN_SERVICE_ACCOUNT}"
2616issuer_url = "https://ok.example"
2617
2618[[servers]]
2619name = "fame"
2620url = "https://fame.example"
2621credentials = "farzan_service_account"
2622
2623[[servers]]
2624name = "other"
2625url = "https://other.example"
2626token = "${OTHER_TOKEN}"
2627"#,
2628    )
2629    .unwrap();
2630    let mut out = Vec::new();
2631    collect_unresolved_placeholders(&section, &mut out);
2632    out.sort();
2633    assert_eq!(
2634        out,
2635        vec![
2636            "${FARZAN_SERVICE_ACCOUNT}".to_string(),
2637            "${OTHER_TOKEN}".to_string()
2638        ],
2639        "every leftover placeholder in the [mcp] tree must be collected, deduped"
2640    );
2641    // clean section → nothing
2642    let clean: toml::Value = toml::from_str("[x]\ny = \"z\"\n").unwrap();
2643    let mut out2 = Vec::new();
2644    collect_unresolved_placeholders(&clean, &mut out2);
2645    assert!(out2.is_empty());
2646}
2647
2648#[test]
2649fn substitute_env_vars_resolves_and_keeps_leftovers_deterministically() {
2650    let mut secrets = std::collections::HashMap::new();
2651    secrets.insert("A_TOKEN".to_string(), "resolved-value".to_string());
2652    let mut s = "x = \"${A_TOKEN}\" y = \"${MISSING_TOKEN}\"".to_string();
2653    let report = substitute_env_vars(&mut s, &secrets, "test");
2654    assert!(s.contains("resolved-value"), "declared key resolves: {s}");
2655    assert!(
2656        s.contains("${MISSING_TOKEN}"),
2657        "undeclared key keeps the literal (the loader guard turns this into a loud failure): {s}"
2658    );
2659    // 92a36073: the report names each distinct variable and its source.
2660    assert_eq!(report.len(), 2, "one entry per distinct var: {report:?}");
2661    assert!(report.contains(&EnvVarResolution {
2662        name: "A_TOKEN".to_string(),
2663        source: SecretSource::SecretsMap
2664    }));
2665    assert!(report.contains(&EnvVarResolution {
2666        name: "MISSING_TOKEN".to_string(),
2667        source: SecretSource::Unresolved
2668    }));
2669}
2670
2671#[test]
2672fn substitute_env_vars_reports_process_env_shadow() {
2673    // 92a36073: a var ABSENT from the per-user/profile secrets but set
2674    // in the process environment is the global-token shadowing shape —
2675    // it must be REPORTED as ProcessEnv, and the secrets map must still
2676    // win when it DOES carry the key.
2677    let var = format!("RAVAND_PROVENANCE_TEST_{}", std::process::id());
2678    std::env::set_var(&var, "global-env-answer");
2679
2680    // Absent from secrets → process env answers → ProcessEnv reported.
2681    let mut secrets = std::collections::HashMap::new();
2682    let mut s = format!("x = \"${{{var}}}\"");
2683    let report = substitute_env_vars(&mut s, &secrets, "test-shadow");
2684    assert!(s.contains("global-env-answer"), "process env answered: {s}");
2685    assert_eq!(
2686        report,
2687        vec![EnvVarResolution {
2688            name: var.clone(),
2689            source: SecretSource::ProcessEnv
2690        }]
2691    );
2692
2693    // Same var present in secrets → secrets WIN (per-user precedence
2694    // unchanged), reported as SecretsMap.
2695    secrets.insert(var.clone(), "per-user-answer".to_string());
2696    let mut s2 = format!("x = \"${{{var}}}\"");
2697    let report2 = substitute_env_vars(&mut s2, &secrets, "test-wins");
2698    assert!(s2.contains("per-user-answer"), "user secrets win: {s2}");
2699    assert_eq!(
2700        report2,
2701        vec![EnvVarResolution {
2702            name: var.clone(),
2703            source: SecretSource::SecretsMap
2704        }]
2705    );
2706
2707    std::env::remove_var(&var);
2708}
2709
2710#[test]
2711fn materialized_user_missing_env_only_fires_on_dir_without_env() {
2712    // 92a36073 boot-surface: warn shape is dir-present + .env-absent.
2713    let base = std::env::temp_dir().join(format!("trustee-envcheck-{}", std::process::id()));
2714    let _ = std::fs::remove_dir_all(&base);
2715
2716    // Not materialized at all → NOT the warning shape.
2717    assert!(!materialized_user_missing_env(&base.join("ghost")));
2718
2719    // Materialized WITHOUT .env → the warning shape.
2720    std::fs::create_dir_all(&base).unwrap();
2721    assert!(materialized_user_missing_env(&base));
2722
2723    // Materialized WITH .env → not the warning shape.
2724    std::fs::write(base.join(".env"), "K=V\n").unwrap();
2725    assert!(!materialized_user_missing_env(&base));
2726
2727    let _ = std::fs::remove_dir_all(&base);
2728}
2729
2730#[test]
2731fn loader_freshness_decides_all_four_states() {
2732    let now = chrono::Utc::now();
2733    let fp = 0xABCD_u64;
2734    let young = now - chrono::Duration::seconds(60);
2735    let old = now - chrono::Duration::seconds(601);
2736
2737    // Fresh: same fingerprint, young entry.
2738    assert_eq!(
2739        loader_freshness(fp, young, None, None, fp, now),
2740        LoaderFreshness::Fresh
2741    );
2742    // ConfigChanged: fingerprint differs wins over age.
2743    assert_eq!(
2744        loader_freshness(fp, old, None, None, 0x1234, now),
2745        LoaderFreshness::ConfigChanged
2746    );
2747    // AgedOut: same fingerprint past the max age — the 42131deb guard.
2748    assert_eq!(
2749        loader_freshness(fp, old, None, None, fp, now),
2750        LoaderFreshness::AgedOut
2751    );
2752    // Backoff: degraded inside the retry window, even old + unchanged.
2753    let failed = now - chrono::Duration::seconds(10);
2754    match loader_freshness(fp, old, Some("boom"), Some(failed), fp, now) {
2755        LoaderFreshness::Backoff(remaining) => {
2756            assert!(remaining.as_secs() >= 15, "~20s left: {remaining:?}");
2757        }
2758        other => panic!("expected Backoff, got {other:?}"),
2759    }
2760    // Degraded but PAST backoff → age rules apply (rebuild proceeds).
2761    let failed_long = now - chrono::Duration::seconds(120);
2762    assert_eq!(
2763        loader_freshness(fp, old, Some("boom"), Some(failed_long), fp, now),
2764        LoaderFreshness::AgedOut
2765    );
2766}