zeph_core/serve.rs
1// SPDX-FileCopyrightText: 2026 Andrei G <bug-ops>
2// SPDX-License-Identifier: MIT OR Apache-2.0
3
4//! `SessionActor` and `LiveSessionRegistry` for `zeph serve` (spec-068 §9, #5343).
5//!
6//! Each live conversation-session under `zeph serve` is a [`SessionActor`] task: it owns an
7//! [`Agent<LoopbackChannel>`](crate::agent::Agent) exclusively, bridges [`SessionCommand`]s into
8//! the agent's channel input, and forwards the channel's output as [`SessionOutput`] over a
9//! broadcast channel any number of HTTP/SSE or TUI attachments can subscribe to.
10//!
11//! [`LiveSessionRegistry`] is pure bookkeeping (a `HashMap` behind a `parking_lot::Mutex`, never
12//! held across `.await`) — it does not itself supervise tasks. [`SessionActor::spawn`] registers
13//! a *coordinator* task under `TaskSupervisor` via `spawn_oneshot(name: Arc<str>, factory)`
14//! (architect ruling D-7): the dynamic `serve.session.<id>` name and non-restarting `RunOnce`
15//! policy are exactly right for a session actor (re-driving a torn turn/replay after a crash is
16//! unsafe; recovery is a fresh spawn that replays the durable log from the last committed `seq`).
17//!
18//! `Agent<C>`'s futures are `!Send` (documented precedent: `crates/zeph-acp/src/transport/
19//! stdio.rs`, "Agent futures are `!Send` and deeply nested") and `Agent<LoopbackChannel>` itself
20//! cannot cross *any* thread boundary — not just `spawn_oneshot`'s `Send` bound but a
21//! `std::thread::spawn` one too. [`SessionActor::spawn`] resolves this exactly as `zeph-acp`'s
22//! `serve_stdio`/`transport/http.rs` do (architect ruling D-8): the `Agent` is constructed and
23//! driven entirely inside a dedicated OS thread with its own `current_thread` runtime and
24//! `LocalSet` — only `Send`-safe state (a `FnOnce(LoopbackChannel) -> Agent<LoopbackChannel>`
25//! factory, mirroring `zeph-acp`'s `SendAgentSpawner`) crosses into that thread. The
26//! `spawn_oneshot` task is a *thin coordinator*, never the agent driver itself: it awaits the
27//! thread's completion signal and, on process-wide supervisor shutdown, forwards cancellation
28//! onto the session's own [`CancellationToken`] (distinct from the supervisor's) so idle
29//! eviction (spec §9.3) can cancel exactly one session without tearing down every live actor,
30//! while `drive` only ever needs to select on a single cancellation source regardless of trigger.
31
32use std::collections::HashMap;
33use std::future::Future;
34use std::sync::Arc;
35use std::time::{Duration, Instant};
36
37use parking_lot::Mutex;
38use tokio::sync::{broadcast, mpsc};
39use tokio_util::sync::CancellationToken;
40use zeph_common::SessionId;
41use zeph_common::task_supervisor::{BlockingHandle, TaskSupervisor};
42
43use crate::agent::Agent;
44use crate::channel::{ChannelMessage, LoopbackChannel, LoopbackEvent, LoopbackHandle};
45
46/// Stack size for the dedicated per-session thread — mirrors `zeph-acp`'s
47/// `ACP_AGENT_STACK_SIZE` (Agent futures are deeply nested, ~512 KiB measured on overflow with
48/// the default 2 MiB worker-thread stack).
49const SESSION_ACTOR_STACK_SIZE: usize = 8 * 1024 * 1024;
50
51/// Buffer capacity for the `LoopbackChannel` constructed inside [`SessionActor::spawn`]'s
52/// dedicated thread — matches the buffer used by every other headless `LoopbackChannel` consumer
53/// (e.g. `zeph-acp`'s A2A bridge).
54const LOOPBACK_CHANNEL_CAPACITY: usize = 8;
55
56/// A command sent to a live [`SessionActor`] (spec §9.2).
57#[derive(Debug)]
58pub enum SessionCommand {
59 /// Submit a new user prompt for this turn.
60 Prompt {
61 /// The prompt text.
62 text: String,
63 },
64 /// Interrupt the agent's current operation (mirrors the existing `LoopbackHandle` cancel
65 /// signal used by every other headless channel consumer).
66 Cancel,
67 /// Gracefully end the actor: closes the agent's channel input, letting `Agent::run` observe
68 /// channel closure and exit its loop on its own rather than being aborted mid-turn.
69 Shutdown,
70}
71
72/// An event streamed out of a live [`SessionActor`] (spec §9.2).
73///
74/// `Serialize`s as an adjacently tagged JSON object (`{"type": "token", "data": "..."}`) for
75/// `GET /sessions/:id/events`'s SSE stream (spec §9.4) — adjacent rather than internal tagging
76/// because [`Self::Token`] and [`Self::Error`] wrap a bare `String`, which cannot be flattened
77/// into an internally tagged object.
78#[derive(Debug, Clone, serde::Serialize)]
79#[serde(tag = "type", content = "data", rename_all = "snake_case")]
80pub enum SessionOutput {
81 /// A streamed or full-message text chunk from the agent's response.
82 Token(String),
83 /// A tool call started.
84 ToolCall {
85 /// Name of the tool being invoked.
86 tool_name: String,
87 /// Opaque tool call ID assigned by the LLM.
88 tool_call_id: String,
89 },
90 /// A tool call produced output.
91 ToolResult {
92 /// Name of the tool that produced this output.
93 tool_name: String,
94 /// Human-readable output text.
95 display: String,
96 },
97 /// The current turn finished.
98 TurnComplete,
99 /// The agent loop ended with an error.
100 Error(String),
101}
102
103/// Default broadcast channel capacity for [`SessionOutput`] — generous enough to absorb a burst
104/// of streamed tokens between a slow subscriber's polls without lagging.
105const OUTPUT_CHANNEL_CAPACITY: usize = 256;
106
107/// Owns a live [`Agent<LoopbackChannel>`](crate::agent::Agent) for one conversation-session
108/// (spec §9.2).
109///
110/// This is a driver, not a stored value — [`SessionActor::spawn`] returns a
111/// [`SessionActorHandle`] (cheap to clone, holds only channel senders) for
112/// [`LiveSessionRegistry`] to track; the actor's own state lives entirely inside the spawned
113/// task.
114pub struct SessionActor;
115
116impl SessionActor {
117 /// Spawn a new `SessionActor` under `supervisor` (architect ruling D-8).
118 ///
119 /// `build_agent` is called *inside* a dedicated thread (see the module doc) with a freshly
120 /// constructed `LoopbackChannel`, and must return the `Agent<LoopbackChannel>` built from it
121 /// — never pass an already-built `Agent` in, since `Agent<LoopbackChannel>` is `!Send` and
122 /// cannot cross the thread boundary. Mirrors `zeph-acp`'s `SendAgentSpawner`
123 /// (`Arc<dyn Fn(...) -> Agent + Send + Sync>`): typical callers wrap an existing
124 /// `AgentBuilder` pipeline (the same one used for CLI/TUI/Telegram/ACP sessions) in a closure
125 /// capturing only `Send`-safe dependencies (provider, skill registry, tool executor, the
126 /// session's `Arc<SessionEventLog>`, session id — all `Send`/`Sync`). `Agent::new` is sync,
127 /// so `build_agent` is a plain sync closure — no async-construction-in-thread complexity.
128 ///
129 /// Registers a *coordinator* task under the dynamic name `serve.session.<id>` via
130 /// [`TaskSupervisor::spawn_oneshot`] — visible through `supervisor.snapshot()`. The
131 /// coordinator never touches the `!Send` `Agent`; it only awaits the dedicated thread's
132 /// completion signal and, if the supervisor's own `CancellationToken` fires first (process
133 /// shutdown), forwards cancellation onto the session's own token so [`Self::drive`] observes
134 /// exactly one cancellation source regardless of trigger. Session actors intentionally do not
135 /// auto-restart on panic or unexpected exit (`spawn_oneshot`'s `RestartPolicy::RunOnce`):
136 /// re-driving a torn turn or replay in place is unsafe. Recovery is a fresh spawn (re-attach)
137 /// that replays the durable log from the last committed `seq`, not an in-place restart.
138 ///
139 /// Returns the [`SessionActorHandle`] for [`LiveSessionRegistry`] plus the raw
140 /// [`BlockingHandle`] — callers that need a forced-abort fallback (e.g. if a `serve.evict`
141 /// idle-eviction task's cancellation overruns its TTL grace) should hold onto the latter. The
142 /// graceful paths are sending [`SessionCommand::Shutdown`] over [`SessionActorHandle::tx`],
143 /// or cancelling [`SessionActorHandle::cancel`] directly (what idle eviction uses, spec §9.3,
144 /// to target exactly this session without affecting any other live actor).
145 #[must_use]
146 pub fn spawn<F>(
147 supervisor: &TaskSupervisor,
148 registry: &Arc<LiveSessionRegistry>,
149 session_id: &SessionId,
150 build_agent: F,
151 mailbox_capacity: usize,
152 ) -> (SessionActorHandle, BlockingHandle<()>)
153 where
154 F: FnOnce(LoopbackChannel) -> Agent<LoopbackChannel> + Send + 'static,
155 {
156 let (cmd_tx, cmd_rx) = mpsc::channel(mailbox_capacity.max(1));
157 let (tx_out, _first_subscriber) = broadcast::channel(OUTPUT_CHANNEL_CAPACITY);
158 let tx_out_for_actor = tx_out.clone();
159 let id_str = session_id.as_str().to_owned();
160
161 // Per-session cancellation, distinct from the supervisor's process-wide token, so idle
162 // eviction can target exactly this session (spec §9.3) without tearing down every live
163 // actor. The coordinator forwards a supervisor-wide shutdown onto this same token.
164 let session_cancel = CancellationToken::new();
165 let session_cancel_for_thread = session_cancel.clone();
166 let session_cancel_for_coordinator = session_cancel.clone();
167
168 let (done_tx, done_rx) = tokio::sync::oneshot::channel::<()>();
169 let thread_name = format!("serve-session-{id_str}");
170 let thread_session_id = id_str.clone();
171 let spawn_result = std::thread::Builder::new()
172 .name(thread_name)
173 .stack_size(SESSION_ACTOR_STACK_SIZE)
174 .spawn(move || {
175 let rt = match tokio::runtime::Builder::new_current_thread()
176 .enable_all()
177 .build()
178 {
179 Ok(rt) => rt,
180 Err(e) => {
181 tracing::error!(
182 session_id = %thread_session_id,
183 error = %e,
184 "failed to build session actor tokio runtime"
185 );
186 let _ = done_tx.send(());
187 return;
188 }
189 };
190 let (channel, handle) = LoopbackChannel::pair(LOOPBACK_CHANNEL_CAPACITY);
191 let agent = build_agent(channel);
192 let local = tokio::task::LocalSet::new();
193 rt.block_on(local.run_until(Self::drive(
194 agent,
195 handle,
196 cmd_rx,
197 tx_out_for_actor,
198 session_cancel_for_thread,
199 )));
200 let _ = done_tx.send(());
201 });
202
203 if let Err(e) = spawn_result {
204 tracing::error!(error = %e, "failed to spawn dedicated session actor thread");
205 }
206
207 let name: Arc<str> = Arc::from(format!("serve.session.{id_str}"));
208 let supervisor_cancel = supervisor.cancellation_token();
209 let registry_for_coordinator = Arc::clone(registry);
210 let session_id_for_coordinator = session_id.clone();
211 let tx_for_coordinator = cmd_tx.clone();
212 let blocking_handle = supervisor.spawn_oneshot(name, move || {
213 Self::coordinate(
214 done_rx,
215 supervisor_cancel,
216 session_cancel_for_coordinator,
217 registry_for_coordinator,
218 session_id_for_coordinator,
219 tx_for_coordinator,
220 )
221 });
222
223 (
224 SessionActorHandle {
225 tx: cmd_tx,
226 tx_out,
227 last_active: Instant::now(),
228 cancel: session_cancel,
229 },
230 blocking_handle,
231 )
232 }
233
234 /// Thin `Send`-safe coordinator handed to [`TaskSupervisor::spawn_oneshot`]: never touches
235 /// the `!Send` `Agent` — only a thread-completion signal and cancellation tokens. Forwards a
236 /// process-wide supervisor shutdown onto the session's own [`CancellationToken`] so
237 /// [`Self::drive`] (running on the dedicated thread) only ever needs to select on one
238 /// cancellation source.
239 ///
240 /// M1 (impl-critic finding): reaps `registry`'s entry for `session_id` unconditionally once
241 /// the dedicated thread signals completion — regardless of *why* it ended (agent panic,
242 /// normal exit, or supervisor shutdown). Before this, [`LiveSessionRegistry::idle_candidates`]
243 /// was the *only* reap path, and it requires `receiver_count() == 0`; a lingering `GET
244 /// /sessions/:id/events` SSE subscriber (or a crashed actor whose stream simply stops
245 /// emitting, never closing) could pin a dead session in the registry forever, so
246 /// `POST /sessions/:id/prompt` would return `410 Gone` for it permanently with no path back
247 /// to a live actor even under D-12's reactivation. Uses
248 /// [`LiveSessionRegistry::remove_if_current`] (not a plain key-based `remove`) so a
249 /// concurrent reactivation that has already `insert`ed a fresh handle under the same
250 /// `session_id` is never evicted by this now-dead coordinator.
251 async fn coordinate(
252 done_rx: tokio::sync::oneshot::Receiver<()>,
253 supervisor_cancel: CancellationToken,
254 session_cancel: CancellationToken,
255 registry: Arc<LiveSessionRegistry>,
256 session_id: SessionId,
257 tx: mpsc::Sender<SessionCommand>,
258 ) {
259 let mut done_rx = done_rx;
260 tokio::select! {
261 _ = &mut done_rx => {}
262 () = supervisor_cancel.cancelled() => {
263 session_cancel.cancel();
264 let _ = done_rx.await;
265 }
266 }
267 registry.remove_if_current(&session_id, &tx);
268 }
269
270 /// Bridge `cmd_rx` into the agent's `LoopbackChannel` input and forward the channel's output
271 /// as [`SessionOutput`] over `tx_out`, while concurrently driving `agent.run()` to
272 /// completion. A single `tokio::select!` loop — no raw `tokio::spawn` — per the non-blocking
273 /// contract (CLAUDE.md Async & Background Tasks).
274 ///
275 /// Must run inside a `tokio::task::LocalSet` (or be `.await`ed directly, never spawned via
276 /// a `Send`-bound spawner) because `Agent<C>`'s futures are `!Send`.
277 ///
278 /// `cancel` cancelling (via `TaskSupervisor::shutdown_all`) is treated the same as
279 /// [`SessionCommand::Shutdown`] — a graceful flush (drop the channel's input sender, let
280 /// `Agent::run` observe closure and exit, drain buffered output) rather than the abrupt
281 /// task-abort `shutdown_all` would otherwise apply. Abrupt abort is still safe (INV-SP-2
282 /// torn-tail truncation covers a torn trailing write) but an explicit flush avoids leaving a
283 /// truncated trailing event on every shutdown.
284 async fn drive(
285 mut agent: Agent<LoopbackChannel>,
286 handle: LoopbackHandle,
287 mut cmd_rx: mpsc::Receiver<SessionCommand>,
288 tx_out: broadcast::Sender<SessionOutput>,
289 cancel: CancellationToken,
290 ) {
291 let LoopbackHandle {
292 input_tx,
293 mut output_rx,
294 cancel_signal,
295 } = handle;
296 let mut input_tx = Some(input_tx);
297
298 let mut agent_run = std::pin::pin!(agent.run());
299 let mut agent_done = false;
300
301 while !agent_done {
302 tokio::select! {
303 biased;
304 result = &mut agent_run => {
305 agent_done = true;
306 if let Err(e) = result {
307 tracing::warn!(error = %e, "session actor: agent run ended with error");
308 let _ = tx_out.send(SessionOutput::Error(e.to_string()));
309 }
310 }
311 Some(event) = output_rx.recv() => {
312 if let Some(output) = translate_loopback_event(event) {
313 let _ = tx_out.send(output);
314 }
315 }
316 () = cancel.cancelled(), if input_tx.is_some() => {
317 tracing::info!("session actor: supervisor shutdown, flushing and exiting");
318 input_tx = None;
319 }
320 cmd = cmd_rx.recv() => {
321 match cmd {
322 Some(SessionCommand::Prompt { text }) => {
323 if let Some(tx) = &input_tx {
324 let msg = ChannelMessage {
325 text,
326 attachments: Vec::new(),
327 is_guest_context: false,
328 is_from_bot: false,
329 };
330 tracing::debug!("session actor: forwarding prompt to agent channel");
331 let _ = tx.send(msg).await;
332 tracing::debug!("session actor: prompt forwarded");
333 }
334 }
335 Some(SessionCommand::Cancel) => cancel_signal.notify_one(),
336 Some(SessionCommand::Shutdown) | None => {
337 // Drop the sender: `Agent::run`'s `next_event()` observes the closed
338 // channel and returns `Ok(None)`, ending the loop gracefully (see
339 // `crates/zeph-core/src/agent/mod.rs::next_event` doc comment).
340 input_tx = None;
341 }
342 }
343 }
344 }
345 }
346
347 // Drain any output events buffered just before the agent loop ended.
348 while let Ok(event) = output_rx.try_recv() {
349 if let Some(output) = translate_loopback_event(event) {
350 let _ = tx_out.send(output);
351 }
352 }
353 }
354}
355
356/// Maps a [`LoopbackEvent`] to a [`SessionOutput`], or `None` for event kinds not yet part of
357/// the spec §9.2 `SessionOutput` schema (`Status`/`ThinkingChunk`/`Usage`/`SessionTitle`/`Plan`/
358/// `Stop`) — dropped rather than guessed at an ad hoc extension.
359fn translate_loopback_event(event: LoopbackEvent) -> Option<SessionOutput> {
360 match event {
361 LoopbackEvent::Chunk(text) | LoopbackEvent::FullMessage(text) => {
362 Some(SessionOutput::Token(text))
363 }
364 LoopbackEvent::Flush => Some(SessionOutput::TurnComplete),
365 LoopbackEvent::ToolStart(ev) => Some(SessionOutput::ToolCall {
366 tool_name: ev.tool_name.to_string(),
367 tool_call_id: ev.tool_call_id,
368 }),
369 LoopbackEvent::ToolOutput(ev) => Some(SessionOutput::ToolResult {
370 tool_name: ev.tool_name.to_string(),
371 display: ev.display,
372 }),
373 _ => None,
374 }
375}
376
377/// Bookkeeping handle for one live session, held by [`LiveSessionRegistry`] (spec §9.3).
378///
379/// Cheap to clone — `tx`/`tx_out` are `Arc`-backed channel senders.
380#[derive(Clone)]
381pub struct SessionActorHandle {
382 /// Mailbox for [`SessionCommand`]s; same-session prompts are serialized by this mpsc's FIFO
383 /// ordering (spec §9.2 concurrency policy — no separate turn-lock needed).
384 pub tx: mpsc::Sender<SessionCommand>,
385 /// Broadcast source new subscribers (SSE connections, TUI attach) subscribe to.
386 pub tx_out: broadcast::Sender<SessionOutput>,
387 /// Updated by [`LiveSessionRegistry::get`] on every lookup; read by
388 /// [`LiveSessionRegistry::idle_candidates`] for TTL eviction.
389 pub last_active: Instant,
390 /// Cancelling this token ends exactly this session's actor gracefully — the same effect as
391 /// sending [`SessionCommand::Shutdown`], but usable without an owned mpsc permit. What
392 /// `serve.evict` idle eviction (spec §9.3) calls to target one session without affecting any
393 /// other live actor. Distinct from the process-wide `TaskSupervisor` cancellation token.
394 pub cancel: CancellationToken,
395}
396
397/// Registry of live [`SessionActor`]s for `zeph serve` (spec §9.3).
398///
399/// Distinct from the TUI's own `SessionRegistry`/`SlotId` tab-switching abstraction
400/// (`zeph-tui/src/session.rs`) — this is `zeph serve`-specific bookkeeping only. The internal
401/// `sessions` mutex is never held across `.await`; [`Self::get_or_reactivate`]'s
402/// `reactivation_lock` is a separate `tokio::sync::Mutex` deliberately held across `.await` for
403/// its entire critical section — see that method's doc comment.
404#[derive(Default)]
405pub struct LiveSessionRegistry {
406 sessions: Mutex<HashMap<SessionId, SessionActorHandle>>,
407 /// Serializes [`Self::get_or_reactivate`]'s check-build-insert critical section (N1,
408 /// impl-critic re-verify finding): without it, two concurrent requests for the same
409 /// evicted-but-durable session both miss the fast-path `get`, both independently replay and
410 /// spawn a `SessionActor` over the *same* `SessionEventLog` file, and the second `insert`
411 /// silently orphans the first — two live writers on one INV-D2 single-writer log, corrupting
412 /// it (duplicate `seq`s, a torn line that isn't the trailing one). A single process-wide lock
413 /// (not a per-session map) is deliberate: reactivation is a rare recovery path, not the hot
414 /// `get()` path every prompt/events call takes first — serializing distinct sessions'
415 /// reactivations against each other is an acceptable trade for not needing to manage a
416 /// second, dynamically-growing lock table's own cleanup/eviction lifecycle.
417 reactivation_lock: tokio::sync::Mutex<()>,
418}
419
420impl LiveSessionRegistry {
421 /// Construct an empty registry.
422 #[must_use]
423 pub fn new() -> Self {
424 Self::default()
425 }
426
427 /// Look up a live session by id, refreshing its `last_active` timestamp on hit.
428 #[must_use]
429 pub fn get(&self, id: &SessionId) -> Option<SessionActorHandle> {
430 let mut sessions = self.sessions.lock();
431 let handle = sessions.get_mut(id)?;
432 handle.last_active = Instant::now();
433 Some(handle.clone())
434 }
435
436 /// Register a freshly spawned actor's handle, replacing (and dropping, without aborting) any
437 /// prior entry under the same id.
438 pub fn insert(&self, id: SessionId, handle: SessionActorHandle) {
439 self.sessions.lock().insert(id, handle);
440 }
441
442 /// Look up a live session, reactivating it via `reactivate` if it isn't currently live —
443 /// atomically, so two concurrent callers for the same absent `id` can never both win and
444 /// double-spawn a `SessionActor` over the same durable log (N1, impl-critic re-verify
445 /// finding: `SessionEventLog` is single-writer per INV-D2, and a plain `get()`-miss-then-
446 /// spawn-then-`insert()` sequence has no such guarantee under concurrency).
447 ///
448 /// Fast path (the common case — session already live): a plain `get()`, no lock contention
449 /// with any in-flight reactivation elsewhere. Slow path (session absent): acquires
450 /// `reactivation_lock`, then re-checks `get()` *under the lock* — if a concurrent caller won
451 /// the race and already reactivated `id` while this caller was waiting for the lock, that
452 /// caller's `insert` is visible here and `reactivate` is never invoked a second time. Only
453 /// the loser of the race skips calling `reactivate`; the winner runs it exactly once.
454 ///
455 /// `reactivate` is expected to build+spawn the actor and `insert` it into `self` before
456 /// resolving — it runs to completion holding `reactivation_lock`, so its own `insert` is
457 /// safely serialized against any other concurrent `get_or_reactivate` call for the same
458 /// (or a different) id.
459 #[tracing::instrument(
460 name = "core.serve.registry.get_or_reactivate",
461 skip_all,
462 level = "debug",
463 fields(session_id = id.as_str())
464 )]
465 pub async fn get_or_reactivate<F, Fut>(
466 &self,
467 id: &SessionId,
468 reactivate: F,
469 ) -> Option<SessionActorHandle>
470 where
471 F: FnOnce() -> Fut,
472 Fut: Future<Output = Option<SessionActorHandle>>,
473 {
474 if let Some(handle) = self.get(id) {
475 return Some(handle);
476 }
477 let _guard = self.reactivation_lock.lock().await;
478 if let Some(handle) = self.get(id) {
479 return Some(handle);
480 }
481 reactivate().await
482 }
483
484 /// Remove `id`'s entry only if it is still the exact handle identified by `tx` (compared via
485 /// [`mpsc::Sender::same_channel`], which is `true` iff both senders share the same underlying
486 /// channel).
487 ///
488 /// Used by [`SessionActor`]'s coordinator (M1) to reap its own registry entry on completion
489 /// without racing a concurrent reactivation (D-12) that has already `insert`ed a *fresh*
490 /// handle under the same id — a plain `remove(id)` there would delete the new entry too, an
491 /// unconditional key-based removal cannot tell "my own now-dead entry" from "a different,
492 /// live entry that happens to share this id".
493 pub fn remove_if_current(
494 &self,
495 id: &SessionId,
496 tx: &mpsc::Sender<SessionCommand>,
497 ) -> Option<SessionActorHandle> {
498 let mut sessions = self.sessions.lock();
499 if sessions.get(id).is_some_and(|h| h.tx.same_channel(tx)) {
500 sessions.remove(id)
501 } else {
502 None
503 }
504 }
505
506 /// Remove a session's handle (used by idle eviction and explicit shutdown), returning it if
507 /// present.
508 pub fn remove(&self, id: &SessionId) -> Option<SessionActorHandle> {
509 self.sessions.lock().remove(id)
510 }
511
512 /// Session ids with no attached broadcast receivers (`receiver_count() == 0`) whose
513 /// `last_active` is at least `ttl` old — eviction candidates for a `serve.evict` task
514 /// (spec §9.3).
515 #[must_use]
516 pub fn idle_candidates(&self, ttl: Duration) -> Vec<SessionId> {
517 let sessions = self.sessions.lock();
518 let now = Instant::now();
519 sessions
520 .iter()
521 .filter(|(_, handle)| {
522 handle.tx_out.receiver_count() == 0 && now.duration_since(handle.last_active) >= ttl
523 })
524 .map(|(id, _)| id.clone())
525 .collect()
526 }
527
528 /// Ids of all live sessions currently tracked, in arbitrary order.
529 #[must_use]
530 pub fn ids(&self) -> Vec<SessionId> {
531 self.sessions.lock().keys().cloned().collect()
532 }
533
534 /// Number of live sessions currently tracked.
535 #[must_use]
536 pub fn len(&self) -> usize {
537 self.sessions.lock().len()
538 }
539
540 /// `true` when no sessions are tracked.
541 #[must_use]
542 pub fn is_empty(&self) -> bool {
543 self.len() == 0
544 }
545}
546
547#[cfg(test)]
548mod tests {
549 use std::time::Duration;
550
551 use super::*;
552
553 fn make_handle() -> SessionActorHandle {
554 let (tx, _rx) = mpsc::channel(4);
555 let (tx_out, _sub) = broadcast::channel(4);
556 SessionActorHandle {
557 tx,
558 tx_out,
559 last_active: Instant::now(),
560 cancel: CancellationToken::new(),
561 }
562 }
563
564 #[test]
565 fn registry_insert_and_get_round_trips() {
566 let registry = LiveSessionRegistry::new();
567 let id = SessionId::new("s1");
568 registry.insert(id.clone(), make_handle());
569 assert!(registry.get(&id).is_some());
570 assert_eq!(registry.len(), 1);
571 }
572
573 #[test]
574 fn registry_get_missing_returns_none() {
575 let registry = LiveSessionRegistry::new();
576 assert!(registry.get(&SessionId::new("nope")).is_none());
577 }
578
579 #[test]
580 fn registry_remove_drops_entry() {
581 let registry = LiveSessionRegistry::new();
582 let id = SessionId::new("s1");
583 registry.insert(id.clone(), make_handle());
584 assert!(registry.remove(&id).is_some());
585 assert!(registry.get(&id).is_none());
586 assert!(registry.is_empty());
587 }
588
589 #[test]
590 fn registry_remove_if_current_drops_matching_entry() {
591 let registry = LiveSessionRegistry::new();
592 let id = SessionId::new("s1");
593 let handle = make_handle();
594 let tx = handle.tx.clone();
595 registry.insert(id.clone(), handle);
596
597 assert!(registry.remove_if_current(&id, &tx).is_some());
598 assert!(registry.get(&id).is_none());
599 }
600
601 /// M1 regression: a stale coordinator's `remove_if_current` (its own now-dead handle's `tx`)
602 /// must NOT evict a fresh entry a concurrent reactivation (D-12) already `insert`ed under the
603 /// same id — the exact race a plain key-based `remove` would lose.
604 #[test]
605 fn registry_remove_if_current_ignores_stale_tx_after_reactivation() {
606 let registry = LiveSessionRegistry::new();
607 let id = SessionId::new("s1");
608 let stale_handle = make_handle();
609 let stale_tx = stale_handle.tx.clone();
610 registry.insert(id.clone(), stale_handle);
611
612 // A concurrent reactivation replaces the entry with a fresh handle under the same id.
613 registry.insert(id.clone(), make_handle());
614
615 // The stale coordinator's reap call must be a no-op — the fresh entry survives.
616 assert!(registry.remove_if_current(&id, &stale_tx).is_none());
617 assert!(
618 registry.get(&id).is_some(),
619 "a stale coordinator must never evict a concurrently-reactivated entry"
620 );
621 }
622
623 /// N1 regression (impl-critic re-verify finding): a genuine concurrency test, not a
624 /// sequential table-driven one — real `tokio::spawn` tasks race `get_or_reactivate` for the
625 /// same absent session id, each `.await`ing a `yield_now()` inside its `reactivate` closure
626 /// so they actually interleave (without the yield, the first task could run its whole
627 /// closure to completion before the second is even polled, defeating the point of the test).
628 /// Before the `reactivation_lock`, every task's fast-path `get()` would miss and every task
629 /// would run its `reactivate` closure — spawning N independent `SessionActorHandle`s that
630 /// each `insert` under the same id, the exact double-spawn-over-one-log scenario N1
631 /// describes. With the fix, exactly one task's closure must run to completion.
632 #[tokio::test]
633 async fn get_or_reactivate_serializes_concurrent_reactivation_for_the_same_id() {
634 use std::sync::atomic::{AtomicUsize, Ordering};
635
636 let registry = Arc::new(LiveSessionRegistry::new());
637 let id = SessionId::new("race-test");
638 let reactivate_calls = Arc::new(AtomicUsize::new(0));
639
640 let mut tasks = Vec::new();
641 for _ in 0..8 {
642 let registry = Arc::clone(®istry);
643 let id = id.clone();
644 let reactivate_calls = Arc::clone(&reactivate_calls);
645 tasks.push(tokio::spawn(async move {
646 registry
647 .get_or_reactivate(&id, || {
648 let registry = Arc::clone(®istry);
649 let id = id.clone();
650 let reactivate_calls = Arc::clone(&reactivate_calls);
651 async move {
652 // Force a real interleaving window — without this, tasks could
653 // resolve strictly in spawn order without ever actually contending
654 // for `reactivation_lock`.
655 tokio::task::yield_now().await;
656 reactivate_calls.fetch_add(1, Ordering::SeqCst);
657 let handle = make_handle();
658 registry.insert(id, handle.clone());
659 Some(handle)
660 }
661 })
662 .await
663 }));
664 }
665
666 for task in tasks {
667 assert!(
668 task.await.unwrap().is_some(),
669 "every concurrent caller must resolve to a live handle, win or lose the race"
670 );
671 }
672
673 assert_eq!(
674 reactivate_calls.load(Ordering::SeqCst),
675 1,
676 "exactly one concurrent caller may run the reactivation closure — a second run means \
677 two SessionActors would have been spawned over the same durable log (N1)"
678 );
679 }
680
681 #[test]
682 fn registry_ids_lists_all_tracked_sessions() {
683 let registry = LiveSessionRegistry::new();
684 assert!(registry.ids().is_empty());
685 registry.insert(SessionId::new("s1"), make_handle());
686 registry.insert(SessionId::new("s2"), make_handle());
687 let mut ids: Vec<String> = registry
688 .ids()
689 .into_iter()
690 .map(|id| id.as_str().to_owned())
691 .collect();
692 ids.sort();
693 assert_eq!(ids, vec!["s1".to_owned(), "s2".to_owned()]);
694 }
695
696 #[test]
697 fn registry_idle_candidates_requires_no_subscribers_and_expired_ttl() {
698 let registry = LiveSessionRegistry::new();
699 let id = SessionId::new("s1");
700 let mut handle = make_handle();
701 // No subscriber to tx_out, but last_active is "now" — not yet past a long TTL.
702 handle.last_active = Instant::now();
703 registry.insert(id.clone(), handle);
704 assert!(registry.idle_candidates(Duration::from_hours(1)).is_empty());
705 }
706
707 #[test]
708 fn registry_idle_candidates_skips_sessions_with_active_subscribers() {
709 let registry = LiveSessionRegistry::new();
710 let id = SessionId::new("s1");
711 let mut handle = make_handle();
712 handle.last_active = Instant::now()
713 .checked_sub(Duration::from_secs(9999))
714 .unwrap();
715 let _subscriber = handle.tx_out.subscribe();
716 registry.insert(id, handle);
717 assert!(registry.idle_candidates(Duration::from_secs(1)).is_empty());
718 }
719
720 #[test]
721 fn registry_idle_candidates_returns_expired_unattached_sessions() {
722 let registry = LiveSessionRegistry::new();
723 let id = SessionId::new("s1");
724 let mut handle = make_handle();
725 handle.last_active = Instant::now()
726 .checked_sub(Duration::from_secs(9999))
727 .unwrap();
728 registry.insert(id.clone(), handle);
729 let candidates = registry.idle_candidates(Duration::from_secs(1));
730 assert_eq!(candidates, vec![id]);
731 }
732
733 #[tokio::test]
734 async fn session_actor_drive_shuts_down_cleanly_on_command() {
735 use crate::agent::Agent;
736 use crate::agent::agent_tests::{MockToolExecutor, create_test_registry, mock_provider};
737
738 // `LoopbackChannel::pair` gives exactly the (channel, handle) split `SessionActor::drive`
739 // bridges between — the same split every other headless channel consumer (A2A) uses.
740 let (channel, handle) = LoopbackChannel::pair(8);
741 let provider = mock_provider(vec!["ok".to_owned()]);
742 let registry = create_test_registry();
743 let executor = MockToolExecutor::no_tools();
744 let agent: Agent<LoopbackChannel> =
745 Agent::new(provider, channel, registry, None, 5, executor);
746
747 let (cmd_tx, cmd_rx) = mpsc::channel::<SessionCommand>(4);
748 let (tx_out, _sub) = broadcast::channel::<SessionOutput>(16);
749
750 // `Agent<C>`'s futures are `!Send` (see the module doc), so `drive`'s future cannot be
751 // handed to `tokio::spawn`. Pre-queue both commands into the bounded mpsc buffer, then
752 // `.await` `drive` directly in this test task — no cross-thread Send requirement applies
753 // to a future that is only ever polled in place, never spawned.
754 cmd_tx
755 .send(SessionCommand::Prompt {
756 text: "hello".to_owned(),
757 })
758 .await
759 .unwrap();
760 cmd_tx.send(SessionCommand::Shutdown).await.unwrap();
761 drop(cmd_tx);
762
763 // `drive`'s future embeds the whole `Agent` state (large) — box it per
764 // `clippy::large_futures` rather than growing this test task's stack.
765 tokio::time::timeout(
766 Duration::from_secs(10),
767 Box::pin(SessionActor::drive(
768 agent,
769 handle,
770 cmd_rx,
771 tx_out,
772 CancellationToken::new(),
773 )),
774 )
775 .await
776 .expect("drive must finish within the timeout");
777 }
778
779 #[tokio::test]
780 async fn session_actor_spawn_runs_on_dedicated_thread_and_shuts_down() {
781 use crate::agent::Agent;
782 use crate::agent::agent_tests::{MockToolExecutor, create_test_registry, mock_provider};
783
784 let supervisor = TaskSupervisor::new(CancellationToken::new());
785 let registry = Arc::new(LiveSessionRegistry::new());
786 let session_id = SessionId::new("spawn-test");
787
788 // `build_agent` is `Send` (captures only `Send`-safe test doubles); the `Agent` it
789 // constructs is built *inside* the dedicated thread `spawn` creates from the
790 // caller-supplied `LoopbackChannel`, never crossing a thread boundary itself.
791 let (handle, blocking_handle) = SessionActor::spawn(
792 &supervisor,
793 ®istry,
794 &session_id,
795 move |channel| {
796 let provider = mock_provider(vec!["ok".to_owned()]);
797 let registry = create_test_registry();
798 let executor = MockToolExecutor::no_tools();
799 let agent: Agent<LoopbackChannel> =
800 Agent::new(provider, channel, registry, None, 5, executor);
801 agent
802 },
803 4,
804 );
805
806 handle
807 .tx
808 .send(SessionCommand::Prompt {
809 text: "hello".to_owned(),
810 })
811 .await
812 .unwrap();
813 handle.tx.send(SessionCommand::Shutdown).await.unwrap();
814
815 tokio::time::timeout(Duration::from_secs(10), blocking_handle.join())
816 .await
817 .expect("session actor must finish within the timeout")
818 .expect("session actor task must not panic or be aborted");
819
820 // M1 (impl-critic finding): the coordinator must reap its own registry entry once the
821 // dedicated thread signals completion, not only via `idle_candidates`'s
822 // `receiver_count() == 0` TTL path — otherwise a session whose actor died with no
823 // eviction ever running stays permanently registered as "live" with a dead mailbox.
824 assert!(
825 registry.get(&session_id).is_none(),
826 "registry entry must be reaped once the session actor's coordinator completes"
827 );
828 }
829
830 #[tokio::test]
831 async fn session_actor_handle_cancel_shuts_down_without_supervisor_shutdown() {
832 use crate::agent::Agent;
833 use crate::agent::agent_tests::{MockToolExecutor, create_test_registry, mock_provider};
834
835 // Regression test for the D-8 per-session cancellation path: `serve.evict` idle eviction
836 // cancels one session's own `SessionActorHandle::cancel` — it must terminate that actor
837 // without the supervisor's own token ever being cancelled (i.e. without a process-wide
838 // shutdown), proving the two cancellation sources are genuinely independent.
839 let supervisor = TaskSupervisor::new(CancellationToken::new());
840 let registry = Arc::new(LiveSessionRegistry::new());
841 let session_id = SessionId::new("cancel-test");
842
843 let (handle, blocking_handle) = SessionActor::spawn(
844 &supervisor,
845 ®istry,
846 &session_id,
847 move |channel| {
848 let provider = mock_provider(vec!["ok".to_owned()]);
849 let registry = create_test_registry();
850 let executor = MockToolExecutor::no_tools();
851 Agent::new(provider, channel, registry, None, 5, executor)
852 },
853 4,
854 );
855
856 // No `SessionCommand::Shutdown` sent — cancel the per-session token directly, as idle
857 // eviction would.
858 handle.cancel.cancel();
859
860 tokio::time::timeout(Duration::from_secs(10), blocking_handle.join())
861 .await
862 .expect("session actor must finish within the timeout after its own token cancels")
863 .expect("session actor task must not panic or be aborted");
864 }
865
866 /// Samples this process's own RSS via `sysinfo` — the same technique
867 /// `system_metrics::spawn_system_metrics_task` uses in production — so results are directly
868 /// comparable to other in-process RSS measurements.
869 fn sample_rss(sys: &mut sysinfo::System, pid: sysinfo::Pid) -> u64 {
870 sys.refresh_processes(sysinfo::ProcessesToUpdate::Some(&[pid]), true);
871 sys.process(pid).map_or(0, sysinfo::Process::memory)
872 }
873
874 // TODO(critic): decompose idle floor (stack-resident vs Agent heap vs skill registry) and
875 // add a production-realistic (non-mock) upper-bound variant.
876 //
877 /// NFR-P7 follow-up (#5840): the #5445 standalone harness measured `SessionActor::spawn`'s
878 /// structural housing cost in isolation (thread stack, runtime, channels — ~65-93 KiB/actor,
879 /// see `specs/068-session-persistence/nfr.md`) but could not construct a real
880 /// `Agent<LoopbackChannel>`, which is private to this crate. This test closes that gap: it
881 /// spawns real `SessionActor`s wrapping real (mock-provider-backed, since no live LLM/Qdrant
882 /// is needed to measure idle housing) `Agent<LoopbackChannel>` instances in-process, so the
883 /// measured RSS also includes `Agent`'s own owned state — not just the actor's housing.
884 ///
885 /// **This is a zero-conversation-turn floor** (no prompts are ever sent), so the ~875-905 KiB
886 /// measured here is *not* `Agent`'s message-history buffer, which is empty throughout. The
887 /// dominant contributor is not yet decomposed (see the `TODO` above) but is more likely extra
888 /// resident pages of the actor's 8 MiB thread stack ([`SESSION_ACTOR_STACK_SIZE`]) touched by
889 /// the deeper `Agent::new`/`drive` call chain, and/or the shared `SkillRegistry` load —
890 /// neither of which #5445's Agent-less harness ever touched.
891 ///
892 /// **Lower bound, not an upper bound**: `mock_provider` has no HTTP client/connection pool,
893 /// [`MockToolExecutor::no_tools`] carries no tool definitions, and the shared registry below
894 /// loads exactly one trivial skill with no embedding vectors. A production session's real
895 /// `SkillRegistry` (many skills + embeddings), real provider (reqwest client + pool), and any
896 /// `SemanticMemory` state will all measure higher than what this test reports — a pass here is
897 /// not proof that a production idle session stays under NFR-P7's budget.
898 ///
899 /// **Composite vs. NFR-P7's own scope**: `specs/068-session-persistence/nfr.md`'s NFR-P7 is
900 /// defined as housing-only (thread stack + runtime + channels); the composite this test
901 /// measures (housing + `Agent`'s owned state) is a distinct, larger quantity the spec's #5445
902 /// rationale explicitly separates out. The `assert!` below reuses NFR-P7's 1 MiB number as a
903 /// convenience threshold for this composite measurement, not as a formal claim that NFR-P7
904 /// (as specified) is satisfied — see the "NFR-P7 follow-up (#5840)" rationale note in
905 /// `nfr.md` for the recorded distinction. This test is also `#[ignore]`d and matched by no CI
906 /// workflow filter, so it never runs automatically — it is a manual tripwire for whoever
907 /// invokes it by hand, not an automated regression gate.
908 ///
909 /// Mirrors the #5445 harness's approach of measuring marginal RSS across growing cumulative
910 /// batches (10, 25, 50, 100 actors) so one-time fixed costs (allocator warmup, first-touch
911 /// page faults) amortize out. The reported floor is the delta between the *last two*
912 /// checkpoints (50→100), not an earlier window — the 50→100 and 25→50 deltas agree to within
913 /// noise (confirming convergence), whereas the 10→25 window still carries first-batch fixed
914 /// costs and reads noticeably higher; using the converged tail avoids attributing one-time
915 /// warmup cost to the per-session marginal figure.
916 #[tokio::test(flavor = "multi_thread")]
917 #[ignore = "spawns up to 100 real OS threads; run explicitly to verify NFR-P7, e.g. \
918 `cargo nextest run -p zeph-core -E 'test(nfr_p7)' --run-ignored ignored-only \
919 --no-capture`"]
920 async fn nfr_p7_real_agent_idle_session_memory_floor() {
921 use crate::agent::Agent;
922 use crate::agent::agent_tests::{MockToolExecutor, create_test_registry, mock_provider};
923
924 let supervisor = TaskSupervisor::new(CancellationToken::new());
925 let registry = Arc::new(LiveSessionRegistry::new());
926 let pid = sysinfo::get_current_pid().expect("current pid must be resolvable");
927 let mut sys = sysinfo::System::new();
928
929 // Shared across every spawned Agent, mirroring production (`src/serve/agent_factory.rs`'s
930 // `build_agent_factory`, which passes one `Clone`-shared `deps.registry` to
931 // `Agent::new_with_registry_arc` for every session) rather than each actor building its
932 // own private registry via the simpler `Agent::new`. A 101st real session costs one `Arc`
933 // clone, not a whole new registry — this keeps the measured marginal cost representative
934 // of production instead of overstating it with a per-actor registry that doesn't scale.
935 let skill_registry = Arc::new(parking_lot::RwLock::new(create_test_registry()));
936
937 let mut actors = Vec::new();
938 let mut checkpoints: Vec<(usize, u64)> = Vec::new();
939
940 for target in [10usize, 25, 50, 100] {
941 while actors.len() < target {
942 let session_id = SessionId::new(format!("nfr-p7-{}", actors.len()));
943 let shared_registry = Arc::clone(&skill_registry);
944 let (handle, blocking) = SessionActor::spawn(
945 &supervisor,
946 ®istry,
947 &session_id,
948 move |channel| {
949 let provider = mock_provider(vec!["ok".to_owned()]);
950 let embedding_provider = provider.clone();
951 let executor = MockToolExecutor::no_tools();
952 Agent::new_with_registry_arc(
953 provider,
954 embedding_provider,
955 channel,
956 shared_registry,
957 None,
958 5,
959 executor,
960 )
961 },
962 4,
963 );
964 actors.push((handle, blocking));
965 }
966 // Let newly spawned dedicated threads finish constructing their Agent and settle into
967 // `drive`'s idle `select!` loop before sampling.
968 tokio::time::sleep(Duration::from_millis(200)).await;
969 checkpoints.push((target, sample_rss(&mut sys, pid)));
970 }
971
972 for (handle, _) in &actors {
973 let _ = handle.tx.send(SessionCommand::Shutdown).await;
974 }
975 for (_, blocking) in actors {
976 let _ = tokio::time::timeout(Duration::from_secs(10), blocking.join()).await;
977 }
978
979 let (n_prev, rss_prev) = checkpoints[checkpoints.len() - 2];
980 let (n_last, rss_last) = checkpoints[checkpoints.len() - 1];
981 let marginal_per_session = rss_last.saturating_sub(rss_prev) / (n_last - n_prev) as u64;
982
983 eprintln!("NFR-P7 real-Agent memory floor checkpoints: {checkpoints:?}");
984 eprintln!(
985 "NFR-P7 real-Agent per-session marginal floor (housing + Agent owned state, \
986 synthetic mock-backed lower bound): {marginal_per_session} bytes ({} KiB)",
987 marginal_per_session / 1024
988 );
989
990 assert!(
991 marginal_per_session < 1_048_576,
992 "NFR-P7 composite budget (informational threshold, see nfr.md's #5840 rationale) \
993 exceeded: measured {marginal_per_session} bytes/session >= 1 MiB \
994 (checkpoints: {checkpoints:?})"
995 );
996 }
997}