Skip to main content

term_session_server/
session_server.rs

1use std::collections::HashMap;
2use std::sync::Arc;
3use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
4
5use muxio_core::rpc::rpc_internals::RpcStreamEvent;
6use muxio_rpc_service::prebuffered::RpcMethodPrebuffered;
7use muxio_rpc_service_caller::prebuffered::RpcCallPrebuffered;
8use muxio_rpc_service_endpoint::{RpcServiceEndpointInterface, StreamResponder};
9use muxio_tokio_rpc_ipc_server::{RpcIpcConnectionContextHandle, RpcIpcServer, RpcIpcServerEvent};
10use portable_pty::PtySize;
11use tokio::sync::{Mutex, Notify, RwLock, mpsc, oneshot};
12
13use term_session_muxio_service_definitions::{
14    Attach, ChannelInfo, ChannelName, ClientInfo, CloseSession, KillChannel, KillClient,
15    ListChannels, ListChannelsResponse, OnPtyResized, RPC_ERROR_LIVE_PARTICIPANTS,
16    RPC_ERROR_LIVE_SESSIONS, RPC_ERROR_SHUTTING_DOWN, RPC_ERROR_UNATTACHED, ResizePty,
17    STREAM_INPUT_METHOD_ID, SUBSCRIBE_OUTPUT_METHOD_ID, SessionInfo, ShutdownGateway, Spawn,
18    SpawnRequest, SpawnResponse, WriteInput,
19};
20use term_wm_pty_engine::PtyStatus;
21
22use crate::session::Session;
23
24/// Session id per channel (each channel hosts a single PTY at a time).
25const SESSION_ID: u64 = 1;
26/// Bounded input channel capacity — memory safety against extreme input bursts.
27const INPUT_CHANNEL_CAPACITY: usize = 128;
28
29/// Grace period to let the transport flush end-of-stream frames after the
30/// session exits, before the gateway process terminates.
31const SESSION_EXIT_FLUSH_GRACE: std::time::Duration = std::time::Duration::from_millis(100);
32
33/// How often the output polling task wakes to re-check the session's exit
34/// status, as a fallback for a missed or raced PTY EOF notification.
35const SESSION_EXIT_POLL_INTERVAL: std::time::Duration = std::time::Duration::from_millis(100);
36
37/// Upper bound on the per-channel `output_cache` retained when a session exits
38/// with no subscribers attached. Only the last `MAX_RETAINED_OUTPUT_BYTES` of
39/// the session's final output are kept, bounding heap memory regardless of how
40/// much an unsubscribed background session emitted before terminating.
41const MAX_RETAINED_OUTPUT_BYTES: usize = 64 * 1024;
42
43/// How long to wait for the PTY reader thread to finish its EOF processing
44/// before draining a dead session's final output. On Unix the reader EOFs
45/// within microseconds of process exit, so this is effectively free; on Windows
46/// ConPTY (where EOF can be swallowed) the grace bounds the wait before a
47/// best-effort drain.
48const READER_DRAIN_GRACE: std::time::Duration = std::time::Duration::from_millis(50);
49
50/// Grace between SIGTERM and SIGKILL when terminating a session's process tree.
51const SIGKILL_GRACE: std::time::Duration = std::time::Duration::from_millis(500);
52
53/// SIGTERM for cooperative process-group termination. On non-Unix platforms
54/// the value is unused (kill paths fall back to `kill_child`); it is kept a
55/// named `const` so the call sites read identically.
56#[cfg(unix)]
57const SIGTERM: i32 = libc::SIGTERM;
58#[cfg(not(unix))]
59const SIGTERM: i32 = 15;
60/// SIGKILL for straggler escalation after the grace window.
61#[cfg(unix)]
62const SIGKILL: i32 = libc::SIGKILL;
63#[cfg(not(unix))]
64#[allow(dead_code)]
65const SIGKILL: i32 = 9;
66
67/// Grace before the gateway exits after the last ShutdownGateway response is
68/// flushed: the RPC handler returns immediately and a detached task sleeps
69/// this long so the muxio transport drains the `()` frame before the socket
70/// is torn down.
71const SHUTDOWN_FLUSH_GRACE_MS: u64 = 50;
72
73/// A connection's bind state. Identity is server-assigned (`conn_id`); a
74/// connection must `Attach` before it may spawn/resize/write.
75#[derive(Clone)]
76enum ConnState {
77    Unattached,
78    Attached(ChannelName),
79}
80
81#[derive(Clone)]
82struct ConnEntry {
83    handle: RpcIpcConnectionContextHandle,
84    state: ConnState,
85    hostname: String,
86    connected_at_unix: u64,
87    /// Client process OS PID (reported at Attach).
88    pid: u64,
89    /// OS user running the client process (reported at Attach).
90    user: String,
91    /// Client binary version (reported at Attach).
92    version: String,
93    /// Remote peer IP for SSH attaches; `None` for local (reported at Attach).
94    ssh_ip: Option<String>,
95}
96
97#[derive(Clone)]
98struct ClientEntry {
99    caller: Option<RpcIpcConnectionContextHandle>,
100    hostname: String,
101    connected_at_unix: u64,
102    pid: u64,
103    user: String,
104    version: String,
105    ssh_ip: Option<String>,
106    cols: u16,
107    rows: u16,
108}
109
110struct SubscriberEntry {
111    conn_id: usize,
112    respond: StreamResponder,
113}
114
115/// Per-channel state. One gateway process hosts many channels; each channel
116/// owns its own session, connected clients, subscribers, and input channel.
117struct ChannelState {
118    session: Option<Session>,
119    clients: HashMap<usize, ClientEntry>,
120    subscribers: Vec<SubscriberEntry>,
121    notify: Arc<Notify>,
122    /// Unix seconds when the channel was first created on the gateway.
123    created_at_unix: u64,
124    /// Final output retained when a session exits with no subscribers
125    /// attached, so a later subscriber can still receive it. Capped at
126    /// `MAX_RETAINED_OUTPUT_BYTES` (tail retention) and cleared on respawn.
127    output_cache: Vec<u8>,
128    /// Monotonic creation sequence (across the whole gateway process). Sort key
129    /// for `ListChannels`: creation order, newest last. `created_at_unix` is
130    /// only second-resolution wall clock and cannot disambiguate same-second
131    /// creations, so this monotonic counter is authoritative.
132    created_seq: u64,
133    /// Command template used to respawn the session after it exits.
134    cmd: Vec<String>,
135    input_tx: mpsc::Sender<Vec<u8>>,
136    /// True between a SIGTERM request and the process group's actual exit (or
137    /// the SIGKILL escalation). Cleared when the session is observed exited.
138    kill_pending: bool,
139    /// Tombstone set by GC just before removal; any Attach that read a stale
140    /// Arc re-checks this under the write lock (safe double-checked locking).
141    is_reaped: bool,
142}
143
144/// Gateway coordination. Two tiers:
145/// - `conns` is a read-mostly routing table (RwLock).
146/// - `channels` maps channel names to independently-mutexed channel states.
147///
148/// The gateway never holds more than one lock at a time (resolve-and-drop).
149struct ServerState {
150    conns: RwLock<HashMap<usize, ConnEntry>>,
151    channels: RwLock<HashMap<ChannelName, Arc<Mutex<ChannelState>>>>,
152    is_shutting_down: AtomicBool,
153    /// Monotonic source of `ChannelState::created_seq` (creation order).
154    next_channel_seq: AtomicU64,
155    /// Per-connection ordered input forwarders. The synchronous `StreamInput`
156    /// handler forwards each chunk into this queue in wire order (the handler
157    /// is invoked sequentially per stream by muxio); a single consumer task
158    /// per connection drains it FIFO into the channel's `input_tx`, so bursty
159    /// input (e.g. IME voice typing) is never reordered by racing tasks.
160    input_forwarders: std::sync::Mutex<HashMap<usize, mpsc::Sender<Vec<u8>>>>,
161}
162
163type SharedState = Arc<ServerState>;
164
165fn rpc_err(message: &str) -> Box<dyn std::error::Error + Send + Sync> {
166    Box::new(std::io::Error::other(message.to_string()))
167}
168
169/// Lift an `io::Error` into the boxed handler error type.
170fn boxed_io(e: std::io::Error) -> Box<dyn std::error::Error + Send + Sync> {
171    Box::new(e)
172}
173
174/// Unix seconds for a connection's `connected_at_unix` wire field.
175fn now_unix() -> u64 {
176    std::time::SystemTime::now()
177        .duration_since(std::time::UNIX_EPOCH)
178        .map(|d| d.as_secs())
179        .unwrap_or(0)
180}
181
182impl ChannelState {
183    fn new(
184        cmd: Vec<String>,
185        input_tx: mpsc::Sender<Vec<u8>>,
186        notify: Arc<Notify>,
187        created_seq: u64,
188    ) -> Self {
189        Self {
190            session: None,
191            clients: HashMap::new(),
192            subscribers: Vec::new(),
193            notify,
194            created_at_unix: now_unix(),
195            output_cache: Vec::new(),
196            created_seq,
197            cmd,
198            input_tx,
199            kill_pending: false,
200            is_reaped: false,
201        }
202    }
203
204    /// Replace the current session and attach the Notify callback so the
205    /// background polling task wakes on PTY output.
206    fn set_session(&mut self, mut session: Session) {
207        // A respawned session must not replay the previous session's retained
208        // final output to new subscribers.
209        self.output_cache.clear();
210        let n = self.notify.clone();
211        session.set_status_callback(Some(Box::new(move |status| {
212            if matches!(status, PtyStatus::Wakeup | PtyStatus::Exited) {
213                n.notify_one();
214            }
215        })));
216        self.session = Some(session);
217        // Prime notify to process initial startup output generated before the
218        // callback was registered.
219        self.notify.notify_one();
220    }
221
222    /// Append bytes to the retained-output cache with tail retention: at most
223    /// `MAX_RETAINED_OUTPUT_BYTES` of the session's final output is ever kept,
224    /// so an unsubscribed high-volume session cannot grow the cache unboundedly.
225    fn retain_final_output(&mut self, bytes: &[u8]) {
226        if bytes.len() >= MAX_RETAINED_OUTPUT_BYTES {
227            self.output_cache.clear();
228            self.output_cache
229                .extend_from_slice(&bytes[bytes.len() - MAX_RETAINED_OUTPUT_BYTES..]);
230        } else if self.output_cache.len() + bytes.len() > MAX_RETAINED_OUTPUT_BYTES {
231            let drop = self.output_cache.len() + bytes.len() - MAX_RETAINED_OUTPUT_BYTES;
232            self.output_cache.drain(..drop);
233            self.output_cache.extend_from_slice(bytes);
234        } else {
235            self.output_cache.extend_from_slice(bytes);
236        }
237    }
238
239    /// Signal the session's process group (non-blocking) and arm the kill
240    /// escalation flag. The caller is responsible for spawning the detached
241    /// escalation task (see `spawn_kill_escalation`). Mechanism only — no
242    /// sleeps, no waits, no `SIGKILL` here.
243    fn request_session_kill(&mut self, signal: i32) {
244        let _ = &signal;
245        if let Some(session) = self.session.as_mut() {
246            #[cfg(unix)]
247            let _ = session.pty.signal_process_group(signal);
248            #[cfg(not(unix))]
249            let _ = session.pty.kill_child();
250        }
251        self.kill_pending = true;
252        self.notify.notify_one();
253    }
254
255    /// Flush remaining PTY buffers and stream completion markers to all active
256    /// subscribers, then drop them. Used by kill paths and on session exit.
257    fn finalize_subscribers(&mut self) {
258        if let Some(session) = self.session.as_mut() {
259            let raw = session.read_output();
260            if !raw.is_empty() {
261                for sub in &self.subscribers {
262                    sub.respond.respond(raw.clone(), false);
263                }
264            }
265        }
266        for sub in &self.subscribers {
267            sub.respond.respond(Vec::new(), true);
268        }
269        self.subscribers.clear();
270    }
271
272    /// Constrain the PTY to the smallest geometry across all connected clients.
273    /// This guarantees the virtual buffer never exceeds any attached monitor.
274    ///
275    /// Geometry is strictly client-driven: if no connected client has reported
276    /// real dimensions yet (all are still `u16::MAX`), nothing is constrained
277    /// and the session keeps its spawn-time size. No hardcoded default size is
278    /// ever imposed.
279    fn recalculate_pty_size(&mut self) {
280        let Some(session) = self.session.as_mut() else {
281            return;
282        };
283        let Some(min_cols) = self
284            .clients
285            .values()
286            .map(|c| c.cols)
287            .filter(|&c| c != u16::MAX)
288            .min()
289        else {
290            return;
291        };
292        let Some(min_rows) = self
293            .clients
294            .values()
295            .map(|c| c.rows)
296            .filter(|&r| r != u16::MAX)
297            .min()
298        else {
299            return;
300        };
301        let size = PtySize {
302            rows: min_rows,
303            cols: min_cols,
304            pixel_width: 0,
305            pixel_height: 0,
306        };
307        let _ = session.pty.resize(size);
308        session.cols = min_cols;
309        session.rows = min_rows;
310    }
311
312    /// Broadcast geometry to all connected clients via detached async tasks.
313    /// Call AFTER releasing the channel lock.
314    fn notify_clients(&self, clients: &[ClientEntry], cols: u16, rows: u16) {
315        for client in clients {
316            let Some(caller) = client.caller.clone() else {
317                continue;
318            };
319            tokio::spawn(async move {
320                if let Err(e) = OnPtyResized::call(&caller, (cols, rows)).await {
321                    tracing::debug!(error = ?e, "Failed to deliver OnPtyResized notification");
322                }
323            });
324        }
325    }
326
327    fn to_info(&self, name: &ChannelName) -> ChannelInfo {
328        let session = self.session.as_ref().map(|s| SessionInfo {
329            id: s.id,
330            cols: s.cols,
331            rows: s.rows,
332            exited: s.exited,
333            exit_code: s.exit_code,
334            title: s.title.clone().unwrap_or_default(),
335        });
336        // Sort by `conn_id`, which muxio assigns monotonically at connection
337        // accept — ascending order = connection order, newest client last.
338        let mut clients: Vec<ClientInfo> = self
339            .clients
340            .iter()
341            .map(|(conn_id, c)| ClientInfo {
342                conn_id: *conn_id,
343                pid: c.pid,
344                hostname: c.hostname.clone(),
345                connected_at_unix: c.connected_at_unix,
346                cols: c.cols,
347                rows: c.rows,
348                user: c.user.clone(),
349                version: c.version.clone(),
350                ssh_ip: c.ssh_ip.clone(),
351            })
352            .collect();
353        clients.sort_by_key(|c| c.conn_id);
354        ChannelInfo {
355            name: name.to_string(),
356            created_at_unix: self.created_at_unix,
357            session,
358            clients,
359        }
360    }
361}
362
363/// Resolve the channel a connection is bound to, or `None` if unattached.
364async fn bound_channel(state: &ServerState, conn_id: usize) -> Option<ChannelName> {
365    let conns = state.conns.read().await;
366    match conns.get(&conn_id)?.state {
367        ConnState::Attached(ref name) => Some(name.clone()),
368        ConnState::Unattached => None,
369    }
370}
371
372/// Fetch an `Arc<ChannelState>` for the channel, resolving then dropping the
373/// routing guard (never holding `conns` while locking the channel).
374async fn resolve_channel(
375    state: &ServerState,
376    name: &ChannelName,
377) -> Option<Arc<Mutex<ChannelState>>> {
378    let channels = state.channels.read().await;
379    channels.get(name).cloned()
380}
381
382/// Drain one connection's ordered input queue, forwarding chunks to the
383/// bound channel's `input_tx` in exact arrival order. Exits when the forwarder
384/// sender is dropped (connection End/Error, eviction, or Attach re-bind) or
385/// the channel's input receiver closes. Exactly one of these tasks exists per
386/// connection, so chunk ordering is preserved end-to-end even under bursts.
387///
388/// Chunks queued during bursts (e.g. mouse drags or IME voice typing) are
389/// coalesced via non-blocking `try_recv` before forwarding, and `input_tx` is
390/// cached across chunks to eliminate per-chunk routing lookup overhead.
391async fn drain_input_forwarder(
392    state: SharedState,
393    conn_id: usize,
394    mut rx: mpsc::Receiver<Vec<u8>>,
395) {
396    let mut cached_tx: Option<mpsc::Sender<Vec<u8>>> = None;
397
398    while let Some(mut bytes) = rx.recv().await {
399        // Coalesce any additional chunks currently queued in the forwarder channel.
400        while let Ok(mut next) = rx.try_recv() {
401            bytes.append(&mut next);
402        }
403
404        // Re-resolve the target channel's `input_tx` if not cached or closed.
405        if cached_tx.as_ref().is_none_or(|tx| tx.is_closed()) {
406            cached_tx = None;
407            if let Some(channel) = bound_channel(state.as_ref(), conn_id).await
408                && let Some(ch) = resolve_channel(state.as_ref(), &channel).await
409            {
410                let guard = ch.lock().await;
411                if !guard.is_reaped {
412                    cached_tx = Some(guard.input_tx.clone());
413                }
414            }
415        }
416
417        if let Some(ref tx) = cached_tx {
418            // Backpressure: `input_tx` is bounded (INPUT_CHANNEL_CAPACITY); a full
419            // buffer parks this consumer instead of silently dropping the chunk.
420            if tx.send(bytes).await.is_err() {
421                cached_tx = None;
422            }
423        }
424    }
425}
426
427/// Create (or fetch, re-verifying under the write lock) a channel.
428/// Safe double-checked locking: a racing Attach that saw a reaped Arc is
429/// redirected to the canonical instance instead of overwriting it.
430async fn get_or_create_channel(
431    state: &SharedState,
432    name: &ChannelName,
433) -> Arc<Mutex<ChannelState>> {
434    {
435        let channels = state.channels.read().await;
436        if let Some(existing) = channels.get(name) {
437            let arc = existing.clone();
438            drop(channels);
439            let is_reaped = arc.lock().await.is_reaped;
440            if !is_reaped {
441                return arc;
442            }
443        }
444    }
445
446    let mut channels = state.channels.write().await;
447    // Re-check under the write lock: a racing thread may have inserted a
448    // non-reaped channel between our read and write acquisition. A reaped
449    // (or absent) entry falls through and is replaced with a fresh channel.
450    if let Some(existing) = channels.get(name) {
451        let arc = existing.clone();
452        let is_reaped = arc.lock().await.is_reaped;
453        if !is_reaped {
454            return arc;
455        }
456    }
457    let (input_tx, input_rx) = mpsc::channel::<Vec<u8>>(INPUT_CHANNEL_CAPACITY);
458    let notify = Arc::new(Notify::new());
459    let created_seq = state.next_channel_seq.fetch_add(1, Ordering::Relaxed);
460    let channel = Arc::new(Mutex::new(ChannelState::new(
461        Vec::new(),
462        input_tx,
463        notify,
464        created_seq,
465    )));
466    let ch = Arc::clone(&channel);
467    tokio::spawn(async move {
468        let mut input_rx = input_rx;
469        while let Some(mut data) = input_rx.recv().await {
470            // Coalesce any additional chunks currently queued in input_rx so a
471            // burst of tiny chunks (e.g. mouse drags) is written to the PTY in
472            // a single blocking call instead of one spawn_blocking per chunk.
473            while let Ok(mut next) = input_rx.try_recv() {
474                data.append(&mut next);
475            }
476
477            let writer = {
478                let guard = ch.lock().await;
479                guard.session.as_ref().map(|s| s.pty.writer_handle())
480            };
481            if let Some(writer) = writer {
482                let _ = tokio::task::spawn_blocking(move || writer.write_bytes(&data)).await;
483            }
484        }
485        // Note: the input task runs for the daemon lifetime; the channel's
486        // `input_rx` is only dropped when the channel's senders all vanish.
487    });
488
489    // Output polling task: drains PTY output and broadcasts to subscribers;
490    // on session exit finalizes subscribers, clears the session, and reaps
491    // the channel if no clients remain.
492    {
493        let st = Arc::clone(state);
494        let ch = Arc::clone(&channel);
495        let notify = {
496            let locked = ch.lock().await;
497            locked.notify.clone()
498        };
499        let name_for_task = name.clone();
500        tokio::spawn(async move {
501            loop {
502                tokio::select! {
503                    _ = notify.notified() => {}
504                    _ = tokio::time::sleep(SESSION_EXIT_POLL_INTERVAL) => {}
505                }
506                let mut guard = ch.lock().await;
507                if guard.is_reaped {
508                    break;
509                }
510                if guard.subscribers.is_empty() {
511                    if let Some(session) = guard.session.as_mut() {
512                        session.sync_screen();
513                        if session.check_exited() {
514                            tracing::info!(channel = %name_for_task, "Session exited");
515                            // Retain the session's final output (bounded by
516                            // MAX_RETAINED_OUTPUT_BYTES) so a subscriber that
517                            // attaches after teardown still receives it, instead
518                            // of dropping the Pty's pending buffer wholesale.
519                            let final_out = session.read_final_output(READER_DRAIN_GRACE);
520                            guard.retain_final_output(&final_out);
521                            guard.session = None;
522                            guard.kill_pending = false;
523                        }
524                    }
525                } else {
526                    let (raw, exited, code) = {
527                        let Some(session) = guard.session.as_mut() else {
528                            // No live session: finalize any lingering subscribers.
529                            for sub in &guard.subscribers {
530                                sub.respond.respond(Vec::new(), true);
531                            }
532                            guard.subscribers.clear();
533                            guard.notify.notify_one();
534                            continue;
535                        };
536                        let raw = session.read_output();
537                        let exited = session.check_exited();
538                        let code = session.exit_code;
539                        (raw, exited, code)
540                    };
541                    if !raw.is_empty() {
542                        for sub in &guard.subscribers {
543                            sub.respond.respond(raw.clone(), false);
544                        }
545                    }
546                    if exited {
547                        tracing::info!(channel = %name_for_task, "Session exited with code {:?}", code);
548                        for sub in &guard.subscribers {
549                            sub.respond.respond(Vec::new(), true);
550                        }
551                        guard.subscribers.clear();
552                        guard.session = None;
553                        guard.kill_pending = false;
554                        guard.notify.notify_one();
555                    }
556                }
557                let should_reap = guard.session.is_none() && guard.clients.is_empty();
558                drop(guard);
559
560                if should_reap {
561                    // GC: drop the channel guard before requesting `channels.write`
562                    // (strict ordering, no AB-BA), then re-verify under the write lock.
563                    let mut channels = st.channels.write().await;
564                    if let Some(arc) = channels.get(&name_for_task) {
565                        let mut locked = arc.lock().await;
566                        if locked.session.is_none() && locked.clients.is_empty() {
567                            locked.is_reaped = true;
568                            drop(locked);
569                            channels.remove(&name_for_task);
570                            tracing::info!(channel = %name_for_task, "Reaped idle channel");
571                        }
572                    }
573                }
574                // Note: the daemon deliberately persists until an explicit
575                // `ShutdownGateway` / `term-session stop`. Sessions survive
576                // client disconnects; idle channels are reaped above but the
577                // gateway process itself is never torn down implicitly.
578            }
579        });
580    }
581
582    channels.insert(name.clone(), Arc::clone(&channel));
583    channel
584}
585
586/// Drop a connection's input forwarder. Called on disconnect (evict_conn) and
587/// on Attach re-bind so an abrupt drop or a channel re-attach cannot leak the
588/// drain task or route `cached_tx` to a stale channel's `input_tx`.
589fn purge_input_forwarder(state: &ServerState, conn_id: usize) {
590    if let Ok(mut fwd) = state.input_forwarders.lock() {
591        fwd.remove(&conn_id);
592    }
593}
594
595/// Remove a connection from the routing table and prune it from its bound
596/// channel's client/subscriber maps (authoritative teardown on disconnect).
597async fn evict_conn(state: &ServerState, conn_id: usize) {
598    purge_input_forwarder(state, conn_id);
599    let channel = {
600        let mut conns = state.conns.write().await;
601        let entry = conns.remove(&conn_id);
602        entry.and_then(|e| match e.state {
603            ConnState::Attached(name) => Some(name),
604            ConnState::Unattached => None,
605        })
606    };
607    let Some(channel) = channel else {
608        return;
609    };
610    let Some(ch) = resolve_channel(state, &channel).await else {
611        return;
612    };
613    let mut guard = ch.lock().await;
614    guard.clients.remove(&conn_id);
615    guard.subscribers.retain(|s| s.conn_id != conn_id);
616    guard.recalculate_pty_size();
617    // Broadcast the session's actual (client-driven) geometry to remaining
618    // clients. If no session exists there is nothing to broadcast.
619    let session_size = guard.session.as_ref().map(|s| (s.cols, s.rows));
620    let targets: Vec<ClientEntry> = guard.clients.values().cloned().collect();
621    drop(guard);
622    let Some((ncols, nrows)) = session_size else {
623        return;
624    };
625    // Best-effort geometry broadcast; the channel may be reaped concurrently,
626    // in which case clients already see end-of-stream.
627    if let Some(ch) = resolve_channel(state, &channel).await {
628        let guard = ch.lock().await;
629        guard.notify_clients(&targets, ncols, nrows);
630    }
631}
632
633/// Spawn a detached escalation task for a kill-requested session, returning
634/// its `JoinHandle` so the caller (e.g. shutdown teardown) can await it.
635///
636/// Policy owned by the daemon supervisor (never the pty engine): after an
637/// async `SIGKILL_GRACE` sleep, re-check the session state; if it already
638/// exited during the grace window (reaped by the output-polling task), abort
639/// instantly — no blind `SIGKILL` to a possibly-recycled pgid. Only escalate
640/// if the session is still alive and the kill is still pending.
641async fn spawn_kill_escalation(
642    state: &SharedState,
643    name: &ChannelName,
644) -> tokio::task::JoinHandle<()> {
645    let state = Arc::clone(state);
646    let name = name.clone();
647    tokio::spawn(async move {
648        tokio::time::sleep(SIGKILL_GRACE).await;
649        let Some(ch) = resolve_channel(&state, &name).await else {
650            return;
651        };
652        let mut guard = ch.lock().await;
653        if !guard.kill_pending {
654            return;
655        }
656        // Abort if the session exited during the grace (or is already gone).
657        let alive = guard
658            .session
659            .as_ref()
660            .is_some_and(|s| !s.exited && s.pty.reader_is_alive());
661        guard.kill_pending = false;
662        if !alive {
663            return;
664        }
665        if let Some(session) = guard.session.as_mut() {
666            #[cfg(unix)]
667            let _ = session.pty.signal_process_group(SIGKILL);
668            #[cfg(not(unix))]
669            let _ = session.pty.kill_child();
670        }
671    })
672}
673
674/// Run the gateway daemon. Hosts every channel in one process; returns after
675/// a `ShutdownGateway` (or transport error).
676pub async fn run_gateway(
677    gateway: ChannelName,
678) -> Result<i32, Box<dyn std::error::Error + Send + Sync>> {
679    let socket_name = gateway.to_string();
680    let state: SharedState = Arc::new(ServerState {
681        conns: RwLock::new(HashMap::new()),
682        channels: RwLock::new(HashMap::new()),
683        is_shutting_down: AtomicBool::new(false),
684        next_channel_seq: AtomicU64::new(0),
685        input_forwarders: std::sync::Mutex::new(HashMap::new()),
686    });
687
688    let (event_tx, mut event_rx) = mpsc::unbounded_channel();
689    let server = RpcIpcServer::new(Some(event_tx));
690    let endpoint = server.endpoint();
691
692    // ── Attach ────────────────────────────────────────────────────────
693    let st = Arc::clone(&state);
694    endpoint
695        .register_prebuffered(Attach::METHOD_ID, move |payload, ctx| {
696            let state = Arc::clone(&st);
697            async move {
698                if state.is_shutting_down.load(Ordering::SeqCst) {
699                    return Err(rpc_err(RPC_ERROR_SHUTTING_DOWN));
700                }
701                let req = Attach::decode_request(&payload)?;
702                let name = ChannelName::parse(&req.channel).map_err(|e| rpc_err(&e))?;
703                let _channel = get_or_create_channel(&state, &name).await;
704                let conn_id = ctx.conn_id;
705                // Purge any existing input forwarder for this connection so a
706                // re-attach to a different channel invalidates `cached_tx` and
707                // forces fresh target resolution.
708                purge_input_forwarder(&state, conn_id);
709                let mut conns = state.conns.write().await;
710                // The `ClientConnected` event is processed by a separate async
711                // loop, so it may not have inserted the entry yet when this
712                // handler runs. Ensure the entry exists before binding.
713                let entry = conns.entry(conn_id).or_insert_with(|| ConnEntry {
714                    handle: RpcIpcConnectionContextHandle(ctx.clone()),
715                    state: ConnState::Unattached,
716                    hostname: String::new(),
717                    connected_at_unix: now_unix(),
718                    pid: 0,
719                    user: String::new(),
720                    version: String::new(),
721                    ssh_ip: None,
722                });
723                entry.state = ConnState::Attached(name);
724                entry.hostname = req.hostname;
725                entry.connected_at_unix = now_unix();
726                entry.pid = req.pid;
727                entry.user = req.user;
728                entry.version = req.version;
729                entry.ssh_ip = req.ssh_ip;
730                Attach::encode_response(conn_id).map_err(boxed_io)
731            }
732        })
733        .await
734        .map_err(|e| format!("register Attach: {e:?}"))?;
735
736    // ── Spawn ────────────────────────────────────────────────────────
737    let st = Arc::clone(&state);
738    endpoint
739        .register_prebuffered(Spawn::METHOD_ID, move |payload, ctx| {
740            let state = Arc::clone(&st);
741            async move {
742                if state.is_shutting_down.load(Ordering::SeqCst) {
743                    return Err(rpc_err(RPC_ERROR_SHUTTING_DOWN));
744                }
745                let SpawnRequest {
746                    cmd,
747                    cols,
748                    rows,
749                    cwd,
750                } = Spawn::decode_request(&payload)?;
751                let channel = bound_channel(state.as_ref(), ctx.conn_id).await;
752                let Some(channel) = channel else {
753                    return Err(rpc_err(RPC_ERROR_UNATTACHED));
754                };
755                let ch = get_or_create_channel(&state, &channel).await;
756                // Fetch the connection's entry (caller handle, hostname,
757                // identity) before locking the channel (never hold two locks).
758                let conn_meta = {
759                    let conns = state.conns.read().await;
760                    conns.get(&ctx.conn_id).cloned()
761                };
762                let mut guard = ch.lock().await;
763                let entry = guard
764                    .clients
765                    .entry(ctx.conn_id)
766                    .or_insert_with(|| ClientEntry {
767                        caller: conn_meta.as_ref().map(|c| c.handle.clone()),
768                        hostname: conn_meta
769                            .as_ref()
770                            .map(|c| c.hostname.clone())
771                            .unwrap_or_default(),
772                        connected_at_unix: conn_meta
773                            .as_ref()
774                            .map(|c| c.connected_at_unix)
775                            .unwrap_or(0),
776                        pid: conn_meta.as_ref().map(|c| c.pid).unwrap_or(0),
777                        user: conn_meta
778                            .as_ref()
779                            .map(|c| c.user.clone())
780                            .unwrap_or_default(),
781                        version: conn_meta
782                            .as_ref()
783                            .map(|c| c.version.clone())
784                            .unwrap_or_default(),
785                        ssh_ip: conn_meta.as_ref().and_then(|c| c.ssh_ip.clone()),
786                        cols,
787                        rows,
788                    });
789                entry.cols = cols;
790                entry.rows = rows;
791
792                // If a session already exists and hasn't exited, reuse it.
793                if guard.session.as_ref().is_some_and(|s| !s.exited) {
794                    guard.recalculate_pty_size();
795                    let session = guard.session.as_ref().unwrap();
796                    let (ncols, nrows) = (session.cols, session.rows);
797                    let targets: Vec<ClientEntry> = guard.clients.values().cloned().collect();
798                    let id = session.id;
799                    let cols = session.cols;
800                    let rows = session.rows;
801                    drop(guard);
802                    if let Some(ch) = resolve_channel(state.as_ref(), &channel).await {
803                        let g = ch.lock().await;
804                        g.notify_clients(&targets, ncols, nrows);
805                    }
806                    return Spawn::encode_response(SpawnResponse { id, cols, rows })
807                        .map_err(boxed_io);
808                }
809
810                // Respawn: new non-empty cmd overwrites the stored template;
811                // empty/None falls back to the existing template.
812                let effective_cmd = if let Some(c) = cmd
813                    && !c.is_empty()
814                {
815                    guard.cmd = c.clone();
816                    Some(c)
817                } else if !guard.cmd.is_empty() {
818                    Some(guard.cmd.clone())
819                } else {
820                    None
821                };
822                // Spawn in the client's launch directory when provided (the
823                // caller expects to land where they ran `term-session`), else
824                // fall back to the daemon's cwd for legacy/empty payloads.
825                let effective_cwd = cwd.filter(|c| !c.is_empty());
826                let id = SESSION_ID;
827                let session = Session::spawn(
828                    id,
829                    effective_cmd,
830                    cols,
831                    rows,
832                    Some(&channel),
833                    effective_cwd.as_ref(),
834                )?;
835                guard.set_session(session);
836                guard.recalculate_pty_size();
837                let targets: Vec<ClientEntry> = guard.clients.values().cloned().collect();
838                let session = guard.session.as_ref().unwrap();
839                let (sid, scol, srow) = (session.id, session.cols, session.rows);
840                let (ncols, nrows) = (scol, srow);
841                drop(guard);
842                if let Some(ch) = resolve_channel(state.as_ref(), &channel).await {
843                    let g = ch.lock().await;
844                    g.notify_clients(&targets, ncols, nrows);
845                }
846                Spawn::encode_response(SpawnResponse {
847                    id: sid,
848                    cols: scol,
849                    rows: srow,
850                })
851                .map_err(boxed_io)
852            }
853        })
854        .await
855        .map_err(|e| format!("register Spawn: {e:?}"))?;
856
857    // ── ResizePty ────────────────────────────────────────────────────
858    let st = Arc::clone(&state);
859    endpoint
860        .register_prebuffered(ResizePty::METHOD_ID, move |payload, ctx| {
861            let state = Arc::clone(&st);
862            async move {
863                if state.is_shutting_down.load(Ordering::SeqCst) {
864                    return Err(rpc_err(RPC_ERROR_SHUTTING_DOWN));
865                }
866                let (_id, cols, rows) = ResizePty::decode_request(&payload)?;
867                let channel = bound_channel(state.as_ref(), ctx.conn_id).await;
868                let Some(channel) = channel else {
869                    return Err(rpc_err(RPC_ERROR_UNATTACHED));
870                };
871                let ch = resolve_channel(state.as_ref(), &channel)
872                    .await
873                    .ok_or_else(|| rpc_err("channel not found"))?;
874                let mut guard = ch.lock().await;
875                if let Some(client) = guard.clients.get_mut(&ctx.conn_id) {
876                    client.cols = cols;
877                    client.rows = rows;
878                }
879                guard.recalculate_pty_size();
880                let (ncols, nrows) = guard
881                    .session
882                    .as_ref()
883                    .map(|s| (s.cols, s.rows))
884                    .unwrap_or((cols, rows));
885                let targets: Vec<ClientEntry> = guard.clients.values().cloned().collect();
886                drop(guard);
887                if let Some(ch) = resolve_channel(state.as_ref(), &channel).await {
888                    let g = ch.lock().await;
889                    g.notify_clients(&targets, ncols, nrows);
890                }
891                ResizePty::encode_response((ncols, nrows)).map_err(boxed_io)
892            }
893        })
894        .await
895        .map_err(|e| format!("register ResizePty: {e:?}"))?;
896
897    // ── CloseSession ─────────────────────────────────────────────────
898    let st = Arc::clone(&state);
899    endpoint
900        .register_prebuffered(CloseSession::METHOD_ID, move |payload, ctx| {
901            let state = Arc::clone(&st);
902            async move {
903                if state.is_shutting_down.load(Ordering::SeqCst) {
904                    return Err(rpc_err(RPC_ERROR_SHUTTING_DOWN));
905                }
906                let _id = CloseSession::decode_request(&payload)?;
907                let channel = bound_channel(state.as_ref(), ctx.conn_id).await;
908                let Some(channel) = channel else {
909                    return Err(rpc_err(RPC_ERROR_UNATTACHED));
910                };
911                let ch = resolve_channel(state.as_ref(), &channel)
912                    .await
913                    .ok_or_else(|| rpc_err("channel not found"))?;
914                let mut guard = ch.lock().await;
915                guard.request_session_kill(SIGTERM);
916                guard.finalize_subscribers();
917                drop(guard);
918                // Spawn the exited-checked SIGKILL escalation for stragglers.
919                spawn_kill_escalation(&state, &channel).await;
920                CloseSession::encode_response(()).map_err(boxed_io)
921            }
922        })
923        .await
924        .map_err(|e| format!("register CloseSession: {e:?}"))?;
925
926    // ── WriteInput ───────────────────────────────────────────────────
927    let st = Arc::clone(&state);
928    endpoint
929        .register_prebuffered(WriteInput::METHOD_ID, move |payload, ctx| {
930            let state = Arc::clone(&st);
931            async move {
932                if state.is_shutting_down.load(Ordering::SeqCst) {
933                    return Err(rpc_err(RPC_ERROR_SHUTTING_DOWN));
934                }
935                let (id, data) = WriteInput::decode_request(&payload)?;
936                let channel = bound_channel(state.as_ref(), ctx.conn_id).await;
937                let Some(channel) = channel else {
938                    return Err(rpc_err(RPC_ERROR_UNATTACHED));
939                };
940                let ch = resolve_channel(state.as_ref(), &channel)
941                    .await
942                    .ok_or_else(|| rpc_err("channel not found"))?;
943                let writer = {
944                    let guard = ch.lock().await;
945                    guard
946                        .session
947                        .as_ref()
948                        .filter(|s| s.id == id)
949                        .map(|s| s.pty.writer_handle())
950                };
951                // PTY writes are blocking I/O (kernel input buffer); offload
952                // to the blocking pool so a full buffer never stalls an async
953                // worker or holds the state lock.
954                if let Some(writer) = writer {
955                    let _ = tokio::task::spawn_blocking(move || writer.write_bytes(&data)).await;
956                }
957                WriteInput::encode_response(()).map_err(boxed_io)
958            }
959        })
960        .await
961        .map_err(|e| format!("register WriteInput: {e:?}"))?;
962
963    // ── StreamInput ──────────────────────────────────────────────────
964    let st = Arc::clone(&state);
965    endpoint
966        .register_stream_handler(STREAM_INPUT_METHOD_ID, move |event, _responder, ctx| {
967            let state = Arc::clone(&st);
968            let conn_id = ctx.conn_id;
969            match event {
970                RpcStreamEvent::PayloadChunk { bytes, .. } => {
971                    // Forward the chunk into the connection's ordered queue
972                    // synchronously — the handler is invoked sequentially in
973                    // wire order per stream, so `try_send` here preserves chunk
974                    // ordering. A single consumer task per connection drains
975                    // FIFO into the channel's `input_tx`, so bursty input
976                    // (e.g. IME voice typing) is never reordered by racing
977                    // spawned tasks.
978                    let forwarder = {
979                        let mut fwd = state
980                            .input_forwarders
981                            .lock()
982                            .unwrap_or_else(|e| e.into_inner());
983                        if let Some(tx) = fwd.get(&conn_id) {
984                            tx.clone()
985                        } else {
986                            let (tx, rx) = mpsc::channel(INPUT_CHANNEL_CAPACITY);
987                            fwd.insert(conn_id, tx.clone());
988                            let fwd_state = Arc::clone(&state);
989                            tokio::spawn(async move {
990                                drain_input_forwarder(fwd_state, conn_id, rx).await;
991                            });
992                            tx
993                        }
994                    };
995                    if let Err(e) = forwarder.try_send(bytes) {
996                        tracing::warn!(error = %e, "gateway input buffer full; dropping input chunk");
997                    }
998                }
999                RpcStreamEvent::End { .. } | RpcStreamEvent::Error { .. } => {
1000                    // Close the forwarder: dropping the sender makes the
1001                    // consumer task's `recv()` return `None` so it exits after
1002                    // draining. The channel's `input_tx` persists, so the
1003                    // session survives the client disconnect.
1004                    if let Ok(mut fwd) = state.input_forwarders.lock() {
1005                        fwd.remove(&conn_id);
1006                    }
1007                }
1008                _ => {}
1009            }
1010        })
1011        .await
1012        .map_err(|e| format!("register stream handler STREAM_INPUT: {e:?}"))?;
1013
1014    // ── SubscribeOutput ──────────────────────────────────────────────
1015    let st = Arc::clone(&state);
1016    endpoint
1017        .register_stream_handler(SUBSCRIBE_OUTPUT_METHOD_ID, move |event, respond, ctx| {
1018            let is_new = matches!(&event, RpcStreamEvent::Header { .. });
1019            if is_new {
1020                let st = Arc::clone(&st);
1021                let conn_id = ctx.conn_id;
1022                tokio::spawn(async move {
1023                    let channel = bound_channel(&st, conn_id).await;
1024                    let Some(channel) = channel else {
1025                        return;
1026                    };
1027                    let ch = resolve_channel(&st, &channel).await;
1028                    let Some(ch) = ch else {
1029                        return;
1030                    };
1031                    let mut guard = ch.lock().await;
1032                    // Drain accumulated PTY output and the retained final output
1033                    // from a dead session (by clone, so a subscriber that
1034                    // attaches and drops does not consume the bytes for later
1035                    // subscribers) and capture the raw bytes so they can be sent
1036                    // to the new subscriber.
1037                    let early = {
1038                        let mut all = guard.output_cache.clone();
1039                        if let Some(session) = guard.session.as_mut() {
1040                            all.extend_from_slice(&session.read_output());
1041                        }
1042                        if all.is_empty() { None } else { Some(all) }
1043                    };
1044                    let snapshot = guard.session.as_mut().map(|s| s.generate_snapshot());
1045                    guard.subscribers.push(SubscriberEntry {
1046                        conn_id,
1047                        respond: respond.clone(),
1048                    });
1049                    guard.notify.notify_one();
1050                    let is_dead = guard.session.is_none();
1051                    // Deliver the retained/live output and the end-of-stream
1052                    // marker while still holding the channel guard, so the
1053                    // polling task's dead-session finalization cannot interleave
1054                    // an EOF ahead of this subscriber's data. respond() is
1055                    // synchronous and already called under the guard elsewhere.
1056                    if let Some(data) = snapshot
1057                        && !data.is_empty()
1058                    {
1059                        respond.respond(data, false);
1060                    }
1061                    if let Some(data) = early {
1062                        respond.respond(data, false);
1063                    }
1064                    if is_dead {
1065                        respond.respond(Vec::new(), true);
1066                    }
1067                    drop(guard);
1068                });
1069            }
1070        })
1071        .await
1072        .map_err(|e| format!("register SubscribeOutput: {e:?}"))?;
1073
1074    // ── ListChannels ─────────────────────────────────────────────────
1075    let st = Arc::clone(&state);
1076    let list_socket = socket_name.clone();
1077    endpoint
1078        .register_prebuffered(ListChannels::METHOD_ID, move |_payload, _ctx| {
1079            let state = Arc::clone(&st);
1080            let socket = list_socket.clone();
1081            async move {
1082                let channels = {
1083                    let chans = state.channels.read().await;
1084                    chans
1085                        .iter()
1086                        .map(|(k, v)| (k.clone(), v.clone()))
1087                        .collect::<Vec<_>>()
1088                };
1089                // Creation order, newest last (monotonic `created_seq`, not
1090                // second-resolution wall clock). The seq is read under the same
1091                // per-channel lock used to build `ChannelInfo` below.
1092                let mut out: Vec<(u64, ChannelInfo)> = Vec::with_capacity(channels.len());
1093                for (name, ch) in channels {
1094                    let guard = ch.lock().await;
1095                    out.push((guard.created_seq, guard.to_info(&name)));
1096                }
1097                out.sort_by_key(|(seq, _)| *seq);
1098                let out: Vec<ChannelInfo> = out.into_iter().map(|(_, info)| info).collect();
1099                ListChannels::encode_response(ListChannelsResponse {
1100                    gateway_pid: std::process::id() as u64,
1101                    socket,
1102                    channels: out,
1103                })
1104                .map_err(boxed_io)
1105            }
1106        })
1107        .await
1108        .map_err(|e| format!("register ListChannels: {e:?}"))?;
1109
1110    // ── KillChannel ──────────────────────────────────────────────────
1111    let st = Arc::clone(&state);
1112    endpoint
1113        .register_prebuffered(KillChannel::METHOD_ID, move |payload, _ctx| {
1114            let state = Arc::clone(&st);
1115            async move {
1116                if state.is_shutting_down.load(Ordering::SeqCst) {
1117                    return Err(rpc_err(RPC_ERROR_SHUTTING_DOWN));
1118                }
1119                let (channel_str, force) = KillChannel::decode_request(&payload)?;
1120                let name = ChannelName::parse(&channel_str).map_err(|e| rpc_err(&e))?;
1121                // 1) Snapshot the target connections under `conns.write`, then release.
1122                let target_conns: Vec<usize> = {
1123                    let conns = state.conns.read().await;
1124                    conns
1125                        .iter()
1126                        .filter(|(_, entry)| matches!(entry.state, ConnState::Attached(ref n) if n == &name))
1127                        .map(|(conn_id, _)| *conn_id)
1128                        .collect()
1129                };
1130                // 2) Lock the channel once: refuse an accidental kill while
1131                //    participants are attached unless the caller forced it,
1132                //    then signal the session tree and evict every socket.
1133                //    Holding the lock through check + teardown serializes
1134                //    against concurrent Attach/Spawn, so a participant cannot
1135                //    slip in between the verification and the eviction.
1136                if let Some(ch) = resolve_channel(state.as_ref(), &name).await {
1137                    let mut guard = ch.lock().await;
1138                    let n = guard.clients.len();
1139                    if !force && n > 0 {
1140                        return Err(rpc_err(&format!(
1141                            "{RPC_ERROR_LIVE_PARTICIPANTS} ({n} participant(s) attached)"
1142                        )));
1143                    }
1144                    guard.request_session_kill(SIGTERM);
1145                    guard.finalize_subscribers();
1146                    for conn_id in &target_conns {
1147                        guard.clients.remove(conn_id);
1148                        guard.subscribers.retain(|s| s.conn_id != *conn_id);
1149                    }
1150                    drop(guard);
1151                    spawn_kill_escalation(&state, &name).await;
1152                }
1153                // 3) Evict the ConnEntry records from the routing table.
1154                let mut conns = state.conns.write().await;
1155                for conn_id in &target_conns {
1156                    conns.remove(conn_id);
1157                }
1158                KillChannel::encode_response(()).map_err(boxed_io)
1159            }
1160        })
1161        .await
1162        .map_err(|e| format!("register KillChannel: {e:?}"))?;
1163
1164    // ── KillClient ───────────────────────────────────────────────────
1165    let st = Arc::clone(&state);
1166    endpoint
1167        .register_prebuffered(KillClient::METHOD_ID, move |payload, _ctx| {
1168            let state = Arc::clone(&st);
1169            async move {
1170                if state.is_shutting_down.load(Ordering::SeqCst) {
1171                    return Err(rpc_err(RPC_ERROR_SHUTTING_DOWN));
1172                }
1173                let (channel_str, conn_id) = KillClient::decode_request(&payload)?;
1174                let name = ChannelName::parse(&channel_str).map_err(|e| rpc_err(&e))?;
1175                // Reject if the conn does not exist or is not attached to the
1176                // named channel — kill-client must target a real client.
1177                let bound_channel = {
1178                    let conns = state.conns.read().await;
1179                    conns.get(&conn_id).and_then(|c| match &c.state {
1180                        ConnState::Attached(n) if n == &name => Some(n.clone()),
1181                        _ => None,
1182                    })
1183                };
1184                let Some(bound) = bound_channel else {
1185                    return Err(rpc_err(&format!(
1186                        "client {conn_id} is not attached to channel '{name}'"
1187                    )));
1188                };
1189                // Evict the conn first (conns → channel ordering).
1190                {
1191                    let mut conns = state.conns.write().await;
1192                    conns.remove(&conn_id);
1193                }
1194                if let Some(ch) = resolve_channel(state.as_ref(), &bound).await {
1195                    let mut guard = ch.lock().await;
1196                    // End the evicted subscriber's stream before dropping it.
1197                    let mut evicted: Vec<StreamResponder> = Vec::new();
1198                    let mut keep = Vec::with_capacity(guard.subscribers.len());
1199                    for sub in guard.subscribers.drain(..) {
1200                        if sub.conn_id == conn_id {
1201                            evicted.push(sub.respond);
1202                        } else {
1203                            keep.push(sub);
1204                        }
1205                    }
1206                    guard.subscribers = keep;
1207                    for respond in evicted {
1208                        respond.respond(Vec::new(), true);
1209                    }
1210                    guard.clients.remove(&conn_id);
1211                    guard.recalculate_pty_size();
1212                    let session_size = guard.session.as_ref().map(|s| (s.cols, s.rows));
1213                    let targets: Vec<ClientEntry> = guard.clients.values().cloned().collect();
1214                    drop(guard);
1215                    if let (Some((ncols, nrows)), Some(ch)) =
1216                        (session_size, resolve_channel(state.as_ref(), &bound).await)
1217                    {
1218                        let g = ch.lock().await;
1219                        g.notify_clients(&targets, ncols, nrows);
1220                    }
1221                }
1222                KillClient::encode_response(()).map_err(boxed_io)
1223            }
1224        })
1225        .await
1226        .map_err(|e| format!("register KillClient: {e:?}"))?;
1227
1228    // ── ShutdownGateway ──────────────────────────────────────────────
1229    let st = Arc::clone(&state);
1230    let (shutdown_tx, mut shutdown_rx) = oneshot::channel::<()>();
1231    let shutdown_tx = Arc::new(Mutex::new(Some(shutdown_tx)));
1232    endpoint
1233        .register_prebuffered(ShutdownGateway::METHOD_ID, move |payload, _ctx| {
1234            let state = Arc::clone(&st);
1235            let shutdown_tx = Arc::clone(&shutdown_tx);
1236            async move {
1237                // Refuse an accidental shutdown while live sessions are running
1238                // unless the caller explicitly forced it. Checked BEFORE the
1239                // `is_shutting_down` seal so a refused stop leaves the gateway
1240                // fully operational (no half-sealed state, no orphaned teardown).
1241                let force = ShutdownGateway::decode_request(&payload).map_err(boxed_io)?;
1242                if !force {
1243                    let live = {
1244                        let chans = state.channels.read().await;
1245                        let mut n = 0usize;
1246                        for ch in chans.values() {
1247                            let guard = ch.lock().await;
1248                            if guard.session.as_ref().is_some_and(|s| !s.exited) {
1249                                n += 1;
1250                            }
1251                        }
1252                        n
1253                    };
1254                    if live > 0 {
1255                        return Err(rpc_err(&format!(
1256                            "{RPC_ERROR_LIVE_SESSIONS} ({live} live session(s))"
1257                        )));
1258                    }
1259                }
1260                // Atomic seal: reject all further RPCs before teardown starts.
1261                state.is_shutting_down.store(true, Ordering::SeqCst);
1262                // Snapshot the channels, then release the map lock.
1263                let channels: Vec<(ChannelName, Arc<Mutex<ChannelState>>)> = {
1264                    let chans = state.channels.read().await;
1265                    chans.iter().map(|(k, v)| (k.clone(), v.clone())).collect()
1266                };
1267                // Off-lock: signal each session's process group (non-blocking),
1268                // finalize subscribers, and collect the escalation handles so
1269                // the exit signal fires only after every child tree is reaped.
1270                let mut escalations = Vec::new();
1271                for (name, ch) in channels {
1272                    let mut guard = ch.lock().await;
1273                    tracing::info!(channel = %name, "Shutdown: signaling session tree");
1274                    guard.request_session_kill(SIGTERM);
1275                    guard.finalize_subscribers();
1276                    drop(guard);
1277                    escalations.push(spawn_kill_escalation(&state, &name).await);
1278                }
1279                // Deferred exit signal: await the SIGKILL escalation tasks so
1280                // all child process trees are definitively terminated, then
1281                // sleep a grace so the transport flushes the `()` response
1282                // frame, then fire the oneshot that ends run_gateway. Never
1283                // fire it synchronously from the handler.
1284                tokio::spawn(async move {
1285                    for handle in escalations {
1286                        let _ = handle.await;
1287                    }
1288                    tokio::time::sleep(std::time::Duration::from_millis(SHUTDOWN_FLUSH_GRACE_MS))
1289                        .await;
1290                    let mut tx_guard = shutdown_tx.lock().await;
1291                    if let Some(tx) = tx_guard.take() {
1292                        let _ = tx.send(());
1293                    }
1294                });
1295                ShutdownGateway::encode_response(()).map_err(boxed_io)
1296            }
1297        })
1298        .await
1299        .map_err(|e| format!("register ShutdownGateway: {e:?}"))?;
1300
1301    // ── Connection event loop ────────────────────────────────────────
1302    let st = Arc::clone(&state);
1303    tokio::spawn(async move {
1304        while let Some(event) = event_rx.recv().await {
1305            match event {
1306                RpcIpcServerEvent::ClientConnected(handle) => {
1307                    tracing::info!("Client {} connected", handle.0.conn_id);
1308                    let mut conns = st.conns.write().await;
1309                    // A fast client may send Attach before this event is
1310                    // processed; the Attach handler already inserted an
1311                    // `Attached` entry. `insert` would clobber that binding and
1312                    // a subsequent Spawn would see the state reset to
1313                    // Unattached. Only create the entry if it is absent.
1314                    conns.entry(handle.0.conn_id).or_insert_with(|| ConnEntry {
1315                        handle: handle.clone(),
1316                        state: ConnState::Unattached,
1317                        hostname: String::new(),
1318                        connected_at_unix: now_unix(),
1319                        pid: 0,
1320                        user: String::new(),
1321                        version: String::new(),
1322                        ssh_ip: None,
1323                    });
1324                }
1325                RpcIpcServerEvent::ClientDisconnected(conn_id) => {
1326                    tracing::info!("Client {conn_id} disconnected");
1327                    evict_conn(st.as_ref(), conn_id).await;
1328                }
1329            }
1330        }
1331    });
1332
1333    tracing::info!("Gateway listening on channel {gateway}");
1334
1335    // Wait for either the server to finish or a shutdown signal.
1336    let exit_code = tokio::select! {
1337        result = async {
1338            server.serve(&socket_name).await.map_err(|e| format!("serve: {e:?}"))
1339        } => {
1340            result?;
1341            0
1342        }
1343        _ = &mut shutdown_rx => {
1344            // Give the transport time to flush final subscriber frames.
1345            tokio::time::sleep(SESSION_EXIT_FLUSH_GRACE).await;
1346            0
1347        }
1348    };
1349
1350    Ok(exit_code)
1351}
1352
1353#[cfg(test)]
1354mod tests {
1355    use super::*;
1356    use muxio_core::rpc::RpcDispatcher;
1357    use muxio_tokio_rpc_ipc_server::RpcIpcConnectionContext;
1358
1359    /// Build a ServerState where `conn_id = 1` is attached to `test/coalesce`,
1360    /// whose ChannelState carries the given `input_tx` (observed via the paired
1361    /// receiver in the test).
1362    fn state_with_input(input_tx: mpsc::Sender<Vec<u8>>) -> SharedState {
1363        let name = ChannelName::parse("test/coalesce").expect("parse channel");
1364        let channel = Arc::new(Mutex::new(ChannelState::new(
1365            Vec::new(),
1366            input_tx,
1367            Arc::new(Notify::new()),
1368            1,
1369        )));
1370        let mut channels = HashMap::new();
1371        channels.insert(name.clone(), channel);
1372        let (write_tx, _write_rx) = mpsc::unbounded_channel();
1373        let conn = ConnEntry {
1374            handle: RpcIpcConnectionContextHandle(Arc::new(RpcIpcConnectionContext {
1375                write_tx,
1376                conn_id: 1,
1377                is_connected: Arc::new(AtomicBool::new(true)),
1378                dispatcher: Arc::new(Mutex::new(RpcDispatcher::new())),
1379            })),
1380            state: ConnState::Attached(name),
1381            hostname: String::new(),
1382            connected_at_unix: 0,
1383            pid: 0,
1384            user: String::new(),
1385            version: String::new(),
1386            ssh_ip: None,
1387        };
1388        let mut conns = HashMap::new();
1389        conns.insert(1, conn);
1390        Arc::new(ServerState {
1391            conns: RwLock::new(conns),
1392            channels: RwLock::new(channels),
1393            is_shutting_down: AtomicBool::new(false),
1394            next_channel_seq: AtomicU64::new(0),
1395            input_forwarders: std::sync::Mutex::new(HashMap::new()),
1396        })
1397    }
1398
1399    #[tokio::test]
1400    async fn drain_input_forwarder_coalesces_queued_chunks() {
1401        let (input_tx, mut input_rx) = mpsc::channel(128);
1402        let state = state_with_input(input_tx);
1403
1404        // Pre-fill the forwarder channel so coalescing is deterministic.
1405        let (fwd_tx, fwd_rx) = mpsc::channel(128);
1406        fwd_tx.send(b"chunk1".to_vec()).await.unwrap();
1407        fwd_tx.send(b"chunk2".to_vec()).await.unwrap();
1408        fwd_tx.send(b"chunk3".to_vec()).await.unwrap();
1409        drop(fwd_tx); // so the worker task exits after draining
1410
1411        tokio::spawn(drain_input_forwarder(state, 1, fwd_rx));
1412
1413        // The production worker must coalesce all three into one send.
1414        let received = input_rx.recv().await.expect("coalesced input");
1415        assert_eq!(received, b"chunk1chunk2chunk3");
1416    }
1417
1418    #[tokio::test]
1419    async fn drain_input_forwarder_forwards_isolated_chunk_unchanged() {
1420        let (input_tx, mut input_rx) = mpsc::channel(128);
1421        let state = state_with_input(input_tx);
1422
1423        let (fwd_tx, fwd_rx) = mpsc::channel(128);
1424        fwd_tx.send(b"only".to_vec()).await.unwrap();
1425        drop(fwd_tx);
1426
1427        tokio::spawn(drain_input_forwarder(state, 1, fwd_rx));
1428
1429        let received = input_rx.recv().await.expect("forwarded input");
1430        assert_eq!(received, b"only");
1431    }
1432
1433    #[tokio::test]
1434    async fn evict_conn_purges_input_forwarder() {
1435        let (input_tx, _input_rx) = mpsc::channel(128);
1436        let state = state_with_input(input_tx);
1437        let (fwd_tx, _fwd_rx) = mpsc::channel(128);
1438        state.input_forwarders.lock().unwrap().insert(1, fwd_tx);
1439
1440        evict_conn(&state, 1).await;
1441
1442        assert!(!state.input_forwarders.lock().unwrap().contains_key(&1));
1443    }
1444
1445    #[test]
1446    fn reattach_purges_existing_forwarder() {
1447        // Exercises the SAME production `purge_input_forwarder` that both
1448        // `evict_conn` and the `Attach::METHOD_ID` handler call — not an
1449        // inlined `fwd.remove`. A socket re-attach to a different channel
1450        // must drop the old forwarder so `cached_tx` cannot route input to the
1451        // previous channel's `input_tx`.
1452        let (input_tx, _input_rx) = mpsc::channel(128);
1453        let state = state_with_input(input_tx);
1454        let (fwd_tx, _fwd_rx) = mpsc::channel(128);
1455        state.input_forwarders.lock().unwrap().insert(1, fwd_tx);
1456
1457        purge_input_forwarder(&state, 1);
1458
1459        assert!(!state.input_forwarders.lock().unwrap().contains_key(&1));
1460    }
1461}