Skip to main content

mj_controller/
session_manager.rs

1//! Multiplexed controller-side ownership of durable ACP relay sessions.
2
3use std::collections::{BTreeMap, VecDeque};
4use std::path::PathBuf;
5use std::sync::{Arc, Mutex};
6use std::time::{Duration, Instant};
7
8use anyhow::{Context, Result, bail, ensure};
9use tokio::sync::{mpsc, oneshot, watch};
10
11use crate::database::{
12    ProjectionApplyOutcome, ProjectionIntegrityError, apply_projection_page,
13    save_materialized_session,
14};
15use crate::worker_client::{RelayClient, RelayEventPage, RelayRejected, RelayTransportDead};
16use mj_checkpoint::archive::verify_archive_streaming;
17use mj_core::credentials::{CredentialSyncSignal, relay_event_credential_sync_reason};
18use mj_core::elicitation::ElicitationResponse;
19use mj_core::state::{ManagedSessionSnapshot, MaterializedSession};
20use mj_transcript::projection::{
21    ProjectionIndex, apply_committed_projection_event_indexed, materialized_session_from_canonical,
22    project_relay_event_indexed,
23};
24
25use crate::targets::{
26    CancellableProcessExecutor, CommandExecutor, CommandPlan, CommandSpec, TargetLocator,
27    TargetRecoveryOutcome, TargetRecoveryPlan, ensure_recovery_target_running,
28};
29use mj_core::relay::{RelayCommand, RelayCursor, RelayOperationalState};
30
31pub use mj_client::session::{
32    ManagedSessionView, ReviewerAction, ReviewerOutcome, ViewError, new_command_id,
33};
34#[cfg(test)]
35use mj_core::worker_launch::ReviewerLaunchConfig;
36
37const SESSION_SYNC_INTERVAL: Duration = Duration::from_millis(150);
38/// Release SQLite's single writer between bounded pieces of a large relay
39/// catch-up. One transport page can contain thousands of terminal events and
40/// must not prevent every other session actor from publishing its view.
41const PROJECTION_TRANSACTION_EVENT_BUDGET: usize = 128;
42const RECONNECT_INTERVAL: Duration = Duration::from_secs(1);
43/// Ceiling for reconnect backoff. A worker that exited stays gone until the
44/// user acts, so retrying it every second only burns process spawns.
45const RECONNECT_BACKOFF_CEILING: Duration = Duration::from_secs(30);
46const UNREACHABLE_FAILURE_THRESHOLD: u32 = 2;
47const WORKER_RESTART_TIMEOUT: Duration = Duration::from_secs(30);
48const WORKER_RESTART_COOLDOWN: Duration = Duration::from_secs(60);
49const SESSION_MANAGER_SHUTDOWN_GRACE: Duration = Duration::from_millis(750);
50
51#[derive(Debug)]
52struct ProjectionAdvancedError {
53    event_ordinal: u64,
54}
55
56impl std::fmt::Display for ProjectionAdvancedError {
57    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
58        write!(
59            formatter,
60            "another projector committed relay event {} first",
61            self.event_ordinal
62        )
63    }
64}
65
66impl std::error::Error for ProjectionAdvancedError {}
67
68/// Delay before the next reconnect attempt after `failures` consecutive
69/// failures. Doubles from `RECONNECT_INTERVAL` up to the ceiling.
70fn reconnect_delay(failures: u32) -> Duration {
71    let doubling = failures.saturating_sub(1).min(u32::BITS - 1);
72    RECONNECT_INTERVAL
73        .saturating_mul(1_u32 << doubling)
74        .min(RECONNECT_BACKOFF_CEILING)
75}
76
77#[derive(Debug, Clone, PartialEq, Eq)]
78pub struct RelaySessionTarget {
79    pub session_id: String,
80    pub spec: CommandSpec,
81    /// Prove the exact worker is absent before restarting it in place. Direct
82    /// relay clients omit recovery; controller-managed sessions self-heal
83    /// without turning a shared transport outage into destructive restarts.
84    pub worker_recovery: Option<WorkerRecoveryPlan>,
85    pub project_memory: Option<ProjectMemorySyncTarget>,
86}
87
88#[derive(Debug, Clone, PartialEq, Eq)]
89pub struct ProjectMemorySyncTarget {
90    pub canonical_root: std::path::PathBuf,
91}
92
93/// The working directory a bare-target worker must be able to enter before it
94/// can serve a relay handshake. Container availability is checked separately
95/// by the target recovery plan; bare targets have no runtime object to inspect.
96#[derive(Debug, Clone, PartialEq, Eq)]
97pub struct WorkerWorkspace {
98    pub target: mj_core::state::ManagedWorktreeTarget,
99    pub directory: PathBuf,
100}
101
102#[derive(Debug, Clone, PartialEq, Eq)]
103pub struct WorkerRecoveryPlan {
104    /// Durable target identity when this plan was built. A stale actor must
105    /// never recover a resource after the session moves or starts destruction.
106    pub source_target: mj_core::state::TargetLocator,
107    pub target: Option<TargetRecoveryPlan>,
108    pub workspace: Option<WorkerWorkspace>,
109    pub liveness_probe: CommandSpec,
110    /// Refresh a stale installed worker before restarting it. The digest is
111    /// computed inside the recovery task so hashing a large binary never
112    /// blocks a controller UI loop.
113    pub binary_refresh: Option<WorkerBinaryRefresh>,
114    /// Keep the worker executable and its launch schema paired. Configuration
115    /// bytes travel through redacted stdin only when their digest is stale.
116    pub launch_refresh: Option<WorkerLaunchRefreshPlan>,
117    pub restart: CommandPlan,
118}
119
120/// How recovery refreshes a stale installed worker binary before restarting.
121///
122/// Local targets resolve the source and the copy at plan-build time, which is
123/// cheap. Remote targets cannot: choosing the binary needs the target's
124/// architecture, and that probe plus hashing the remote binary are blocking
125/// ssh round-trips that must not run on the plan-build/UI path. So a remote
126/// refresh carries only what is cheap to compute and resolves the rest inside
127/// the recovery task.
128#[derive(Debug, Clone, PartialEq, Eq)]
129pub enum WorkerBinaryRefresh {
130    Prepared(WorkerBinaryRefreshPlan),
131    Remote(RemoteWorkerBinaryRefresh),
132}
133
134#[derive(Debug, Clone, PartialEq, Eq)]
135pub struct WorkerBinaryRefreshPlan {
136    pub source: PathBuf,
137    pub installed_digest: CommandSpec,
138    pub replace: CommandPlan,
139}
140
141/// A remote worker refresh resolved at recovery time: select the worker binary
142/// for the target's own architecture, compare it to the installed one, and
143/// copy only when they differ.
144#[derive(Debug, Clone, PartialEq, Eq)]
145pub struct RemoteWorkerBinaryRefresh {
146    pub locator: TargetLocator,
147    pub session_id: String,
148    pub installed_digest: CommandSpec,
149}
150
151#[derive(Debug, Clone, PartialEq, Eq)]
152pub struct WorkerLaunchRefreshPlan {
153    pub expected_sha256: String,
154    pub installed_digest: CommandSpec,
155    pub replace: CommandPlan,
156}
157
158/// Whether a relay failure means the transport to the worker is gone, so
159/// restarting that worker is the only recovery left.
160///
161/// Every failure that proves it is marked with [`RelayTransportDead`] where it
162/// is produced, and this decision downcasts for that marker. Message text is
163/// never read: a reworded diagnostic must not be able to disable auto-restart.
164pub(crate) fn worker_connect_needs_restart(error: &anyhow::Error) -> bool {
165    RelayTransportDead::marks(error)
166}
167
168fn worker_connect_allows_live_restart(error: &anyhow::Error) -> bool {
169    RelayTransportDead::marks_failed_handshake(error)
170}
171
172#[derive(Debug, Clone, PartialEq, Eq)]
173pub(crate) enum WorkerRecoveryOutcome {
174    Alive,
175    Starting,
176    TargetMissing,
177    Suppressed,
178    WorkspaceMissing(PathBuf),
179    RestartedDead,
180    RestartedUnresponsive,
181}
182
183fn refresh_worker_binary_if_stale(
184    executor: &impl CommandExecutor,
185    refresh: Option<&WorkerBinaryRefresh>,
186) -> Result<()> {
187    match refresh {
188        None => Ok(()),
189        Some(WorkerBinaryRefresh::Prepared(plan)) => {
190            let expected = mj_core::worker_launch::worker_executable_digest(&plan.source)?;
191            if installed_digest_matches(executor, &plan.installed_digest, &expected) {
192                return Ok(());
193            }
194            plan.replace
195                .execute(executor)
196                .context("replace stale relay worker binary")?;
197            Ok(())
198        }
199        // Remote: pick the binary for the target's architecture and copy only
200        // if it differs. Runs here in the recovery task, never on the UI path.
201        Some(WorkerBinaryRefresh::Remote(refresh)) => {
202            crate::controller::refresh_remote_worker_binary_if_stale(executor, refresh)
203        }
204    }
205}
206
207fn installed_digest_matches(
208    executor: &impl CommandExecutor,
209    command: &CommandSpec,
210    expected: &str,
211) -> bool {
212    executor.execute(command).as_ref().is_ok_and(|output| {
213        output.status == 0
214            && String::from_utf8_lossy(&output.stdout)
215                .split_whitespace()
216                .next()
217                .is_some_and(|digest| digest.eq_ignore_ascii_case(expected))
218    })
219}
220
221fn refresh_worker_launch_if_stale(
222    executor: &impl CommandExecutor,
223    plan: Option<&WorkerLaunchRefreshPlan>,
224) -> Result<()> {
225    let Some(plan) = plan else {
226        return Ok(());
227    };
228    if installed_digest_matches(executor, &plan.installed_digest, &plan.expected_sha256) {
229        return Ok(());
230    }
231    plan.replace
232        .execute(executor)
233        .context("replace stale relay worker launch config")?;
234    Ok(())
235}
236
237#[cfg(test)]
238async fn recover_worker(
239    plan: WorkerRecoveryPlan,
240    restart_unresponsive: bool,
241) -> Result<WorkerRecoveryOutcome> {
242    recover_worker_for_session(plan, restart_unresponsive, None).await
243}
244
245async fn recover_worker_for_session(
246    plan: WorkerRecoveryPlan,
247    restart_unresponsive: bool,
248    session_id: Option<String>,
249) -> Result<WorkerRecoveryOutcome> {
250    tokio::task::spawn_blocking(move || {
251        let executor = CancellableProcessExecutor::with_timeout(WORKER_RESTART_TIMEOUT);
252        recover_worker_controlled(plan, restart_unresponsive, session_id.as_deref(), &executor)
253    })
254    .await
255    .context("worker recovery task failed")?
256}
257
258pub(crate) fn recover_worker_controlled(
259    mut plan: WorkerRecoveryPlan,
260    restart_unresponsive: bool,
261    session_id: Option<&str>,
262    executor: &impl CommandExecutor,
263) -> Result<WorkerRecoveryOutcome> {
264    let target_mutex = session_id.map(crate::recovery_gate::worker_target_mutex);
265    let _target_guard = target_mutex
266        .as_ref()
267        .map(|lock| {
268            lock.lock()
269                .map_err(|_| anyhow::anyhow!("worker target ownership lock poisoned"))
270        })
271        .transpose()?;
272    if let Some(id) = session_id {
273        let state =
274            crate::database::load_state().context("read durable session before worker recovery")?;
275        let eligible = state.sessions.get(id).is_some_and(|session| {
276            crate::pollers::session_target_is_pollable(session)
277                && session.target.as_ref() == Some(&plan.source_target)
278        });
279        if !eligible || crate::controller::move_session::move_owns_session(id) {
280            return Ok(WorkerRecoveryOutcome::Suppressed);
281        }
282    }
283    // A failed Move can leave this actor with a plan from before recovery.
284    // Never overwrite the durable checkpoint-only launch with that old plan.
285    if let Some(id) = session_id
286        && crate::database::load_move_operation(id)?
287            .is_some_and(|op| op.source_checkpoint_only && op.destination_target.is_none())
288    {
289        plan = crate::controller::Controller::load()?.worker_recovery_plan(id)?;
290    }
291    if ensure_recovery_target_running(executor, plan.target.as_ref())
292        .context("restore relay worker target")?
293        == TargetRecoveryOutcome::Missing
294    {
295        return Ok(WorkerRecoveryOutcome::TargetMissing);
296    }
297    let output = executor
298        .execute(&plan.liveness_probe)
299        .context("probe relay worker liveness")?;
300    if output.status != 0 {
301        bail!(
302            "{} failed with status {}: {}",
303            plan.liveness_probe.purpose,
304            output.status,
305            String::from_utf8_lossy(&output.stderr).trim()
306        );
307    }
308    match String::from_utf8_lossy(&output.stdout).trim() {
309        "starting" => Ok(WorkerRecoveryOutcome::Starting),
310        "alive" if !restart_unresponsive => Ok(WorkerRecoveryOutcome::Alive),
311        "alive" => {
312            if let Some(workspace) = plan.workspace.as_ref()
313                && !crate::controller::path_exists_on_managed_target(
314                    executor,
315                    &workspace.target,
316                    &workspace.directory,
317                )?
318            {
319                return Ok(WorkerRecoveryOutcome::WorkspaceMissing(
320                    workspace.directory.clone(),
321                ));
322            }
323            refresh_worker_binary_if_stale(executor, plan.binary_refresh.as_ref())?;
324            refresh_worker_launch_if_stale(executor, plan.launch_refresh.as_ref())?;
325            plan.restart.execute(executor)?;
326            Ok(WorkerRecoveryOutcome::RestartedUnresponsive)
327        }
328        "dead" => {
329            if let Some(workspace) = plan.workspace.as_ref()
330                && !crate::controller::path_exists_on_managed_target(
331                    executor,
332                    &workspace.target,
333                    &workspace.directory,
334                )?
335            {
336                return Ok(WorkerRecoveryOutcome::WorkspaceMissing(
337                    workspace.directory.clone(),
338                ));
339            }
340            refresh_worker_binary_if_stale(executor, plan.binary_refresh.as_ref())?;
341            refresh_worker_launch_if_stale(executor, plan.launch_refresh.as_ref())?;
342            plan.restart.execute(executor)?;
343            Ok(WorkerRecoveryOutcome::RestartedDead)
344        }
345        output => bail!("worker liveness probe returned unexpected output {output:?}"),
346    }
347}
348
349#[derive(Debug, Clone)]
350pub struct SessionManagerUpdate {
351    pub session_id: String,
352    pub view: ManagedSessionView,
353}
354
355pub struct SessionManagerChannels {
356    pub targets: watch::Sender<Vec<RelaySessionTarget>>,
357    pub control: SessionManagerControl,
358    pub updates: SessionManagerUpdates,
359    pub shutdown: SessionManagerShutdown,
360}
361
362/// Client-side half of a remotely owned session manager.
363///
364/// The daemon remains the only process with relay connections. A control
365/// surface publishes the daemon's latest views here and forwards requests from
366/// [`RemoteSessionRequests`] over its authenticated transport.
367pub struct RemoteSessionManagerChannels {
368    pub targets: watch::Sender<Vec<RelaySessionTarget>>,
369    pub control: SessionManagerControl,
370    pub updates: SessionManagerUpdates,
371    pub shutdown: SessionManagerShutdown,
372    pub publisher: RemoteSessionPublisher,
373    pub requests: RemoteSessionRequests,
374}
375
376#[derive(Clone)]
377pub struct RemoteSessionPublisher {
378    updates: mpsc::UnboundedSender<RemoteManagerUpdate>,
379}
380
381impl RemoteSessionPublisher {
382    pub async fn publish(&self, session_id: String, view: ManagedSessionView) -> Result<()> {
383        self.updates
384            .send(RemoteManagerUpdate::Publish { session_id, view })
385            .context("remote session manager stopped")
386    }
387
388    pub fn try_publish(&self, session_id: String, view: ManagedSessionView) -> Result<()> {
389        self.updates
390            .send(RemoteManagerUpdate::Publish { session_id, view })
391            .context("remote session manager update queue is unavailable")
392    }
393}
394
395pub struct RemoteSessionRequests {
396    requests: mpsc::Receiver<RemoteSessionRequest>,
397}
398
399impl RemoteSessionRequests {
400    pub async fn recv(&mut self) -> Option<RemoteSessionRequest> {
401        self.requests.recv().await
402    }
403}
404
405pub enum RemoteSessionRequest {
406    Submit {
407        session_id: String,
408        command_id: String,
409        command: RelayCommand,
410        admission: Option<ReviewDeliveryAdmission>,
411        reply: oneshot::Sender<std::result::Result<u64, String>>,
412    },
413    Sync {
414        session_id: String,
415        reply: oneshot::Sender<std::result::Result<(), String>>,
416    },
417    RespondElicitation {
418        session_id: String,
419        elicitation_id: String,
420        response: ElicitationResponse,
421        reply: oneshot::Sender<std::result::Result<(), String>>,
422    },
423    StopBackgroundTask {
424        session_id: String,
425        background_task_id: String,
426        reply: oneshot::Sender<std::result::Result<(), String>>,
427    },
428    Reviewer {
429        session_id: String,
430        /// Which reviewing role the action drives; `None` is the default one.
431        role: Option<String>,
432        action: ReviewerAction,
433        reply: oneshot::Sender<std::result::Result<ReviewerOutcome, String>>,
434    },
435}
436
437impl RemoteSessionRequest {
438    /// The session this request acts on. Requests for one session have to be
439    /// carried out in the order they were made.
440    pub fn session_id(&self) -> &str {
441        match self {
442            Self::Submit { session_id, .. }
443            | Self::Sync { session_id, .. }
444            | Self::RespondElicitation { session_id, .. }
445            | Self::StopBackgroundTask { session_id, .. }
446            | Self::Reviewer { session_id, .. } => session_id,
447        }
448    }
449}
450
451/// Keeps each session's relay requests in the order they were made, while
452/// letting different sessions overlap.
453///
454/// A bridge that spawns every request concurrently loses the order the caller
455/// submitted them in, and the order is load-bearing: `/effort` followed by a
456/// prompt has to reach the relay that way round, or the prompt runs under the
457/// old setting. Awaiting each request inline would restore the order but would
458/// also make one slow session block every other one, so instead each request
459/// waits on its own session's previous request and nothing else.
460#[derive(Default)]
461pub struct SessionRequestOrder {
462    latest: std::collections::HashMap<SessionRequestStream, tokio::task::JoinHandle<()>>,
463}
464
465#[derive(Debug, PartialEq, Eq, Hash)]
466enum SessionRequestStream {
467    Primary(String),
468    Reviewer(String, Option<String>),
469}
470
471impl SessionRequestOrder {
472    #[must_use]
473    pub fn new() -> Self {
474        Self::default()
475    }
476
477    /// Runs `forward` for `request` after everything already queued for the
478    /// same primary or reviewer role has finished. Independent reviewers
479    /// must not delay primary controls or one another.
480    pub fn dispatch<F, Fut>(&mut self, request: RemoteSessionRequest, forward: F)
481    where
482        F: FnOnce(RemoteSessionRequest) -> Fut + Send + 'static,
483        Fut: std::future::Future<Output = ()> + Send,
484    {
485        // Sessions that have gone quiet leave a finished handle behind; drop
486        // them here so the map tracks live work rather than every session the
487        // bridge has ever served.
488        self.latest.retain(|_, handle| !handle.is_finished());
489        let stream = match &request {
490            RemoteSessionRequest::Reviewer {
491                session_id, role, ..
492            } => SessionRequestStream::Reviewer(session_id.clone(), role.clone()),
493            _ => SessionRequestStream::Primary(request.session_id().to_owned()),
494        };
495        let previous = self.latest.remove(&stream);
496        let handle = tokio::spawn(async move {
497            if let Some(previous) = previous {
498                // A panicked predecessor still releases its successor: the
499                // request behind it is the user's, and dropping it silently
500                // would be worse than running it late.
501                if let Err(error) = previous.await {
502                    tracing::error!(%error, "previous session request task failed");
503                }
504            }
505            forward(request).await;
506        });
507        self.latest.insert(stream, handle);
508    }
509}
510
511/// Exclusive owner of the manager task and every relay actor below it.
512///
513/// Long-running control surfaces explicitly await [`Self::shutdown`] before
514/// their Tokio runtime goes away. Drop remains an aborting fallback for tests
515/// and early-return paths that cannot await.
516pub struct SessionManagerShutdown {
517    signal: Option<oneshot::Sender<()>>,
518    task: Option<tokio::task::JoinHandle<()>>,
519}
520
521impl SessionManagerShutdown {
522    pub async fn shutdown(mut self) -> Result<()> {
523        if let Some(signal) = self.signal.take() {
524            let _ = signal.send(());
525        }
526        if let Some(task) = self.task.take() {
527            task.await.context("session manager shutdown task failed")?;
528        }
529        Ok(())
530    }
531}
532
533impl Drop for SessionManagerShutdown {
534    fn drop(&mut self) {
535        if let Some(signal) = self.signal.take() {
536            let _ = signal.send(());
537        }
538        if let Some(task) = self.task.take() {
539            task.abort();
540        }
541    }
542}
543
544#[derive(Clone)]
545struct CoalescedUpdateSender {
546    pending: Arc<Mutex<BTreeMap<String, SessionManagerUpdate>>>,
547    wake: mpsc::Sender<()>,
548}
549
550/// Bounded latest-state feed for the dashboard. At most one snapshot per
551/// session is retained while the consumer is busy.
552pub struct SessionManagerUpdates {
553    pending: Arc<Mutex<BTreeMap<String, SessionManagerUpdate>>>,
554    wake: mpsc::Receiver<()>,
555}
556
557impl CoalescedUpdateSender {
558    fn send(&self, update: SessionManagerUpdate) {
559        if self.wake.is_closed() {
560            return;
561        }
562        self.pending
563            .lock()
564            .expect("session update coalescer poisoned")
565            .insert(update.session_id.clone(), update);
566        let _ = self.wake.try_send(());
567    }
568}
569
570impl SessionManagerUpdates {
571    fn pop_pending(&self) -> Option<SessionManagerUpdate> {
572        self.pending
573            .lock()
574            .expect("session update coalescer poisoned")
575            .pop_first()
576            .map(|(_, update)| update)
577    }
578
579    pub async fn recv(&mut self) -> Option<SessionManagerUpdate> {
580        loop {
581            if let Some(update) = self.pop_pending() {
582                return Some(update);
583            }
584            self.wake.recv().await?;
585        }
586    }
587
588    pub fn try_recv(
589        &mut self,
590    ) -> std::result::Result<SessionManagerUpdate, mpsc::error::TryRecvError> {
591        if let Some(update) = self.pop_pending() {
592            return Ok(update);
593        }
594        self.wake.try_recv()?;
595        self.pop_pending().ok_or(mpsc::error::TryRecvError::Empty)
596    }
597}
598
599fn coalesced_update_channel() -> (CoalescedUpdateSender, SessionManagerUpdates) {
600    let pending = Arc::new(Mutex::new(BTreeMap::new()));
601    let (wake_tx, wake_rx) = mpsc::channel(1);
602    (
603        CoalescedUpdateSender {
604            pending: pending.clone(),
605            wake: wake_tx,
606        },
607        SessionManagerUpdates {
608            pending,
609            wake: wake_rx,
610        },
611    )
612}
613
614#[derive(Clone)]
615pub struct SessionManagerControl {
616    commands: mpsc::Sender<ManagerCommand>,
617}
618
619#[derive(Clone, Debug)]
620pub struct ManagedSessionHandle {
621    session_id: String,
622    commands: mpsc::Sender<ActorCommand>,
623    releases: mpsc::UnboundedSender<ReturnedConnection>,
624    view: watch::Receiver<ManagedSessionView>,
625}
626
627/// A one-command capability issued by the review host while its prompt hold
628/// is open. It is intentionally opaque to callers: the session actor checks
629/// it against the host's live hold registry before bypassing prompt refusal.
630#[derive(Debug, Clone, PartialEq, Eq)]
631pub struct ReviewDeliveryAdmission {
632    session_id: String,
633    epoch: u64,
634    command_id: String,
635}
636
637impl ReviewDeliveryAdmission {
638    pub(crate) fn new(session_id: String, epoch: u64, command_id: String) -> Self {
639        Self {
640            session_id,
641            epoch,
642            command_id,
643        }
644    }
645
646    pub(crate) fn session_id(&self) -> &str {
647        &self.session_id
648    }
649
650    pub(crate) const fn epoch(&self) -> u64 {
651        self.epoch
652    }
653
654    pub(crate) fn command_id(&self) -> &str {
655        &self.command_id
656    }
657}
658
659/// Exclusive ownership of a session actor's existing relay connection.
660///
661/// Lifecycle operations use this instead of opening a competing projection
662/// client. Dropping an unreleased lease drops the proxy connection, which in
663/// turn cancels any ordinary relay checkpoint barrier.
664///
665/// Prompt submissions that arrive while the lease is active are not rejected.
666/// The actor queues them and forwards them in arrival order once the lease is
667/// released or dropped.
668pub struct ManagedSessionLease {
669    session_id: String,
670    lease_id: Option<u64>,
671    connection: Option<StandaloneSession>,
672    releases: mpsc::UnboundedSender<ReturnedConnection>,
673}
674
675impl ManagedSessionLease {
676    pub fn connection_mut(&mut self) -> &mut StandaloneSession {
677        self.connection
678            .as_mut()
679            .expect("managed session lease has already been released")
680    }
681
682    /// Swap the leased proxy after the worker process behind it was replaced.
683    /// The actor stays leased, so queued prompts cannot race the new latch.
684    pub fn replace_connection(&mut self, connection: StandaloneSession) {
685        drop(self.connection.take());
686        self.connection = Some(connection);
687    }
688
689    pub fn release(mut self) {
690        let lease_id = self
691            .lease_id
692            .take()
693            .expect("managed session lease has already been released");
694        let connection = self.connection.take();
695        if let Err(error) = self.releases.send(ReturnedConnection {
696            lease_id,
697            connection,
698        }) {
699            tracing::warn!(
700                session_id = %self.session_id,
701                operation = "lease_release",
702                %error,
703                "session actor stopped before receiving released relay connection"
704            );
705        }
706    }
707}
708
709impl Drop for ManagedSessionLease {
710    fn drop(&mut self) {
711        let Some(lease_id) = self.lease_id.take() else {
712            return;
713        };
714        // Drop the proxy before telling the actor to reconnect so the relay
715        // observes EOF and releases any abandoned checkpoint barrier first.
716        drop(self.connection.take());
717        if let Err(error) = self.releases.send(ReturnedConnection {
718            lease_id,
719            connection: None,
720        }) {
721            tracing::warn!(
722                session_id = %self.session_id,
723                operation = "lease_drop",
724                %error,
725                "session actor stopped before receiving dropped relay lease"
726            );
727        }
728    }
729}
730
731impl ManagedSessionHandle {
732    /// Narrow this controller-owned handle to the operations a control surface uses.
733    pub fn client(&self) -> mj_client::session::SessionHandle {
734        mj_client::session::SessionHandle::new(ClientSessionHandle(self.clone()))
735    }
736
737    pub fn session_id(&self) -> &str {
738        &self.session_id
739    }
740
741    pub fn view(&self) -> ManagedSessionView {
742        self.view.borrow().clone()
743    }
744
745    /// Whether the per-session actor behind this handle has retired. The
746    /// manager itself may still be alive with a replacement actor, so callers
747    /// holding long-lived handles use this to reacquire the current one.
748    pub fn is_stopped(&self) -> bool {
749        self.commands.is_closed()
750    }
751
752    pub fn has_changed(&self) -> Result<bool> {
753        self.view.has_changed().context("session manager stopped")
754    }
755
756    pub async fn changed(&mut self) -> Result<ManagedSessionView> {
757        self.view
758            .changed()
759            .await
760            .context("session manager stopped")?;
761        Ok(self.view())
762    }
763
764    pub async fn submit(&self, command_id: String, command: RelayCommand) -> Result<u64> {
765        self.enqueue_submit(command_id, command).await?.wait().await
766    }
767
768    /// Submit the review's corrective prompt through the one admission that
769    /// corresponds to its live prompt hold. Generic submissions continue to
770    /// use [`Self::submit`] and remain subject to review refusal.
771    pub(crate) async fn submit_review_delivery(
772        &self,
773        admission: ReviewDeliveryAdmission,
774        command: RelayCommand,
775    ) -> Result<u64> {
776        let command_id = admission.command_id.clone();
777        self.enqueue_submit_with_admission(command_id, command, Some(admission))
778            .await?
779            .wait()
780            .await
781    }
782
783    pub async fn enqueue_submit(
784        &self,
785        command_id: String,
786        command: RelayCommand,
787    ) -> Result<PendingRelaySubmit> {
788        self.enqueue_submit_with_admission(command_id, command, None)
789            .await
790    }
791
792    async fn enqueue_submit_with_admission(
793        &self,
794        command_id: String,
795        command: RelayCommand,
796        admission: Option<ReviewDeliveryAdmission>,
797    ) -> Result<PendingRelaySubmit> {
798        let (reply, response) = oneshot::channel();
799        self.commands
800            .send(ActorCommand::Submit {
801                command_id,
802                command,
803                admission,
804                reply,
805            })
806            .await
807            .context("session manager stopped")?;
808        Ok(PendingRelaySubmit { response })
809    }
810
811    pub async fn sync_now(&self) -> Result<()> {
812        self.enqueue_sync().await?.wait().await
813    }
814
815    pub async fn respond_elicitation(
816        &self,
817        elicitation_id: String,
818        response: ElicitationResponse,
819    ) -> Result<()> {
820        let (reply, result) = oneshot::channel();
821        self.commands
822            .send(ActorCommand::RespondElicitation {
823                elicitation_id,
824                response,
825                reply,
826            })
827            .await
828            .context("session manager stopped")?;
829        result
830            .await
831            .context("session manager stopped")?
832            .map_err(anyhow::Error::msg)
833    }
834
835    pub async fn stop_background_task(&self, background_task_id: String) -> Result<()> {
836        let (reply, result) = oneshot::channel();
837        self.commands
838            .send(ActorCommand::StopBackgroundTask {
839                background_task_id,
840                reply,
841            })
842            .await
843            .context("session manager stopped")?;
844        result
845            .await
846            .context("session manager stopped")?
847            .map_err(anyhow::Error::msg)
848    }
849
850    /// Drive the session's second-opinion reviewer.
851    ///
852    /// The reviewer shares this session's relay connection, so its actions
853    /// queue behind the session's own and are refused while a lifecycle
854    /// operation holds the connection.
855    pub async fn reviewer(&self, action: ReviewerAction) -> Result<ReviewerOutcome> {
856        self.reviewer_as(None, action).await
857    }
858
859    /// Drive one reviewing role. `None` is the default role, which is the one
860    /// plan review uses; a turn review in the extended tier names its
861    /// supervisor, its intent analyst, and each specialist lane.
862    pub async fn reviewer_as(
863        &self,
864        role: Option<String>,
865        action: ReviewerAction,
866    ) -> Result<ReviewerOutcome> {
867        let (reply, result) = oneshot::channel();
868        self.commands
869            .send(ActorCommand::Reviewer {
870                role,
871                action,
872                reply,
873            })
874            .await
875            .context("session manager stopped")?;
876        result
877            .await
878            .context("session manager stopped")?
879            .map_err(anyhow::Error::msg)
880    }
881
882    pub async fn enqueue_sync(&self) -> Result<PendingRelaySync> {
883        let (reply, response) = oneshot::channel();
884        self.commands
885            .send(ActorCommand::Sync { reply })
886            .await
887            .context("session manager stopped")?;
888        Ok(PendingRelaySync { response })
889    }
890
891    pub async fn lease_connection(&self) -> Result<ManagedSessionLease> {
892        let (reply, response) = oneshot::channel();
893        self.commands
894            .send(ActorCommand::Lease { reply })
895            .await
896            .context("session manager stopped")?;
897        let (lease_id, connection) = response.await.context("session manager stopped")??;
898        Ok(ManagedSessionLease {
899            session_id: self.session_id.clone(),
900            lease_id: Some(lease_id),
901            connection: Some(connection),
902            releases: self.releases.clone(),
903        })
904    }
905}
906
907pub struct PendingRelaySubmit {
908    response: oneshot::Receiver<std::result::Result<u64, String>>,
909}
910
911impl PendingRelaySubmit {
912    pub async fn wait(self) -> Result<u64> {
913        self.response
914            .await
915            .context("session manager stopped")?
916            .map_err(anyhow::Error::msg)
917    }
918}
919
920pub struct PendingRelaySync {
921    response: oneshot::Receiver<std::result::Result<(), String>>,
922}
923
924impl PendingRelaySync {
925    pub async fn wait(self) -> Result<()> {
926        self.response
927            .await
928            .context("session manager stopped")?
929            .map_err(anyhow::Error::msg)
930    }
931}
932
933#[derive(Clone)]
934struct ClientSessionHandle(ManagedSessionHandle);
935
936impl mj_client::session::SessionHandleBackend for ClientSessionHandle {
937    fn search_prompts(
938        &self,
939        bundle_id: String,
940        scope: mj_core::storage::HistoryScope,
941        query: String,
942    ) -> mj_client::session::BoxFuture<'_, Result<Vec<mj_core::storage::PromptHistoryEntry>>> {
943        let session_id = self.0.session_id().to_owned();
944        Box::pin(async move {
945            tokio::task::spawn_blocking(move || {
946                crate::database::search_prompts(&session_id, &bundle_id, scope, &query)
947            })
948            .await
949            .context("history search task")?
950        })
951    }
952    fn review_state(
953        &self,
954    ) -> mj_client::session::BoxFuture<'_, Result<mj_client::session::ReviewState>> {
955        let session_id = self.0.session_id().to_owned();
956        Box::pin(async move {
957            tokio::task::spawn_blocking(move || {
958                Ok(mj_client::session::ReviewState {
959                    review: crate::database::active_review(&session_id)?,
960                    defaults: crate::database::reviewer_defaults()?,
961                })
962            })
963            .await
964            .context("review restoration task")?
965        })
966    }
967
968    fn config_result(
969        &self,
970        command_id: String,
971    ) -> mj_client::session::BoxFuture<'_, Result<Option<Option<String>>>> {
972        let session_id = self.session_id().to_owned();
973        Box::pin(async move {
974            tokio::task::spawn_blocking(move || {
975                crate::database::load_config_result(&session_id, &command_id)
976            })
977            .await
978            .context("read configuration completion task")?
979        })
980    }
981
982    fn clone_box(&self) -> Box<dyn mj_client::session::SessionHandleBackend> {
983        Box::new(self.clone())
984    }
985
986    fn session_id(&self) -> &str {
987        self.0.session_id()
988    }
989
990    fn view(&self) -> ManagedSessionView {
991        self.0.view()
992    }
993
994    fn is_stopped(&self) -> bool {
995        self.0.is_stopped()
996    }
997
998    fn has_changed(&self) -> Result<bool> {
999        self.0.has_changed()
1000    }
1001
1002    fn changed(&mut self) -> mj_client::session::BoxFuture<'_, Result<ManagedSessionView>> {
1003        Box::pin(self.0.changed())
1004    }
1005
1006    fn enqueue_submit(
1007        &self,
1008        command_id: String,
1009        command: RelayCommand,
1010    ) -> mj_client::session::BoxFuture<'_, Result<mj_client::session::PendingRelaySubmit>> {
1011        Box::pin(async move {
1012            let pending = self.0.enqueue_submit(command_id, command).await?;
1013            Ok(mj_client::session::PendingRelaySubmit::new(Box::pin(
1014                pending.wait(),
1015            )))
1016        })
1017    }
1018
1019    fn enqueue_sync(
1020        &self,
1021    ) -> mj_client::session::BoxFuture<'_, Result<mj_client::session::PendingRelaySync>> {
1022        Box::pin(async move {
1023            let pending = self.0.enqueue_sync().await?;
1024            Ok(mj_client::session::PendingRelaySync::new(Box::pin(
1025                pending.wait(),
1026            )))
1027        })
1028    }
1029
1030    fn respond_elicitation(
1031        &self,
1032        elicitation_id: String,
1033        response: ElicitationResponse,
1034    ) -> mj_client::session::BoxFuture<'_, Result<()>> {
1035        Box::pin(self.0.respond_elicitation(elicitation_id, response))
1036    }
1037
1038    fn stop_background_task(
1039        &self,
1040        background_task_id: String,
1041    ) -> mj_client::session::BoxFuture<'_, Result<()>> {
1042        Box::pin(self.0.stop_background_task(background_task_id))
1043    }
1044
1045    fn reviewer(
1046        &self,
1047        role: Option<String>,
1048        action: ReviewerAction,
1049    ) -> mj_client::session::BoxFuture<'_, Result<ReviewerOutcome>> {
1050        Box::pin(self.0.reviewer_as(role, action))
1051    }
1052}
1053
1054#[derive(Clone)]
1055struct ClientSessionControl(SessionManagerControl);
1056
1057impl mj_client::session::SessionControlBackend for ClientSessionControl {
1058    fn session(
1059        &self,
1060        session_id: String,
1061    ) -> mj_client::session::BoxFuture<'_, Result<mj_client::session::SessionHandle>> {
1062        Box::pin(async move { Ok(self.0.session(session_id).await?.client()) })
1063    }
1064}
1065
1066impl SessionManagerControl {
1067    /// Narrow this controller-owned manager to session lookup for a control surface.
1068    pub fn client(&self) -> mj_client::session::SessionControl {
1069        mj_client::session::SessionControl::new(ClientSessionControl(self.clone()))
1070    }
1071
1072    pub async fn session(&self, session_id: impl Into<String>) -> Result<ManagedSessionHandle> {
1073        let session_id = session_id.into();
1074        let (reply, response) = oneshot::channel();
1075        self.commands
1076            .send(ManagerCommand::Session {
1077                session_id: session_id.clone(),
1078                reply,
1079            })
1080            .await
1081            .context("session manager stopped")?;
1082        response
1083            .await
1084            .context("session manager stopped")?
1085            .with_context(|| format!("session {session_id} is not managed"))
1086    }
1087
1088    pub async fn wait_for_session(
1089        &self,
1090        session_id: &str,
1091        timeout: Duration,
1092    ) -> Result<ManagedSessionHandle> {
1093        tokio::time::timeout(timeout, async {
1094            loop {
1095                match self.session(session_id.to_owned()).await {
1096                    Ok(handle) => return Ok(handle),
1097                    Err(error) => {
1098                        tracing::trace!(session_id, "waiting for session actor: {error:#}");
1099                        tokio::time::sleep(Duration::from_millis(25)).await;
1100                    }
1101                }
1102            }
1103        })
1104        .await
1105        .with_context(|| {
1106            format!(
1107                "session {session_id} did not become available within {} seconds",
1108                timeout.as_secs()
1109            )
1110        })?
1111    }
1112}
1113
1114enum ManagerCommand {
1115    Session {
1116        session_id: String,
1117        reply: oneshot::Sender<Option<ManagedSessionHandle>>,
1118    },
1119}
1120
1121enum ActorCommand {
1122    Submit {
1123        command_id: String,
1124        command: RelayCommand,
1125        admission: Option<ReviewDeliveryAdmission>,
1126        reply: oneshot::Sender<std::result::Result<u64, String>>,
1127    },
1128    Sync {
1129        reply: oneshot::Sender<std::result::Result<(), String>>,
1130    },
1131    RespondElicitation {
1132        elicitation_id: String,
1133        response: ElicitationResponse,
1134        reply: oneshot::Sender<std::result::Result<(), String>>,
1135    },
1136    StopBackgroundTask {
1137        background_task_id: String,
1138        reply: oneshot::Sender<std::result::Result<(), String>>,
1139    },
1140    Reviewer {
1141        role: Option<String>,
1142        action: ReviewerAction,
1143        reply: oneshot::Sender<std::result::Result<ReviewerOutcome, String>>,
1144    },
1145    /// The connection is handed over whole, and so is the failure: a caller
1146    /// that must decide whether to restart the worker needs the typed cause,
1147    /// which formatting the error to a string would destroy.
1148    Lease {
1149        reply: oneshot::Sender<Result<(u64, StandaloneSession)>>,
1150    },
1151}
1152
1153impl ActorCommand {
1154    fn operation_name(&self) -> &'static str {
1155        match self {
1156            Self::Submit { .. } => "submit",
1157            Self::Sync { .. } => "sync",
1158            Self::RespondElicitation { .. } => "respond_elicitation",
1159            Self::StopBackgroundTask { .. } => "stop_background_task",
1160            Self::Reviewer { action, .. } => action.operation_name(),
1161            Self::Lease { .. } => "lease",
1162        }
1163    }
1164
1165    fn reject(self, session_id: &str, message: &str) {
1166        match self {
1167            Self::Submit { reply, .. } => {
1168                if reply.send(Err(message.to_owned())).is_err() {
1169                    tracing::debug!(
1170                        %session_id,
1171                        operation = "submit",
1172                        "submit rejection receiver was already closed"
1173                    );
1174                }
1175            }
1176            Self::Sync { reply } => {
1177                if reply.send(Err(message.to_owned())).is_err() {
1178                    tracing::debug!(
1179                        %session_id,
1180                        operation = "sync",
1181                        "sync rejection receiver was already closed"
1182                    );
1183                }
1184            }
1185            Self::RespondElicitation { reply, .. } => {
1186                if reply.send(Err(message.to_owned())).is_err() {
1187                    tracing::debug!(
1188                        %session_id,
1189                        operation = "respond_elicitation",
1190                        "elicitation rejection receiver was already closed"
1191                    );
1192                }
1193            }
1194            Self::StopBackgroundTask { reply, .. } => {
1195                if reply.send(Err(message.to_owned())).is_err() {
1196                    tracing::debug!(
1197                        %session_id,
1198                        operation = "stop_background_task",
1199                        "background task stop rejection receiver was already closed"
1200                    );
1201                }
1202            }
1203            Self::Reviewer { reply, .. } => {
1204                if reply.send(Err(message.to_owned())).is_err() {
1205                    tracing::debug!(
1206                        %session_id,
1207                        operation = "reviewer",
1208                        "reviewer rejection receiver was already closed"
1209                    );
1210                }
1211            }
1212            Self::Lease { reply } => {
1213                if reply
1214                    .send(Err(anyhow::anyhow!(message.to_owned())))
1215                    .is_err()
1216                {
1217                    tracing::debug!(
1218                        %session_id,
1219                        operation = "lease",
1220                        "lease rejection receiver was already closed"
1221                    );
1222                }
1223            }
1224        }
1225    }
1226}
1227
1228struct ReturnedConnection {
1229    lease_id: u64,
1230    connection: Option<StandaloneSession>,
1231}
1232
1233/// A submission that arrived while a lifecycle operation held the connection.
1234/// The actor replays these in arrival order once the lease comes back.
1235struct DeferredSubmit {
1236    command_id: String,
1237    command: RelayCommand,
1238    admission: Option<ReviewDeliveryAdmission>,
1239    reply: oneshot::Sender<std::result::Result<u64, String>>,
1240}
1241
1242#[derive(Debug, Default)]
1243struct ActorLifecycle {
1244    active_lease: Option<u64>,
1245    retirement_requested: bool,
1246}
1247
1248impl ActorLifecycle {
1249    fn set_retirement_requested(&mut self, requested: bool) {
1250        self.retirement_requested = requested;
1251    }
1252
1253    fn is_leased(&self) -> bool {
1254        self.active_lease.is_some()
1255    }
1256
1257    fn should_stop(&self) -> bool {
1258        self.retirement_requested && !self.is_leased()
1259    }
1260
1261    fn accepts_new_work(&self) -> bool {
1262        !self.retirement_requested
1263    }
1264
1265    fn activate_lease(&mut self, lease_id: u64) {
1266        debug_assert!(self.active_lease.is_none());
1267        self.active_lease = Some(lease_id);
1268    }
1269
1270    fn return_lease(&mut self, lease_id: u64) -> bool {
1271        if self.active_lease != Some(lease_id) {
1272            return false;
1273        }
1274        self.active_lease = None;
1275        true
1276    }
1277}
1278
1279struct ActorRegistration {
1280    target: RelaySessionTarget,
1281    commands: mpsc::Sender<ActorCommand>,
1282    releases: mpsc::UnboundedSender<ReturnedConnection>,
1283    retirement: watch::Sender<bool>,
1284    view: watch::Receiver<ManagedSessionView>,
1285    abort: tokio::task::AbortHandle,
1286}
1287
1288struct RemoteActorRegistration {
1289    commands: mpsc::Sender<ActorCommand>,
1290    releases: mpsc::UnboundedSender<ReturnedConnection>,
1291    view: watch::Receiver<ManagedSessionView>,
1292    view_tx: watch::Sender<ManagedSessionView>,
1293    abort: tokio::task::AbortHandle,
1294}
1295
1296enum RemoteManagerUpdate {
1297    Publish {
1298        session_id: String,
1299        view: ManagedSessionView,
1300    },
1301}
1302
1303#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1304enum ReconcileAction {
1305    Idle,
1306    Spawn,
1307    Keep,
1308    Retire,
1309}
1310
1311fn reconcile_action(
1312    actor: Option<&RelaySessionTarget>,
1313    desired: Option<&RelaySessionTarget>,
1314) -> ReconcileAction {
1315    match (actor, desired) {
1316        (None, None) => ReconcileAction::Idle,
1317        (None, Some(_)) => ReconcileAction::Spawn,
1318        (Some(actor), Some(desired)) if actor == desired => ReconcileAction::Keep,
1319        (Some(_), Some(_) | None) => ReconcileAction::Retire,
1320    }
1321}
1322
1323fn target_map(targets: &[RelaySessionTarget]) -> BTreeMap<String, RelaySessionTarget> {
1324    targets
1325        .iter()
1326        .cloned()
1327        .map(|target| (target.session_id.clone(), target))
1328        .collect()
1329}
1330
1331fn remove_actor_task(
1332    actors: &mut BTreeMap<String, ActorRegistration>,
1333    task_id: tokio::task::Id,
1334) -> Option<String> {
1335    let session_id = actors.iter().find_map(|(session_id, actor)| {
1336        (actor.abort.id() == task_id).then(|| session_id.clone())
1337    })?;
1338    actors.remove(&session_id);
1339    Some(session_id)
1340}
1341
1342fn reconcile_actors(
1343    targets: &BTreeMap<String, RelaySessionTarget>,
1344    actors: &mut BTreeMap<String, ActorRegistration>,
1345    tasks: &mut tokio::task::JoinSet<String>,
1346    updates: &CoalescedUpdateSender,
1347) {
1348    // A completed or cancelled task closes its command receiver before the
1349    // JoinSet completion necessarily wins the manager's select. Do not let
1350    // that dead registration suppress the replacement this reconciliation is
1351    // responsible for starting. Task-ID-aware completion cleanup below keeps
1352    // the old completion from removing the replacement later.
1353    actors.retain(|session_id, actor| {
1354        let live = !actor.commands.is_closed();
1355        if !live {
1356            tracing::warn!(session_id, "replacing stopped session relay actor");
1357        }
1358        live
1359    });
1360
1361    for (session_id, actor) in actors.iter() {
1362        let retiring = matches!(
1363            reconcile_action(Some(&actor.target), targets.get(session_id)),
1364            ReconcileAction::Retire
1365        );
1366        actor.retirement.send_replace(retiring);
1367    }
1368
1369    for (session_id, target) in targets {
1370        if !matches!(
1371            reconcile_action(
1372                actors.get(session_id).map(|actor| &actor.target),
1373                Some(target)
1374            ),
1375            ReconcileAction::Spawn
1376        ) {
1377            continue;
1378        }
1379        let (actor_tx, actor_rx) = mpsc::channel(32);
1380        let (release_tx, release_rx) = mpsc::unbounded_channel();
1381        let (retirement_tx, retirement_rx) = watch::channel(false);
1382        let (view_tx, view_rx) = watch::channel(ManagedSessionView::default());
1383        let actor_updates = updates.clone();
1384        let task_target = target.clone();
1385        let task_id = session_id.clone();
1386        let abort = tasks.spawn(async move {
1387            run_session_actor(
1388                task_target,
1389                actor_rx,
1390                release_rx,
1391                retirement_rx,
1392                view_tx,
1393                actor_updates,
1394            )
1395            .await;
1396            task_id
1397        });
1398        actors.insert(
1399            session_id.clone(),
1400            ActorRegistration {
1401                target: target.clone(),
1402                commands: actor_tx,
1403                releases: release_tx,
1404                retirement: retirement_tx,
1405                view: view_rx,
1406                abort,
1407            },
1408        );
1409    }
1410}
1411
1412async fn run_remote_session_actor(
1413    session_id: String,
1414    mut commands: mpsc::Receiver<ActorCommand>,
1415    requests: mpsc::Sender<RemoteSessionRequest>,
1416) {
1417    while let Some(command) = commands.recv().await {
1418        let request = match command {
1419            ActorCommand::Submit {
1420                command_id,
1421                command,
1422                admission,
1423                reply,
1424            } => RemoteSessionRequest::Submit {
1425                session_id: session_id.clone(),
1426                command_id,
1427                command,
1428                admission,
1429                reply,
1430            },
1431            ActorCommand::Sync { reply } => RemoteSessionRequest::Sync {
1432                session_id: session_id.clone(),
1433                reply,
1434            },
1435            ActorCommand::RespondElicitation {
1436                elicitation_id,
1437                response,
1438                reply,
1439            } => RemoteSessionRequest::RespondElicitation {
1440                session_id: session_id.clone(),
1441                elicitation_id,
1442                response,
1443                reply,
1444            },
1445            ActorCommand::StopBackgroundTask {
1446                background_task_id,
1447                reply,
1448            } => RemoteSessionRequest::StopBackgroundTask {
1449                session_id: session_id.clone(),
1450                background_task_id,
1451                reply,
1452            },
1453            ActorCommand::Reviewer {
1454                role,
1455                action,
1456                reply,
1457            } => RemoteSessionRequest::Reviewer {
1458                session_id: session_id.clone(),
1459                role,
1460                action,
1461                reply,
1462            },
1463            ActorCommand::Lease { reply } => {
1464                let _ = reply.send(Err(anyhow::anyhow!(
1465                    "relay connection leases are available only inside the controller daemon"
1466                )));
1467                continue;
1468            }
1469        };
1470        if let Err(error) = requests.send(request).await {
1471            match error.0 {
1472                RemoteSessionRequest::Submit { reply, .. } => {
1473                    let _ = reply.send(Err("controller daemon request bridge stopped".into()));
1474                }
1475                RemoteSessionRequest::Sync { reply, .. }
1476                | RemoteSessionRequest::RespondElicitation { reply, .. }
1477                | RemoteSessionRequest::StopBackgroundTask { reply, .. } => {
1478                    let _ = reply.send(Err("controller daemon request bridge stopped".into()));
1479                }
1480                RemoteSessionRequest::Reviewer { reply, .. } => {
1481                    let _ = reply.send(Err("controller daemon request bridge stopped".into()));
1482                }
1483            }
1484            break;
1485        }
1486    }
1487}
1488
1489fn spawn_remote_actor(
1490    session_id: String,
1491    view: ManagedSessionView,
1492    requests: &mpsc::Sender<RemoteSessionRequest>,
1493    actors: &mut BTreeMap<String, RemoteActorRegistration>,
1494    updates: &CoalescedUpdateSender,
1495) {
1496    let (actor_tx, actor_rx) = mpsc::channel(32);
1497    let (release_tx, _release_rx) = mpsc::unbounded_channel();
1498    let (view_tx, view_rx) = watch::channel(view.clone());
1499    let abort = tokio::spawn(run_remote_session_actor(
1500        session_id.clone(),
1501        actor_rx,
1502        requests.clone(),
1503    ))
1504    .abort_handle();
1505    actors.insert(
1506        session_id.clone(),
1507        RemoteActorRegistration {
1508            commands: actor_tx,
1509            releases: release_tx,
1510            view: view_rx,
1511            view_tx,
1512            abort,
1513        },
1514    );
1515    updates.send(SessionManagerUpdate { session_id, view });
1516}
1517
1518/// Build the read/control facade used by a control surface whose relay actors
1519/// live in another process. Target updates still decide which session handles
1520/// exist, while [`RemoteSessionPublisher`] supplies their latest views.
1521pub fn spawn_remote_session_manager() -> Result<RemoteSessionManagerChannels> {
1522    let (targets_tx, mut targets_rx) = watch::channel(Vec::<RelaySessionTarget>::new());
1523    let (commands_tx, mut commands_rx) = mpsc::channel(32);
1524    let (updates_tx, updates_rx) = coalesced_update_channel();
1525    let (published_tx, mut published_rx) = mpsc::unbounded_channel();
1526    let (requests_tx, requests_rx) = mpsc::channel(64);
1527    let (shutdown_tx, mut shutdown_rx) = oneshot::channel();
1528    let task = tokio::spawn(async move {
1529        let mut actors = BTreeMap::<String, RemoteActorRegistration>::new();
1530        let mut latest = BTreeMap::<String, ManagedSessionView>::new();
1531        let mut desired = BTreeMap::<String, RelaySessionTarget>::new();
1532        loop {
1533            tokio::select! {
1534                _ = &mut shutdown_rx => break,
1535                changed = targets_rx.changed() => {
1536                    if changed.is_err() {
1537                        break;
1538                    }
1539                    desired = target_map(&targets_rx.borrow_and_update());
1540                    actors.retain(|session_id, actor| {
1541                        if desired.contains_key(session_id) {
1542                            true
1543                        } else {
1544                            actor.abort.abort();
1545                            false
1546                        }
1547                    });
1548                    for session_id in desired.keys() {
1549                        if !actors.contains_key(session_id)
1550                            && let Some(view) = latest.get(session_id).cloned()
1551                        {
1552                            spawn_remote_actor(
1553                                session_id.clone(),
1554                                view,
1555                                &requests_tx,
1556                                &mut actors,
1557                                &updates_tx,
1558                            );
1559                        }
1560                    }
1561                }
1562                command = commands_rx.recv() => {
1563                    let Some(ManagerCommand::Session { session_id, reply }) = command else {
1564                        break;
1565                    };
1566                    let handle = actors.get(&session_id).map(|actor| ManagedSessionHandle {
1567                        session_id: session_id.clone(),
1568                        commands: actor.commands.clone(),
1569                        releases: actor.releases.clone(),
1570                        view: actor.view.clone(),
1571                    });
1572                    let _ = reply.send(handle);
1573                }
1574                published = published_rx.recv() => {
1575                    let Some(RemoteManagerUpdate::Publish { session_id, view }) = published else {
1576                        break;
1577                    };
1578                    latest.insert(session_id.clone(), view.clone());
1579                    if !desired.contains_key(&session_id) {
1580                        continue;
1581                    }
1582                    if let Some(actor) = actors.get(&session_id) {
1583                        publish_view(&session_id, view, &actor.view_tx, &updates_tx);
1584                        continue;
1585                    }
1586                    spawn_remote_actor(
1587                        session_id,
1588                        view,
1589                        &requests_tx,
1590                        &mut actors,
1591                        &updates_tx,
1592                    );
1593                }
1594            }
1595        }
1596        for actor in actors.into_values() {
1597            actor.abort.abort();
1598        }
1599    });
1600    Ok(RemoteSessionManagerChannels {
1601        targets: targets_tx,
1602        control: SessionManagerControl {
1603            commands: commands_tx,
1604        },
1605        updates: updates_rx,
1606        shutdown: SessionManagerShutdown {
1607            signal: Some(shutdown_tx),
1608            task: Some(task),
1609        },
1610        publisher: RemoteSessionPublisher {
1611            updates: published_tx,
1612        },
1613        requests: RemoteSessionRequests {
1614            requests: requests_rx,
1615        },
1616    })
1617}
1618
1619pub fn spawn_session_manager() -> Result<SessionManagerChannels> {
1620    let (targets_tx, mut targets_rx) = watch::channel(Vec::<RelaySessionTarget>::new());
1621    let (commands_tx, mut commands_rx) = mpsc::channel(32);
1622    let (updates_tx, updates_rx) = coalesced_update_channel();
1623    let (shutdown_tx, mut shutdown_rx) = oneshot::channel();
1624    let task = tokio::spawn(async move {
1625        let mut actors = BTreeMap::<String, ActorRegistration>::new();
1626        let mut tasks = tokio::task::JoinSet::<String>::new();
1627        let mut desired_targets = BTreeMap::<String, RelaySessionTarget>::new();
1628        loop {
1629            tokio::select! {
1630                _ = &mut shutdown_rx => break,
1631                changed = targets_rx.changed() => {
1632                    if changed.is_err() {
1633                        break;
1634                    }
1635                    desired_targets = target_map(&targets_rx.borrow_and_update());
1636                    reconcile_actors(
1637                        &desired_targets,
1638                        &mut actors,
1639                        &mut tasks,
1640                        &updates_tx,
1641                    );
1642                }
1643                command = commands_rx.recv() => {
1644                    let Some(ManagerCommand::Session { session_id, reply }) = command else {
1645                        break;
1646                    };
1647                    let handle = actors
1648                        .get(&session_id)
1649                        .filter(|actor| !actor.commands.is_closed())
1650                        .filter(|actor| desired_targets.get(&session_id) == Some(&actor.target))
1651                        .map(|actor| ManagedSessionHandle {
1652                            session_id: session_id.clone(),
1653                            commands: actor.commands.clone(),
1654                            releases: actor.releases.clone(),
1655                            view: actor.view.clone(),
1656                        });
1657                    if reply.send(handle).is_err() {
1658                        tracing::debug!(
1659                            session_id = %session_id,
1660                            operation = "session_lookup",
1661                            "session lookup receiver was already closed"
1662                        );
1663                    }
1664                }
1665                joined = tasks.join_next_with_id(), if !tasks.is_empty() => {
1666                    match joined {
1667                        Some(Ok((task_id, session_id))) => {
1668                            let removed = remove_actor_task(&mut actors, task_id);
1669                            if removed.as_deref().is_some_and(|removed| removed != session_id) {
1670                                tracing::error!(
1671                                    completed_session_id = session_id,
1672                                    registered_session_id = removed,
1673                                    "session relay actor completed under the wrong registration"
1674                                );
1675                            }
1676                            // A watch sender may have published another target while this
1677                            // completion was already ready. Reconcile against its newest
1678                            // value so an intermediate replacement is never started.
1679                            desired_targets = target_map(&targets_rx.borrow());
1680                            reconcile_actors(
1681                                &desired_targets,
1682                                &mut actors,
1683                                &mut tasks,
1684                                &updates_tx,
1685                            );
1686                        }
1687                        Some(Err(error)) if error.is_cancelled() => {
1688                            let cancelled_task = error.id();
1689                            let session_id = remove_actor_task(&mut actors, cancelled_task);
1690                            desired_targets = target_map(&targets_rx.borrow());
1691                            reconcile_actors(
1692                                &desired_targets,
1693                                &mut actors,
1694                                &mut tasks,
1695                                &updates_tx,
1696                            );
1697                            tracing::warn!(
1698                                session_id = ?session_id,
1699                                "cancelled session relay actor was replaced"
1700                            );
1701                        }
1702                        Some(Err(error)) => {
1703                            let failed_task = error.id();
1704                            remove_actor_task(&mut actors, failed_task);
1705                            desired_targets = target_map(&targets_rx.borrow());
1706                            reconcile_actors(
1707                                &desired_targets,
1708                                &mut actors,
1709                                &mut tasks,
1710                                &updates_tx,
1711                            );
1712                            tracing::error!(%error, "session relay actor failed");
1713                        }
1714                        None => {}
1715                    }
1716                }
1717            }
1718        }
1719        shutdown_session_actors(&mut actors, &mut tasks).await;
1720    });
1721    Ok(SessionManagerChannels {
1722        targets: targets_tx,
1723        control: SessionManagerControl {
1724            commands: commands_tx,
1725        },
1726        updates: updates_rx,
1727        shutdown: SessionManagerShutdown {
1728            signal: Some(shutdown_tx),
1729            task: Some(task),
1730        },
1731    })
1732}
1733
1734async fn shutdown_session_actors(
1735    actors: &mut BTreeMap<String, ActorRegistration>,
1736    tasks: &mut tokio::task::JoinSet<String>,
1737) {
1738    for actor in actors.values() {
1739        actor.retirement.send_replace(true);
1740    }
1741    actors.clear();
1742
1743    let graceful = async {
1744        while let Some(joined) = tasks.join_next().await {
1745            match joined {
1746                Ok(_) => {}
1747                Err(error) if error.is_cancelled() => {}
1748                Err(error) => {
1749                    tracing::error!(%error, "session relay actor failed during shutdown");
1750                }
1751            }
1752        }
1753    };
1754    if tokio::time::timeout(SESSION_MANAGER_SHUTDOWN_GRACE, graceful)
1755        .await
1756        .is_ok()
1757    {
1758        return;
1759    }
1760
1761    tracing::warn!(
1762        timeout_ms = SESSION_MANAGER_SHUTDOWN_GRACE.as_millis(),
1763        "session relay actors did not stop before the shutdown deadline; aborting them"
1764    );
1765    tasks.abort_all();
1766    while let Some(joined) = tasks.join_next().await {
1767        if let Err(error) = joined
1768            && !error.is_cancelled()
1769        {
1770            tracing::error!(%error, "session relay actor failed while being aborted");
1771        }
1772    }
1773}
1774
1775async fn run_session_actor(
1776    target: RelaySessionTarget,
1777    mut commands: mpsc::Receiver<ActorCommand>,
1778    mut releases: mpsc::UnboundedReceiver<ReturnedConnection>,
1779    mut retirement: watch::Receiver<bool>,
1780    view_tx: watch::Sender<ManagedSessionView>,
1781    updates: CoalescedUpdateSender,
1782) {
1783    let mut connection: Option<StandaloneSession> = None;
1784    let mut failures = 0_u32;
1785    let mut last_recovery_probe = None;
1786    let mut lifecycle = ActorLifecycle::default();
1787    let mut deferred_submits: VecDeque<DeferredSubmit> = VecDeque::new();
1788    let mut next_lease_id = 1_u64;
1789    let mut reviewer_tasks = tokio::task::JoinSet::new();
1790    let mut reviewer_connections = BTreeMap::new();
1791    let mut reviewer_tails = BTreeMap::new();
1792    let mut reviewer_cancellation = tokio_util::sync::CancellationToken::new();
1793    let mut interval = tokio::time::interval(SESSION_SYNC_INTERVAL);
1794    interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
1795    loop {
1796        lifecycle.set_retirement_requested(*retirement.borrow_and_update());
1797        if lifecycle.should_stop() {
1798            break;
1799        }
1800        tokio::select! {
1801            completed = reviewer_tasks.join_next(), if !reviewer_tasks.is_empty() => {
1802                if let Some(Err(error)) = completed {
1803                    tracing::error!(session_id = %target.session_id, %error, "reviewer operation task failed");
1804                }
1805            }
1806            _ = interval.tick() => {
1807                lifecycle.set_retirement_requested(*retirement.borrow());
1808                if lifecycle.should_stop() {
1809                    break;
1810                }
1811                if lifecycle.is_leased() {
1812                    continue;
1813                }
1814                let result = sync_actor_connection(
1815                    &target,
1816                    &mut connection,
1817                ).await;
1818                match result {
1819                    Ok(snapshot) => {
1820                        failures = 0;
1821                        if let Some(snapshot) = snapshot {
1822                            publish_view(&target.session_id, ManagedSessionView {
1823                                snapshot: Some(snapshot),
1824                                connected: true,
1825                                error: None,
1826                            }, &view_tx, &updates);
1827                        }
1828                    }
1829                    Err(error) => {
1830                        connection = None;
1831                        failures = failures.saturating_add(1);
1832                        // A projection integrity failure repeats on every
1833                        // retry, so report it at once rather than waiting for
1834                        // the unreachable threshold.
1835                        let integrity = projection_integrity_failure(&error);
1836                        tracing::warn!(
1837                            session_id = target.session_id,
1838                            consecutive_failures = failures,
1839                            projection_integrity = integrity,
1840                            transport_dead = worker_connect_needs_restart(&error),
1841                            "session relay sync failed: {error:#}"
1842                        );
1843                        let recovery_due = !integrity
1844                            && !crate::controller::move_session::move_owns_session(&target.session_id)
1845                            && failures >= UNREACHABLE_FAILURE_THRESHOLD
1846                            && worker_connect_needs_restart(&error)
1847                            && target.worker_recovery.is_some()
1848                            && last_recovery_probe.is_none_or(|last: tokio::time::Instant| {
1849                                last.elapsed() >= WORKER_RESTART_COOLDOWN
1850                            });
1851                        if integrity || failures >= UNREACHABLE_FAILURE_THRESHOLD {
1852                            // Bind the clone first: borrowing inside the call
1853                            // would hold the watch read guard while
1854                            // `publish_view` takes the write lock, deadlocking
1855                            // this actor on its own view.
1856                            let snapshot = view_tx.borrow().snapshot.clone();
1857                            let mut detail = format!("{error:#}");
1858                            if recovery_due {
1859                                detail.push_str("; checking whether the relay worker is dead");
1860                            }
1861                            publish_view(&target.session_id, ManagedSessionView {
1862                                snapshot,
1863                                connected: false,
1864                                error: Some(if integrity {
1865                                    ViewError::ProjectionIntegrity(detail)
1866                                } else {
1867                                    ViewError::Unreachable(detail)
1868                                }),
1869                            }, &view_tx, &updates);
1870                        }
1871                        if recovery_due {
1872                            last_recovery_probe = Some(tokio::time::Instant::now());
1873                            let plan = target
1874                                .worker_recovery
1875                                .clone()
1876                                .expect("recovery eligibility requires a plan");
1877                            let restart_unresponsive =
1878                                worker_connect_allows_live_restart(&error);
1879                            tracing::warn!(
1880                                session_id = target.session_id,
1881                                "relay worker is unreachable; probing it before recovery: {error:#}"
1882                            );
1883                            match recover_worker_for_session(plan, restart_unresponsive, Some(target.session_id.clone())).await {
1884                                Ok(
1885                                    outcome @ (WorkerRecoveryOutcome::RestartedDead
1886                                    | WorkerRecoveryOutcome::RestartedUnresponsive),
1887                                ) => {
1888                                    failures = 0;
1889                                    let snapshot = view_tx.borrow().snapshot.clone();
1890                                    let recovery = match outcome {
1891                                        WorkerRecoveryOutcome::RestartedDead => {
1892                                            "confirmed the relay worker was dead and restarted it"
1893                                        }
1894                                        WorkerRecoveryOutcome::RestartedUnresponsive => {
1895                                            "the relay worker was alive but not serving handshakes, so it was restarted"
1896                                        }
1897                                        WorkerRecoveryOutcome::Alive
1898                                        | WorkerRecoveryOutcome::Starting
1899                                        | WorkerRecoveryOutcome::TargetMissing
1900                                        | WorkerRecoveryOutcome::Suppressed
1901                                        | WorkerRecoveryOutcome::WorkspaceMissing(_) => {
1902                                            unreachable!()
1903                                        }
1904                                    };
1905                                    publish_view(&target.session_id, ManagedSessionView {
1906                                        snapshot,
1907                                        connected: false,
1908                                        error: Some(ViewError::Unreachable(format!(
1909                                            "{error:#}; {recovery}"
1910                                        ))),
1911                                    }, &view_tx, &updates);
1912                                    interval.reset_after(RECONNECT_INTERVAL);
1913                                }
1914                                Ok(WorkerRecoveryOutcome::Alive) => {
1915                                    tracing::warn!(
1916                                        session_id = target.session_id,
1917                                        "relay transport failed but the worker is alive; leaving it running"
1918                                    );
1919                                    let snapshot = view_tx.borrow().snapshot.clone();
1920                                    publish_view(&target.session_id, ManagedSessionView {
1921                                        snapshot,
1922                                        connected: false,
1923                                        error: Some(ViewError::Unreachable(format!(
1924                                            "{error:#}; relay worker is still alive, so it was not restarted"
1925                                        ))),
1926                                    }, &view_tx, &updates);
1927                                    interval.reset_after(reconnect_delay(failures));
1928                                }
1929                                Ok(WorkerRecoveryOutcome::Starting) => {
1930                                    tracing::warn!(
1931                                        session_id = target.session_id,
1932                                        "relay worker is still starting; leaving it running"
1933                                    );
1934                                    let snapshot = view_tx.borrow().snapshot.clone();
1935                                    publish_view(&target.session_id, ManagedSessionView {
1936                                        snapshot,
1937                                        connected: false,
1938                                        error: Some(ViewError::Unreachable(format!(
1939                                            "{error:#}; relay worker is still recovering its durable state, so it was not restarted"
1940                                        ))),
1941                                    }, &view_tx, &updates);
1942                                    interval.reset_after(reconnect_delay(failures));
1943                                }
1944                                Ok(WorkerRecoveryOutcome::Suppressed) => {
1945                                    tracing::info!(
1946                                        session_id = target.session_id,
1947                                        "automatic worker recovery suppressed by durable lifecycle or target change"
1948                                    );
1949                                    // The desired-target refresher will remove or replace this
1950                                    // stale actor. It must not reconnect in the meantime.
1951                                    break;
1952                                }
1953                                Ok(WorkerRecoveryOutcome::TargetMissing) => {
1954                                    let snapshot = view_tx.borrow().snapshot.clone();
1955                                    publish_view(&target.session_id, ManagedSessionView {
1956                                        snapshot,
1957                                        connected: false,
1958                                        error: Some(ViewError::TargetMissing(
1959                                            "the managed Podman session container no longer exists"
1960                                                .into(),
1961                                        )),
1962                                    }, &view_tx, &updates);
1963                                    interval.reset_after(RECONNECT_BACKOFF_CEILING);
1964                                }
1965                                Ok(WorkerRecoveryOutcome::WorkspaceMissing(directory)) => {
1966                                    let snapshot = view_tx.borrow().snapshot.clone();
1967                                    publish_view(&target.session_id, ManagedSessionView {
1968                                        snapshot,
1969                                        connected: false,
1970                                        error: Some(ViewError::TargetMissing(format!(
1971                                            "the worker working directory {} is missing; resume this session from its recovery archive to restore it",
1972                                            directory.display(),
1973                                        ))),
1974                                    }, &view_tx, &updates);
1975                                    interval.reset_after(RECONNECT_BACKOFF_CEILING);
1976                                }
1977                                Err(recovery_error) => {
1978                                    tracing::warn!(
1979                                        session_id = target.session_id,
1980                                        "automatic relay worker recovery failed safely: {recovery_error:#}"
1981                                    );
1982                                    let snapshot = view_tx.borrow().snapshot.clone();
1983                                    publish_view(&target.session_id, ManagedSessionView {
1984                                        snapshot,
1985                                        connected: false,
1986                                        error: Some(ViewError::Unreachable(format!(
1987                                            "{error:#}; could not confirm the relay worker was dead, so it was not restarted: {recovery_error:#}"
1988                                        ))),
1989                                    }, &view_tx, &updates);
1990                                    interval.reset_after(reconnect_delay(failures));
1991                                }
1992                            }
1993                        } else {
1994                            interval.reset_after(reconnect_delay(failures));
1995                        }
1996                    }
1997                }
1998            }
1999            command = commands.recv() => {
2000                let Some(command) = command else { break };
2001                lifecycle.set_retirement_requested(*retirement.borrow());
2002                if !lifecycle.accepts_new_work() {
2003                    tracing::debug!(
2004                        session_id = %target.session_id,
2005                        operation = command.operation_name(),
2006                        "rejecting relay operation while session target changes"
2007                    );
2008                    command.reject(&target.session_id, "session target is changing");
2009                    continue;
2010                }
2011                match command {
2012                    ActorCommand::Submit {
2013                        command_id,
2014                        command,
2015                        admission,
2016                        reply,
2017                    } => {
2018                        if crate::controller::move_session::move_refuses_command(&target.session_id, &command) {
2019                            let _ = reply.send(Err("session is moving; keep the draft and retry after Move finishes".into()));
2020                            continue;
2021                        }
2022                        // A turn under review holds its session's prompts. The
2023                        // sole exception is a capability issued by the review
2024                        // host for this exact corrective command; ordinary
2025                        // prompts and controller-authored notices still take
2026                        // the refusal path below.
2027                        let admitted = admission.as_ref().is_some_and(|admission| {
2028                            matches!(&command, RelayCommand::Prompt { .. })
2029                                && admission.command_id() == command_id
2030                                && crate::review_host::review_delivery_admitted(
2031                                    &target.session_id,
2032                                    admission,
2033                                )
2034                        });
2035                        if admission.is_some() && !admitted {
2036                            let _ = reply.send(Err(
2037                                "review delivery admission is no longer valid".to_owned(),
2038                            ));
2039                            continue;
2040                        }
2041                        if matches!(&command, RelayCommand::Prompt { .. })
2042                            && !admitted
2043                            && let Some(refusal) =
2044                                crate::review_host::prompt_refusal(&target.session_id)
2045                        {
2046                            tracing::debug!(
2047                                session_id = %target.session_id,
2048                                %command_id,
2049                                "refusing a prompt while a turn review is unresolved"
2050                            );
2051                            let _ = reply.send(Err(refusal.to_owned()));
2052                            continue;
2053                        }
2054                        if lifecycle.is_leased() {
2055                            // A checkpoint or other lifecycle operation owns the
2056                            // connection. Hold the prompt instead of rejecting it
2057                            // and deliver it when the lease comes back.
2058                            deferred_submits.push_back(DeferredSubmit {
2059                                command_id,
2060                                command,
2061                                admission,
2062                                reply,
2063                            });
2064                            continue;
2065                        }
2066                        deliver_submit(
2067                            &target,
2068                            &mut connection,
2069                            DeferredSubmit { command_id, command, admission, reply },
2070                            &view_tx,
2071                            &updates,
2072                        )
2073                        .await;
2074                    }
2075                    ActorCommand::Sync { reply } => {
2076                        if lifecycle.is_leased() {
2077                            tracing::debug!(
2078                                session_id = %target.session_id,
2079                                operation = "sync",
2080                                "rejecting sync while session is leased"
2081                            );
2082                            if reply
2083                                .send(Err("session is reserved for a lifecycle operation".into()))
2084                                .is_err()
2085                            {
2086                                tracing::debug!(
2087                                    session_id = %target.session_id,
2088                                    operation = "sync",
2089                                    "sync rejection receiver was already closed"
2090                                );
2091                            }
2092                            continue;
2093                        }
2094                        let result = sync_actor_connection(
2095                            &target,
2096                            &mut connection,
2097                        ).await.map(|snapshot| {
2098                            if let Some(snapshot) = snapshot {
2099                                publish_view(&target.session_id, ManagedSessionView {
2100                                    snapshot: Some(snapshot),
2101                                    connected: true,
2102                                    error: None,
2103                                }, &view_tx, &updates);
2104                            }
2105                        });
2106                        if result.is_err() {
2107                            connection = None;
2108                        }
2109                        if let Err(error) = &result {
2110                            tracing::warn!(
2111                                session_id = %target.session_id,
2112                                operation = "sync",
2113                                error = %error,
2114                                "explicit relay synchronization failed"
2115                            );
2116                        }
2117                    if reply.send(result.map_err(|error| format!("{error:#}"))).is_err() {
2118                        tracing::debug!(
2119                            session_id = %target.session_id,
2120                            operation = "sync",
2121                            "sync result receiver was already closed"
2122                        );
2123                    }
2124                    }
2125                    ActorCommand::Reviewer {
2126                        role,
2127                        action,
2128                        reply,
2129                    } => {
2130                        if lifecycle.is_leased() || crate::controller::move_session::move_owns_session(&target.session_id) {
2131                            // A lifecycle operation owns the connection, and a
2132                            // reviewer action is not worth deferring: the user
2133                            // is waiting on its answer now.
2134                            tracing::debug!(
2135                                session_id = %target.session_id,
2136                                operation = action.operation_name(),
2137                                "rejecting a reviewer action while the session is leased"
2138                            );
2139                            if reply
2140                                .send(Err("session is reserved for a lifecycle operation".into()))
2141                                .is_err()
2142                            {
2143                                tracing::debug!(
2144                                    session_id = %target.session_id,
2145                                    operation = "reviewer",
2146                                    "reviewer rejection receiver was already closed"
2147                                );
2148                            }
2149                            continue;
2150                        }
2151                        // A slow harness startup or analysis must not occupy
2152                        // the primary's relay or serialize independent roles.
2153                        // Cache each role's connection so transcript polling
2154                        // does not launch a new SSH/Podman proxy every time.
2155                        let cached = reviewer_connections.entry(role.clone())
2156                            .or_insert_with(|| Arc::new(tokio::sync::Mutex::new(None)))
2157                            .clone();
2158                        let (finished, tail) = oneshot::channel::<()>();
2159                        let previous = reviewer_tails.insert(role.clone(), tail);
2160                        let target = target.clone();
2161                        let cancelled = reviewer_cancellation.clone();
2162                        reviewer_tasks.spawn(async move {
2163                            if let Some(previous) = previous {
2164                                let _ = previous.await;
2165                            }
2166                            run_reviewer_operation(target, role, action, reply, cached, cancelled).await;
2167                            drop(finished);
2168                        });
2169                    }
2170                    ActorCommand::RespondElicitation {
2171                        elicitation_id,
2172                        response,
2173                        reply,
2174                    } => {
2175                        if lifecycle.is_leased() {
2176                            tracing::debug!(
2177                                session_id = %target.session_id,
2178                                operation = "respond_elicitation",
2179                                "rejecting elicitation response while session is leased"
2180                            );
2181                            if reply
2182                                .send(Err("session is reserved for a lifecycle operation".into()))
2183                                .is_err()
2184                            {
2185                                tracing::debug!(
2186                                    session_id = %target.session_id,
2187                                    operation = "respond_elicitation",
2188                                    "elicitation rejection receiver was already closed"
2189                                );
2190                            }
2191                            continue;
2192                        }
2193                        let result = async {
2194                            sync_actor_connection(&target, &mut connection).await?;
2195                            let connection = connection
2196                                .as_mut()
2197                                .context("relay is disconnected")?;
2198                            connection
2199                                .respond_elicitation(elicitation_id, response)
2200                                .await?;
2201                            Ok::<_, anyhow::Error>(connection.snapshot())
2202                        }
2203                        .await;
2204                        match result {
2205                            Ok(ref snapshot) => publish_view(
2206                                &target.session_id,
2207                                ManagedSessionView {
2208                                    snapshot: Some(snapshot.clone()),
2209                                    connected: true,
2210                                    error: None,
2211                                },
2212                                &view_tx,
2213                                &updates,
2214                            ),
2215                            Err(ref error) if !is_final_rejection(error) => connection = None,
2216                            Err(_) => {}
2217                        }
2218                        if let Err(error) = &result {
2219                            tracing::warn!(
2220                                session_id = %target.session_id,
2221                                operation = "respond_elicitation",
2222                                error = %error,
2223                                "relay elicitation response failed"
2224                            );
2225                        }
2226                        if reply
2227                            .send(result.map(|_| ()).map_err(|error| format!("{error:#}")))
2228                            .is_err()
2229                        {
2230                            tracing::debug!(
2231                                session_id = %target.session_id,
2232                                operation = "respond_elicitation",
2233                                "elicitation result receiver was already closed"
2234                            );
2235                        }
2236                    }
2237                    ActorCommand::StopBackgroundTask {
2238                        background_task_id,
2239                        reply,
2240                    } => {
2241                        if lifecycle.is_leased() {
2242                            let _ = reply.send(Err(
2243                                "session is reserved for a lifecycle operation".into(),
2244                            ));
2245                            continue;
2246                        }
2247                        let result = async {
2248                            sync_actor_connection(&target, &mut connection).await?;
2249                            let connection = connection
2250                                .as_mut()
2251                                .context("relay is disconnected")?;
2252                            connection.stop_background_task(background_task_id).await?;
2253                            Ok::<_, anyhow::Error>(connection.snapshot())
2254                        }
2255                        .await;
2256                        match result {
2257                            Ok(ref snapshot) => publish_view(
2258                                &target.session_id,
2259                                ManagedSessionView {
2260                                    snapshot: Some(snapshot.clone()),
2261                                    connected: true,
2262                                    error: None,
2263                                },
2264                                &view_tx,
2265                                &updates,
2266                            ),
2267                            Err(ref error) if !is_final_rejection(error) => connection = None,
2268                            Err(_) => {}
2269                        }
2270                        if let Err(error) = &result {
2271                            tracing::warn!(
2272                                session_id = %target.session_id,
2273                                operation = "stop_background_task",
2274                                error = %error,
2275                                "relay background task stop failed"
2276                            );
2277                        }
2278                        if reply
2279                            .send(result.map(|_| ()).map_err(|error| format!("{error:#}")))
2280                            .is_err()
2281                        {
2282                            tracing::debug!(
2283                                session_id = %target.session_id,
2284                                operation = "stop_background_task",
2285                                "background task stop result receiver was already closed"
2286                            );
2287                        }
2288                    }
2289                    ActorCommand::Lease { reply } => {
2290                        if lifecycle.is_leased() {
2291                            tracing::debug!(
2292                                session_id = %target.session_id,
2293                                operation = "lease",
2294                                "rejecting duplicate session lifecycle lease"
2295                            );
2296                            if reply
2297                                .send(Err(anyhow::anyhow!(
2298                                    "session already has a lifecycle operation"
2299                                )))
2300                                .is_err()
2301                            {
2302                                tracing::debug!(
2303                                    session_id = %target.session_id,
2304                                    operation = "lease",
2305                                    "lease rejection receiver was already closed"
2306                                );
2307                            }
2308                            continue;
2309                        }
2310                        let lease_id = next_lease_id;
2311                        reviewer_cancellation.cancel();
2312                        reviewer_cancellation = tokio_util::sync::CancellationToken::new();
2313                        reviewer_connections.clear();
2314                        reviewer_tails.clear();
2315                        let result = sync_actor_connection(
2316                            &target,
2317                            &mut connection,
2318                        )
2319                        .await
2320                        .map(|_| {
2321                            next_lease_id = next_lease_id.wrapping_add(1).max(1);
2322                            (
2323                                lease_id,
2324                                connection
2325                                    .take()
2326                                    .expect("successful sync retained its connection"),
2327                            )
2328                        });
2329                        if result.is_err() {
2330                            connection = None;
2331                        }
2332                        if let Err(error) = &result {
2333                            tracing::warn!(
2334                                session_id = %target.session_id,
2335                                operation = "lease",
2336                                error = %error,
2337                                "could not acquire relay session lease"
2338                            );
2339                        }
2340                        let acquired = result.is_ok();
2341                        match reply.send(result) {
2342                            Ok(()) if acquired => lifecycle.activate_lease(lease_id),
2343                            Ok(()) => {}
2344                            Err(Ok((_lease_id, returned))) => connection = Some(returned),
2345                            Err(Err(_)) => {}
2346                        }
2347                    }
2348                }
2349            }
2350            returned = releases.recv() => {
2351                let Some(returned) = returned else { continue };
2352                if lifecycle.return_lease(returned.lease_id) {
2353                    // A dropped lease returns no connection; `submit_actor_command`
2354                    // reconnects on demand, so the drain needs no special case.
2355                    connection = returned.connection;
2356                    failures = 0;
2357                    interval.reset();
2358                    // A lease syncs the connection it borrowed, so this actor's
2359                    // next sync can find nothing left to apply. Publish what the
2360                    // returned connection already knows or watchers keep reading
2361                    // pre-lease state.
2362                    if let Some(returned) = connection.as_ref() {
2363                        publish_view(&target.session_id, ManagedSessionView {
2364                            snapshot: Some(returned.snapshot()),
2365                            connected: true,
2366                            error: None,
2367                        }, &view_tx, &updates);
2368                    }
2369                    let retiring = *retirement.borrow();
2370                    while let Some(deferred) = deferred_submits.pop_front() {
2371                        if retiring {
2372                            if deferred
2373                                .reply
2374                                .send(Err("session target is changing".into()))
2375                                .is_err()
2376                            {
2377                                tracing::debug!(
2378                                    session_id = %target.session_id,
2379                                    operation = "submit",
2380                                    "deferred submit rejection receiver was already closed"
2381                                );
2382                            }
2383                            continue;
2384                        }
2385                        deliver_submit(
2386                            &target,
2387                            &mut connection,
2388                            deferred,
2389                            &view_tx,
2390                            &updates,
2391                        )
2392                        .await;
2393                    }
2394                }
2395            }
2396            changed = retirement.changed() => {
2397                if changed.is_err() {
2398                    break;
2399                }
2400            }
2401        }
2402    }
2403    reviewer_cancellation.cancel();
2404    reviewer_connections.clear();
2405    reviewer_tails.clear();
2406    while let Some(completed) = reviewer_tasks.join_next().await {
2407        if let Err(error) = completed {
2408            tracing::error!(session_id = %target.session_id, %error, "reviewer operation task failed during shutdown");
2409        }
2410    }
2411    if let Some(connection) = connection.take()
2412        && let Err(error) = connection.detach().await
2413    {
2414        tracing::warn!(
2415            session_id = %target.session_id,
2416            %error,
2417            "could not detach relay connection during session actor shutdown"
2418        );
2419    }
2420    // No caller may wait forever on a submission this actor will never deliver.
2421    for deferred in deferred_submits {
2422        if deferred
2423            .reply
2424            .send(Err("session manager stopped".into()))
2425            .is_err()
2426        {
2427            tracing::debug!(
2428                session_id = %target.session_id,
2429                operation = "submit",
2430                "deferred submit shutdown receiver was already closed"
2431            );
2432        }
2433    }
2434}
2435
2436/// Submit one relay command and publish the resulting snapshot. Live and
2437/// deferred submissions share this path so both report identical results.
2438async fn deliver_submit(
2439    target: &RelaySessionTarget,
2440    connection: &mut Option<StandaloneSession>,
2441    submission: DeferredSubmit,
2442    view_tx: &watch::Sender<ManagedSessionView>,
2443    updates: &CoalescedUpdateSender,
2444) {
2445    let DeferredSubmit {
2446        command_id,
2447        command,
2448        admission,
2449        reply,
2450    } = submission;
2451    if crate::controller::move_session::move_refuses_command(&target.session_id, &command) {
2452        let _ = reply.send(Err(
2453            "session is moving; keep the draft and retry after Move finishes".into(),
2454        ));
2455        return;
2456    }
2457    if let Some(admission) = admission.as_ref()
2458        && (!matches!(&command, RelayCommand::Prompt { .. })
2459            || admission.command_id() != command_id
2460            || !crate::review_host::review_delivery_admitted(&target.session_id, admission))
2461    {
2462        let _ = reply.send(Err(
2463            "review delivery admission is no longer valid".to_owned()
2464        ));
2465        return;
2466    }
2467    let result = submit_actor_command(target, connection, &command_id, &command).await;
2468    if let Err(error) = result.as_ref() {
2469        tracing::warn!(
2470            session_id = %target.session_id,
2471            operation = "submit",
2472            %command_id,
2473            retryable = !is_final_rejection(error),
2474            error = %error,
2475            "relay command submission failed"
2476        );
2477    }
2478    if let Err(error) = result.as_ref()
2479        && !is_final_rejection(error)
2480    {
2481        *connection = None;
2482    }
2483    let accepted = result.as_ref().ok().copied();
2484    // Answer the caller the moment the relay has the command. Catching the
2485    // local projection up to it is the expensive half and nobody waiting to
2486    // hear "accepted" needs it first: the caller has an ordinal, and the view
2487    // it would read is published below anyway.
2488    if reply
2489        .send(result.map_err(|error| format!("{error:#}")))
2490        .is_err()
2491    {
2492        tracing::debug!(
2493            session_id = %target.session_id,
2494            operation = "submit",
2495            %command_id,
2496            "submit result receiver was already closed"
2497        );
2498    }
2499    let Some(ordinal) = accepted else {
2500        return;
2501    };
2502    tracing::trace!(%ordinal, %command_id, "relay command accepted");
2503    let Some(session) = connection.as_mut() else {
2504        return;
2505    };
2506    // The command landed either way, so a failed catch-up is a connection
2507    // problem to retire rather than a failed submission: the caller has
2508    // already been told the relay took it.
2509    match session.sync().await {
2510        Ok(snapshot) => publish_view(
2511            &target.session_id,
2512            ManagedSessionView {
2513                snapshot: Some(snapshot),
2514                connected: true,
2515                error: None,
2516            },
2517            view_tx,
2518            updates,
2519        ),
2520        Err(error) => {
2521            tracing::warn!(
2522                session_id = %target.session_id,
2523                operation = "submit",
2524                %command_id,
2525                error = %format!("{error:#}"),
2526                "projection could not catch up to an accepted command"
2527            );
2528            *connection = None;
2529        }
2530    }
2531}
2532
2533/// Whether the relay refused this request outright.
2534///
2535/// A refusal is a completed round trip, so the connection is healthy. Dropping
2536/// it would discard whatever that connection owns on the worker, including a
2537/// checkpoint barrier a controller is still holding.
2538fn is_final_rejection(error: &anyhow::Error) -> bool {
2539    error
2540        .downcast_ref::<RelayRejected>()
2541        .is_some_and(|rejected| !rejected.is_retryable())
2542}
2543
2544async fn submit_actor_command(
2545    target: &RelaySessionTarget,
2546    connection: &mut Option<StandaloneSession>,
2547    command_id: &str,
2548    command: &RelayCommand,
2549) -> Result<u64> {
2550    let mut first_error = None;
2551    for attempt in 1..=2 {
2552        if connection.is_none() {
2553            sync_actor_connection(target, connection).await?;
2554        }
2555        let result = connection
2556            .as_mut()
2557            .context("relay is disconnected")?
2558            .submit_accepted(command_id.to_owned(), command.clone())
2559            .await;
2560        match result {
2561            Ok(ordinal) => return Ok(ordinal),
2562            // A final rejection is a completed round trip: the relay read the
2563            // command and refused it, so retrying would only be refused again.
2564            // Reconnecting would also cancel any checkpoint barrier this
2565            // connection owns, which is how a controller probing for a command
2566            // an older worker does not understand would lose it.
2567            Err(error) if is_final_rejection(&error) => return Err(error),
2568            Err(error) => {
2569                tracing::warn!(
2570                    session_id = %target.session_id,
2571                    operation = "submit",
2572                    %command_id,
2573                    attempt,
2574                    retryable = true,
2575                    error = %error,
2576                    "retryable relay command failure; reconnecting"
2577                );
2578                if first_error.is_none() {
2579                    first_error = Some(format!("{error:#}"));
2580                }
2581                *connection = None;
2582            }
2583        }
2584    }
2585    let detail = first_error.unwrap_or_else(|| "relay submission failed".into());
2586    bail!("relay command {command_id} failed after an idempotent reconnect: {detail}")
2587}
2588
2589/// Perform one reviewer action on a synchronized relay connection.
2590///
2591/// The reviewer's own relay answers most of these, so the outcomes mirror the
2592/// primary's: an attach page, an acknowledgement cursor, an accepted command.
2593async fn run_reviewer_operation(
2594    target: RelaySessionTarget,
2595    role: Option<String>,
2596    action: ReviewerAction,
2597    mut reply: oneshot::Sender<std::result::Result<ReviewerOutcome, String>>,
2598    cached: Arc<tokio::sync::Mutex<Option<RelayClient>>>,
2599    cancelled: tokio_util::sync::CancellationToken,
2600) {
2601    let operation = action.operation_name();
2602    let keep_connection = !matches!(&action, ReviewerAction::Pause);
2603    let result = tokio::select! {
2604        biased;
2605        _ = cancelled.cancelled() => Err(anyhow::anyhow!("reviewer operation cancelled for session lifecycle change")),
2606        _ = reply.closed() => return,
2607        result = async {
2608            let mut cache = cached.lock().await;
2609            // Take ownership while a request is in flight: dropping this
2610            // future closes its connection instead of leaving a late reply
2611            // available for the next request to misinterpret.
2612            let mut client = match cache.take() {
2613                Some(client) => client,
2614                None => RelayClient::connect(&target.spec, &target.session_id).await?,
2615            };
2616            let result = drive_reviewer(&mut client, role, action).await;
2617            if keep_connection && (result.is_ok() || result.as_ref().is_err_and(is_final_rejection)) {
2618                *cache = Some(client);
2619            }
2620            result
2621        } => result,
2622    };
2623    if let Err(error) = &result {
2624        tracing::warn!(session_id = %target.session_id, %operation, error = %error, "reviewer action failed");
2625    }
2626    if reply
2627        .send(result.map_err(|error| format!("{error:#}")))
2628        .is_err()
2629    {
2630        tracing::debug!(session_id = %target.session_id, %operation, "reviewer result receiver was already closed");
2631    }
2632}
2633
2634async fn drive_reviewer(
2635    client: &mut RelayClient,
2636    role: Option<String>,
2637    action: ReviewerAction,
2638) -> Result<ReviewerOutcome> {
2639    let role = role.as_deref();
2640    Ok(match action {
2641        ReviewerAction::Start { config } => {
2642            ReviewerOutcome::Started(Box::new(client.start_reviewer(role, *config).await?))
2643        }
2644        ReviewerAction::Submit {
2645            command_id,
2646            command,
2647        } => ReviewerOutcome::Accepted {
2648            ordinal: client.submit_to_reviewer(role, command_id, command).await?,
2649        },
2650        ReviewerAction::Attach {
2651            after_ordinal,
2652            after_digest,
2653        } => ReviewerOutcome::Attached(Box::new(
2654            client
2655                .attach_reviewer(role, after_ordinal, after_digest)
2656                .await?,
2657        )),
2658        ReviewerAction::Acknowledge {
2659            through_ordinal,
2660            through_digest,
2661        } => ReviewerOutcome::Acknowledged(
2662            client
2663                .acknowledge_reviewer(role, through_ordinal, through_digest)
2664                .await?,
2665        ),
2666        ReviewerAction::Status => {
2667            ReviewerOutcome::Status(Box::new(client.reviewer_status(role).await?))
2668        }
2669        ReviewerAction::RespondElicitation {
2670            elicitation_id,
2671            response,
2672        } => {
2673            client
2674                .respond_to_reviewer(role, elicitation_id, response)
2675                .await?;
2676            ReviewerOutcome::ElicitationResolved
2677        }
2678        ReviewerAction::Pause => {
2679            client.pause_reviewer(role).await?;
2680            ReviewerOutcome::Paused
2681        }
2682        ReviewerAction::CaptureDelta { baselines } => ReviewerOutcome::Delta {
2683            repositories: client.capture_review_delta(role, baselines).await?,
2684        },
2685        ReviewerAction::AdvanceBaseline { trees } => {
2686            client.advance_review_baseline(role, trees).await?;
2687            ReviewerOutcome::BaselineAdvanced
2688        }
2689        ReviewerAction::AnalyzeDelta { repositories } => ReviewerOutcome::ChangedFunctions {
2690            packet: client.analyze_review_delta(role, repositories).await?,
2691        },
2692        ReviewerAction::TakeLaneDispatches => ReviewerOutcome::LaneDispatches {
2693            requests: client.take_lane_dispatches().await?,
2694        },
2695    })
2696}
2697
2698async fn sync_actor_connection(
2699    target: &RelaySessionTarget,
2700    connection: &mut Option<StandaloneSession>,
2701) -> Result<Option<ManagedSessionSnapshot>> {
2702    if connection.is_none() {
2703        *connection = Some(StandaloneSession::connect(target).await?);
2704        return Ok(Some(
2705            connection
2706                .as_ref()
2707                .expect("connection was initialized")
2708                .snapshot(),
2709        ));
2710    }
2711    let connection = connection.as_mut().expect("connection was initialized");
2712    if connection.sync_in_place().await? {
2713        Ok(Some(connection.snapshot()))
2714    } else {
2715        Ok(None)
2716    }
2717}
2718
2719/// Cheap equivalence for published views.
2720///
2721/// The materialized projection is a function of the relay event chain, so its
2722/// transcript can only differ when the applied event frontier differs. Every
2723/// sync tick would otherwise walk the whole conversation to prove nothing
2724/// changed. The remaining scalars are compared directly because they are small
2725/// and bound the projection's non-transcript state.
2726fn view_is_unchanged(current: &ManagedSessionView, next: &ManagedSessionView) -> bool {
2727    if current.connected != next.connected || current.error != next.error {
2728        return false;
2729    }
2730    match (&current.snapshot, &next.snapshot) {
2731        (None, None) => true,
2732        (Some(current), Some(next)) => {
2733            let (current_session, next_session) = (&current.materialized, &next.materialized);
2734            current.latest_credential_sync_signal == next.latest_credential_sync_signal
2735                && current.operational == next.operational
2736                && current_session.session_id == next_session.session_id
2737                && current_session.applied_event_ordinal == next_session.applied_event_ordinal
2738                && current_session.applied_event_digest == next_session.applied_event_digest
2739                && current_session.last_activity_at_ms == next_session.last_activity_at_ms
2740                && current_session.execution == next_session.execution
2741                && current_session.session_title == next_session.session_title
2742                && current_session.queued_prompts == next_session.queued_prompts
2743        }
2744        (None, Some(_)) | (Some(_), None) => false,
2745    }
2746}
2747
2748fn publish_view(
2749    session_id: &str,
2750    view: ManagedSessionView,
2751    watch: &watch::Sender<ManagedSessionView>,
2752    updates: &CoalescedUpdateSender,
2753) {
2754    // Compare and replace under one lock acquisition; a separate
2755    // `watch.borrow()` check would reacquire the lock and invite the
2756    // read-then-write deadlock this function's callers must avoid.
2757    let changed = watch.send_if_modified(|current| {
2758        if view_is_unchanged(current, &view) {
2759            return false;
2760        }
2761        *current = view.clone();
2762        true
2763    });
2764    if changed {
2765        updates.send(SessionManagerUpdate {
2766            session_id: session_id.to_owned(),
2767            view,
2768        });
2769    }
2770}
2771
2772/// Read a stored projection without blocking the runtime. The rusqlite read
2773/// and the transcript deserialization behind it are synchronous and grow with
2774/// the conversation, so a long session must not stall a worker thread that
2775/// other actors share.
2776async fn load_projection(session_id: &str) -> Result<MaterializedSession> {
2777    let session_id = session_id.to_owned();
2778    tokio::task::spawn_blocking(move || -> Result<MaterializedSession> {
2779        let loaded = crate::database::load_materialized_session(&session_id)?;
2780        Ok(loaded.unwrap_or_else(|| MaterializedSession::empty(session_id)))
2781    })
2782    .await
2783    .context("controller projection load task failed")?
2784}
2785
2786pub struct StandaloneSession {
2787    client: RelayClient,
2788    materialized: MaterializedSession,
2789    operational: RelayOperationalState,
2790    latest_credential_sync_signal: Option<CredentialSyncSignal>,
2791    project_memory: Option<ProjectMemorySyncTarget>,
2792    subagent_requests: Vec<mj_core::subagent::SubagentToolRequest>,
2793    subagent_results: Vec<mj_core::subagent::SubagentToolResult>,
2794}
2795
2796impl StandaloneSession {
2797    pub fn set_project_memory_target(&mut self, target: Option<ProjectMemorySyncTarget>) {
2798        self.project_memory = target;
2799    }
2800
2801    pub async fn connect(target: &RelaySessionTarget) -> Result<Self> {
2802        // Reach the worker before reading the projection. A stored session can
2803        // be tens of megabytes, and the reconnect loop would otherwise pay that
2804        // whole synchronous read on every attempt against a worker that is down.
2805        let mut client = RelayClient::connect(&target.spec, &target.session_id).await?;
2806        let operational = client.status().await?;
2807        let materialized = load_projection(&target.session_id).await?;
2808        let mut connection = Self {
2809            client,
2810            materialized,
2811            operational,
2812            latest_credential_sync_signal: None,
2813            project_memory: target.project_memory.clone(),
2814            subagent_requests: Vec::new(),
2815            subagent_results: Vec::new(),
2816        };
2817        connection.sync_in_place().await?;
2818        Ok(connection)
2819    }
2820
2821    pub async fn connect_command(spec: &CommandSpec, session_id: &str) -> Result<Self> {
2822        Self::connect(&RelaySessionTarget {
2823            session_id: session_id.to_owned(),
2824            spec: spec.clone(),
2825            worker_recovery: None,
2826            project_memory: None,
2827        })
2828        .await
2829    }
2830
2831    /// Protocol negotiated with the worker behind this connection. Lifecycle
2832    /// operations use it to avoid sending a newly introduced command to an
2833    /// older worker that cannot decode it.
2834    pub fn protocol_version(&self) -> u32 {
2835        self.client.protocol_version()
2836    }
2837
2838    async fn detach(self) -> Result<()> {
2839        self.client.detach().await
2840    }
2841
2842    pub async fn sync(&mut self) -> Result<ManagedSessionSnapshot> {
2843        self.sync_in_place().await?;
2844        Ok(self.snapshot())
2845    }
2846
2847    async fn sync_in_place(&mut self) -> Result<bool> {
2848        let original_ordinal = self.materialized.applied_event_ordinal;
2849        let original_digest = self.materialized.applied_event_digest.clone();
2850        let original_operational = self.operational.clone();
2851        let mut repaired = false;
2852        let mut repaired_frontiers = std::collections::HashSet::new();
2853        loop {
2854            let after_ordinal = self.materialized.applied_event_ordinal;
2855            match self.catch_up_fixed_frontier().await {
2856                Ok(()) => break,
2857                Err(error) if error.downcast_ref::<ProjectionAdvancedError>().is_some() => {
2858                    let durable = load_projection(&self.materialized.session_id).await?;
2859                    if durable.applied_event_ordinal <= after_ordinal {
2860                        return Err(error);
2861                    }
2862                    self.materialized = durable;
2863                    continue;
2864                }
2865                Err(error) if relay_desynchronized(&error) => {
2866                    self.repair_projection()
2867                        .await
2868                        .with_context(|| {
2869                            format!(
2870                                "controller projection for {} cannot catch up from ordinal {after_ordinal}: {error:#}",
2871                                self.materialized.session_id
2872                            )
2873                        })?;
2874                    repaired = true;
2875                    // Repair rebuilds from the same durable checkpoint every
2876                    // time. If catching up from that frontier still desyncs — as
2877                    // it does when relay history is unreadable past the
2878                    // checkpoint — repairing again lands on the same frontier and
2879                    // would loop forever. Fail loudly on the second visit instead
2880                    // of hanging; recovery got everything the checkpoint covers.
2881                    let frontier = self.materialized.applied_event_ordinal;
2882                    if !repaired_frontiers.insert(frontier) {
2883                        bail!(
2884                            "controller projection for {} cannot catch up: relay history is \
2885                             unreadable and rebuilding from checkpoint frontier {frontier} does \
2886                             not get past it",
2887                            self.materialized.session_id
2888                        );
2889                    }
2890                    continue;
2891                }
2892                Err(error) => return Err(error),
2893            }
2894        }
2895        let previous_requests = self.subagent_requests.clone();
2896        let previous_results = self.subagent_results.clone();
2897        (self.subagent_requests, self.subagent_results) = self.client.subagent_requests().await?;
2898        let changed = repaired
2899            || self.materialized.applied_event_ordinal != original_ordinal
2900            || self.materialized.applied_event_digest != original_digest
2901            || self.operational != original_operational
2902            || self.subagent_requests != previous_requests
2903            || self.subagent_results != previous_results;
2904        Ok(changed)
2905    }
2906
2907    /// Apply relay pages through the exact frontier captured by the first
2908    /// response, then acknowledge that frontier once. Every projection page is
2909    /// independently durable; delaying the relay's GC watermark avoids one
2910    /// snapshot fsync per transport-sized page without risking redelivery.
2911    async fn catch_up_fixed_frontier(&mut self) -> Result<()> {
2912        let after = RelayCursor {
2913            ordinal: self.materialized.applied_event_ordinal,
2914            digest: self.materialized.applied_event_digest.clone(),
2915        };
2916        let catch_up = self
2917            .client
2918            .begin_catch_up(after.ordinal, &after.digest)
2919            .await?;
2920        let mut cursor = self.apply_event_page(catch_up.first_page).await?;
2921        let mut pages_remaining = catch_up.frontier.ordinal.saturating_sub(cursor.ordinal);
2922        while cursor.ordinal < catch_up.frontier.ordinal {
2923            ensure!(
2924                pages_remaining > 0,
2925                "relay catch-up exceeded its fixed page bound"
2926            );
2927            pages_remaining -= 1;
2928            let page = self
2929                .client
2930                .next_catch_up_page(&cursor, &catch_up.frontier)
2931                .await?;
2932            cursor = self.apply_event_page(page).await?;
2933        }
2934        ensure!(
2935            cursor == catch_up.frontier,
2936            "controller projection did not reach the captured relay frontier"
2937        );
2938        if cursor.ordinal > 0 {
2939            let acknowledged = self
2940                .client
2941                .acknowledge(cursor.ordinal, &cursor.digest)
2942                .await?;
2943            ensure!(
2944                acknowledged == cursor,
2945                "relay acknowledged cursor {}:{} instead of {}:{}",
2946                acknowledged.ordinal,
2947                acknowledged.digest,
2948                cursor.ordinal,
2949                cursor.digest,
2950            );
2951        }
2952        let mut operational = catch_up.state;
2953        operational.acknowledged_through = cursor.ordinal;
2954        operational.acknowledged_digest = cursor.digest;
2955        self.operational = operational;
2956        Ok(())
2957    }
2958
2959    async fn repair_projection(&mut self) -> Result<()> {
2960        let state = crate::database::load_state()?;
2961        let record = state
2962            .sessions
2963            .get(&self.materialized.session_id)
2964            .context("controller session disappeared while repairing its projection")?;
2965        let Some(checkpoint) = record.checkpoint.as_ref() else {
2966            let replacement = MaterializedSession::empty(&self.materialized.session_id);
2967            self.client
2968                .attach(
2969                    replacement.applied_event_ordinal,
2970                    &replacement.applied_event_digest,
2971                )
2972                .await
2973                .context("relay cannot rebuild the projection from its genesis")?;
2974            save_materialized_session(&replacement)?;
2975            self.materialized = replacement;
2976            return Ok(());
2977        };
2978        let checkpoint_path = checkpoint.archive_path.clone();
2979        let archive = tokio::task::spawn_blocking(move || {
2980            verify_archive_streaming(&checkpoint_path).with_context(|| {
2981                format!(
2982                    "verify projection repair checkpoint {}",
2983                    checkpoint_path.display()
2984                )
2985            })
2986        })
2987        .await
2988        .context("projection repair archive verification task failed")??;
2989        ensure!(
2990            archive.archive_sha256 == checkpoint.sha256,
2991            "projection repair checkpoint checksum does not match controller metadata"
2992        );
2993        ensure!(
2994            archive.manifest.session.id == self.materialized.session_id,
2995            "projection repair checkpoint belongs to session {}, not {}",
2996            archive.manifest.session.id,
2997            self.materialized.session_id
2998        );
2999        let canonical = archive.canonical_session;
3000        ensure!(
3001            canonical.event_frontier == checkpoint.event_frontier,
3002            "projection repair checkpoint metadata frontier {} does not match archive frontier {}",
3003            checkpoint.event_frontier,
3004            canonical.event_frontier
3005        );
3006
3007        // Prove that the relay recognizes this exact event-chain cursor before
3008        // replacing any controller state. A matching ordinal alone is not a
3009        // repair proof.
3010        self.client
3011            .attach(canonical.event_frontier, &canonical.event_frontier_digest)
3012            .await
3013            .context("relay rejected the verified checkpoint repair cursor")?;
3014        let replacement =
3015            materialized_session_from_canonical(&self.materialized.session_id, &canonical)?;
3016        save_materialized_session(&replacement)?;
3017        self.materialized = replacement;
3018        Ok(())
3019    }
3020
3021    pub fn snapshot(&self) -> ManagedSessionSnapshot {
3022        ManagedSessionSnapshot {
3023            window: mj_core::state::ProjectionWindow::of(&self.materialized),
3024            materialized: self.materialized.clone(),
3025            operational: self.operational.clone(),
3026            latest_credential_sync_signal: self.latest_credential_sync_signal.clone(),
3027            worker_build: self.client.worker_build().map(str::to_owned),
3028            subagent_requests: self.subagent_requests.clone(),
3029            subagent_results: self.subagent_results.clone(),
3030        }
3031    }
3032
3033    pub async fn complete_subagent_request(
3034        &mut self,
3035        result: mj_core::subagent::SubagentToolResult,
3036    ) -> Result<()> {
3037        self.client.complete_subagent_request(result).await?;
3038        (self.subagent_requests, self.subagent_results) = self.client.subagent_requests().await?;
3039        Ok(())
3040    }
3041
3042    /// Hands one command to the relay and returns the ordinal it accepted it
3043    /// at, without catching the local projection up to it.
3044    ///
3045    /// Callers that need the projection current call [`Self::sync`] after.
3046    /// Keeping the two apart matters on the prompt path: the catch-up is the
3047    /// expensive half, and a caller waiting to hear that the relay took the
3048    /// command should not wait for it. It also stops a failed catch-up from
3049    /// looking like a failed submission to a caller that would retry.
3050    pub async fn submit_accepted(
3051        &mut self,
3052        command_id: String,
3053        command: RelayCommand,
3054    ) -> Result<u64> {
3055        self.client.submit(command_id, command).await
3056    }
3057
3058    pub async fn submit(&mut self, command_id: String, command: RelayCommand) -> Result<u64> {
3059        let ordinal = self.submit_accepted(command_id, command).await?;
3060        self.sync_in_place().await?;
3061        Ok(ordinal)
3062    }
3063
3064    pub async fn respond_elicitation(
3065        &mut self,
3066        elicitation_id: String,
3067        response: ElicitationResponse,
3068    ) -> Result<()> {
3069        self.client
3070            .respond_elicitation(elicitation_id, response)
3071            .await?;
3072        self.sync_in_place().await?;
3073        Ok(())
3074    }
3075
3076    pub async fn stop_background_task(&mut self, background_task_id: String) -> Result<()> {
3077        self.client.stop_background_task(background_task_id).await?;
3078        self.sync_in_place().await?;
3079        Ok(())
3080    }
3081
3082    /// Persist relay-private context for the next real prompt. It never
3083    /// contributes an event to the canonical projection.
3084    pub async fn install_prompt_context(&mut self, text: String) -> Result<()> {
3085        self.client.install_prompt_context(text).await
3086    }
3087
3088    /// Apply one relay transport page in bounded durable chunks. A transport
3089    /// page can contain thousands of events, but SQLite has one global writer;
3090    /// regularly releasing it lets other session actors keep their views
3091    /// current. The relay GC watermark advances only after the complete page.
3092    async fn apply_event_page(&mut self, page: RelayEventPage) -> Result<RelayCursor> {
3093        for event in &page.events {
3094            if let mj_core::relay::RelayObservation::CommandQueued {
3095                command: RelayCommand::Prompt { prompt },
3096                ..
3097            } = &event.observation
3098            {
3099                for reference in mj_core::attachment::references(prompt)? {
3100                    if let Err(error) = self.client.cache_attachment(&reference).await {
3101                        // History remains readable even if a blob was lost. A
3102                        // later submission still verifies every image before
3103                        // admission, and must report missing data to the user.
3104                        tracing::warn!(
3105                            session_id = %self.materialized.session_id,
3106                            attachment = %reference.sha256,
3107                            %error,
3108                            "could not cache image attachment during replay"
3109                        );
3110                    }
3111                }
3112            }
3113        }
3114
3115        let RelayEventPage {
3116            events,
3117            through_ordinal,
3118            through_digest,
3119        } = page;
3120        let event_count = events.len();
3121        let transaction_count = event_count.div_ceil(PROJECTION_TRANSACTION_EVENT_BUDGET);
3122        let started = Instant::now();
3123        for events in events.chunks(PROJECTION_TRANSACTION_EVENT_BUDGET) {
3124            let session_id = self.materialized.session_id.clone();
3125            let events = events.to_vec();
3126            let projection = self.materialized.clone();
3127            // Projection is CPU work and its durable page uses synchronous
3128            // SQLite. Keep both off the async actor runtime so independent
3129            // sessions stay responsive during each bounded catch-up chunk.
3130            let (projection, credential_sync_signal) = tokio::task::spawn_blocking(
3131                move || -> Result<(MaterializedSession, Option<CredentialSyncSignal>)> {
3132                    // The in-memory projection advances on a working copy and
3133                    // is published only once its page is durable.
3134                    let mut projection = projection;
3135                    let mut projection_index = ProjectionIndex::new(&projection);
3136                    let mut credential_sync_signal = None;
3137                    let mut prepared = Vec::with_capacity(events.len());
3138                    for event in &events {
3139                        let mutation =
3140                            project_relay_event_indexed(&projection, &projection_index, event)?
3141                                .mutation;
3142                        prepared.push((
3143                            event.ordinal,
3144                            event.previous_digest.clone(),
3145                            event.digest.clone(),
3146                            mutation.clone(),
3147                        ));
3148                        apply_committed_projection_event_indexed(
3149                            &mut projection,
3150                            &mut projection_index,
3151                            event,
3152                            mutation,
3153                        )?;
3154                        if let Some(reason) = relay_event_credential_sync_reason(event) {
3155                            credential_sync_signal = Some(CredentialSyncSignal {
3156                                ordinal: event.ordinal,
3157                                reason,
3158                            });
3159                        }
3160                    }
3161                    drop(projection_index);
3162                    apply_projection_page(&session_id, move |committed| {
3163                        for (ordinal, previous_digest, digest, mutation) in prepared {
3164                            match committed.apply(ordinal, &previous_digest, &digest, &mutation)? {
3165                                ProjectionApplyOutcome::Applied => {}
3166                                ProjectionApplyOutcome::AlreadyApplied => {
3167                                    return Err(ProjectionAdvancedError {
3168                                        event_ordinal: ordinal,
3169                                    }
3170                                    .into());
3171                                }
3172                            }
3173                        }
3174                        Ok((projection, credential_sync_signal))
3175                    })
3176                },
3177            )
3178            .await
3179            .context("relay projection page task failed")??;
3180            self.materialized = projection;
3181            if let Some(signal) = credential_sync_signal {
3182                self.latest_credential_sync_signal = Some(signal);
3183            }
3184        }
3185        if transaction_count > 1 {
3186            tracing::debug!(
3187                session_id = self.materialized.session_id,
3188                event_count,
3189                transaction_count,
3190                elapsed_ms = started.elapsed().as_millis(),
3191                "applied a large relay page in bounded projection transactions"
3192            );
3193        }
3194        let delivered_through = self.materialized.applied_event_ordinal;
3195        ensure!(
3196            delivered_through == through_ordinal,
3197            "relay page claimed frontier {} but delivered through {delivered_through}",
3198            through_ordinal
3199        );
3200        ensure!(
3201            self.materialized.applied_event_digest == through_digest,
3202            "relay page digest does not match its claimed frontier"
3203        );
3204        Ok(RelayCursor {
3205            ordinal: delivered_through,
3206            digest: self.materialized.applied_event_digest.clone(),
3207        })
3208    }
3209
3210    /// Reconcile this worker's project-memory replica at an explicit durable
3211    /// boundary. Normal relay attachment and polling must never perform this
3212    /// filesystem work: a degraded target could otherwise turn reconnects
3213    /// into an unbounded queue of timed-out snapshot writes.
3214    pub async fn sync_project_memory(&mut self) -> Result<()> {
3215        let Some(target) = self.project_memory.clone() else {
3216            return Ok(());
3217        };
3218        if !self.client.supports_project_memory_sync() {
3219            tracing::warn!(
3220                session_id = self.materialized.session_id,
3221                "worker protocol predates project-memory synchronization; preserving memory through checkpoints only"
3222            );
3223            self.project_memory = None;
3224            return Ok(());
3225        }
3226        let (baseline, replica) = match self.client.project_memory_snapshot().await {
3227            Ok(snapshot) => snapshot,
3228            Err(error)
3229                if error
3230                    .downcast_ref::<RelayRejected>()
3231                    .is_some_and(|rejected| {
3232                        rejected.0.code == mj_core::relay::RelayErrorCode::InvalidState
3233                    }) =>
3234            {
3235                tracing::warn!(
3236                    session_id = self.materialized.session_id,
3237                    "worker has no project-memory endpoint; preserving memory through checkpoints only"
3238                );
3239                self.project_memory = None;
3240                return Ok(());
3241            }
3242            Err(error) => return Err(error),
3243        };
3244        let canonical_root = target.canonical_root;
3245        let session_id = self.materialized.session_id.clone();
3246        let (reconciliation, worker_install_needed) = tokio::task::spawn_blocking(move || {
3247            let reconciliation = mj_core::project_memory::reconcile_into_canonical(
3248                &canonical_root,
3249                &baseline,
3250                &replica,
3251                &session_id,
3252            )?;
3253            let worker_install_needed =
3254                reconciliation.merged != baseline || reconciliation.merged != replica;
3255            Ok::<_, anyhow::Error>((reconciliation, worker_install_needed))
3256        })
3257        .await
3258        .context("project memory reconciliation task failed")??;
3259        for conflict in &reconciliation.conflicts {
3260            tracing::warn!(session_id = self.materialized.session_id, %conflict, "project memory conflict preserved");
3261        }
3262        if worker_install_needed {
3263            self.client
3264                .install_project_memory_snapshot(reconciliation.merged)
3265                .await?;
3266        }
3267        Ok(())
3268    }
3269}
3270
3271fn relay_desynchronized(error: &anyhow::Error) -> bool {
3272    error.chain().any(|cause| {
3273        cause
3274            .downcast_ref::<RelayRejected>()
3275            .is_some_and(RelayRejected::is_desynchronized)
3276    })
3277}
3278
3279fn projection_integrity_failure(error: &anyhow::Error) -> bool {
3280    error
3281        .chain()
3282        .any(|cause| cause.downcast_ref::<ProjectionIntegrityError>().is_some())
3283}
3284
3285/// A stopped actor and the manager that resolves its live replacement.
3286///
3287/// This fixture and its constructor are compiled unconditionally and hidden
3288/// from the documentation because the chat crate's tests need them, and a
3289/// `#[cfg(test)]` item is invisible to another crate.
3290#[cfg(test)]
3291struct ReplacementSessionTestFixture {
3292    stopped: ManagedSessionHandle,
3293    control: SessionManagerControl,
3294    submitted: mpsc::UnboundedReceiver<RelayCommand>,
3295}
3296
3297/// A stopped actor and a manager that resolves its live replacement. Chat
3298/// tests use this hand-written actor instead of mocking the session manager
3299/// protocol.
3300#[cfg(test)]
3301fn replacement_session_test_fixture(
3302    session_id: &str,
3303    accepted_ordinal: u64,
3304) -> ReplacementSessionTestFixture {
3305    let (stopped_commands, stopped_commands_rx) = mpsc::channel(1);
3306    drop(stopped_commands_rx);
3307    let (stopped_releases, stopped_releases_rx) = mpsc::unbounded_channel();
3308    drop(stopped_releases_rx);
3309    let (stopped_view_tx, stopped_view) = watch::channel(ManagedSessionView::default());
3310    drop(stopped_view_tx);
3311    let stopped = ManagedSessionHandle {
3312        session_id: session_id.to_owned(),
3313        commands: stopped_commands,
3314        releases: stopped_releases,
3315        view: stopped_view,
3316    };
3317
3318    let (commands, mut commands_rx) = mpsc::channel(4);
3319    let (releases, _releases_rx) = mpsc::unbounded_channel();
3320    let (view_tx, view) = watch::channel(ManagedSessionView::default());
3321    let replacement = ManagedSessionHandle {
3322        session_id: session_id.to_owned(),
3323        commands,
3324        releases,
3325        view,
3326    };
3327    let actor_session_id = session_id.to_owned();
3328    let (submitted_tx, submitted) = mpsc::unbounded_channel();
3329    tokio::spawn(async move {
3330        let _view_tx = view_tx;
3331        while let Some(command) = commands_rx.recv().await {
3332            match command {
3333                ActorCommand::Submit { command, reply, .. } => {
3334                    // Tests can drop the optional observer when they only
3335                    // care about acceptance/reconnection.
3336                    let _ = submitted_tx.send(command);
3337                    let _ = reply.send(Ok(accepted_ordinal));
3338                }
3339                ActorCommand::Sync { reply } => {
3340                    let _ = reply.send(Ok(()));
3341                }
3342                command => command.reject(&actor_session_id, "unsupported test operation"),
3343            }
3344        }
3345    });
3346
3347    let (manager_commands, mut manager_commands_rx) = mpsc::channel(4);
3348    let manager_replacement = replacement.clone();
3349    tokio::spawn(async move {
3350        while let Some(ManagerCommand::Session {
3351            session_id: requested,
3352            reply,
3353        }) = manager_commands_rx.recv().await
3354        {
3355            let resolved =
3356                (requested == manager_replacement.session_id).then(|| manager_replacement.clone());
3357            let _ = reply.send(resolved);
3358        }
3359    });
3360    ReplacementSessionTestFixture {
3361        stopped,
3362        submitted,
3363        control: SessionManagerControl {
3364            commands: manager_commands,
3365        },
3366    }
3367}
3368
3369#[cfg(test)]
3370mod tests {
3371    use super::*;
3372
3373    fn recovery_source_target() -> mj_core::state::TargetLocator {
3374        mj_core::state::TargetLocator::LocalBare {
3375            worker_root: PathBuf::from("/test-worker").join(LEASED_RELAY_SESSION),
3376        }
3377    }
3378
3379    #[tokio::test]
3380    async fn client_adapter_preserves_actor_replacement_and_submit_completion() {
3381        let mut fixture = replacement_session_test_fixture("client-session", 73);
3382        let stopped = fixture.stopped.client();
3383        assert!(stopped.is_stopped());
3384
3385        let control = fixture.control.client();
3386        let replacement = control
3387            .wait_for_session("client-session", Duration::from_secs(1))
3388            .await
3389            .unwrap();
3390        assert!(!replacement.is_stopped());
3391        let pending = replacement
3392            .enqueue_submit("client-command".into(), RelayCommand::Cancel)
3393            .await
3394            .unwrap();
3395        assert!(matches!(
3396            fixture.submitted.recv().await,
3397            Some(RelayCommand::Cancel)
3398        ));
3399        assert_eq!(pending.wait().await.unwrap(), 73);
3400    }
3401
3402    #[tokio::test]
3403    async fn session_adoption_deadline_also_bounds_an_unanswered_manager_request() {
3404        let (commands, mut requests) = mpsc::channel(1);
3405        let control = SessionManagerControl { commands };
3406        let request = tokio::spawn(async move {
3407            control
3408                .wait_for_session("muse", Duration::from_millis(20))
3409                .await
3410        });
3411        let ManagerCommand::Session { mut reply, .. } = requests.recv().await.unwrap();
3412        // Retain the reply without answering, like an unresponsive manager.
3413        let error = tokio::time::timeout(Duration::from_secs(1), request)
3414            .await
3415            .expect("the adoption deadline must bound an individual request")
3416            .unwrap()
3417            .unwrap_err();
3418        assert!(error.to_string().contains("did not become available"));
3419        reply.closed().await;
3420    }
3421    #[cfg(unix)]
3422    use agent_client_protocol::schema::v1::{ContentBlock, TextContent};
3423    use sha2::Digest;
3424
3425    fn ordering_request(session_id: &str, command_id: &str) -> RemoteSessionRequest {
3426        let (reply, _response) = oneshot::channel();
3427        RemoteSessionRequest::Submit {
3428            session_id: session_id.into(),
3429            command_id: command_id.into(),
3430            command: RelayCommand::SetConfig {
3431                key: "effort".into(),
3432                value: "high".into(),
3433            },
3434            admission: None,
3435            reply,
3436        }
3437    }
3438
3439    /// `/effort` followed by a prompt has to reach the relay that way round,
3440    /// or the prompt runs under the old setting. A bridge that spawns every
3441    /// request concurrently loses that, so the order is pinned here: the
3442    /// first request is held up, and the second must not overtake it.
3443    #[tokio::test]
3444    async fn one_session_keeps_its_requests_in_the_order_they_were_made() {
3445        let observed = Arc::new(Mutex::new(Vec::new()));
3446        let release = Arc::new(tokio::sync::Notify::new());
3447        let mut order = SessionRequestOrder::new();
3448
3449        for command_id in ["first", "second", "third"] {
3450            let observed = Arc::clone(&observed);
3451            let release = Arc::clone(&release);
3452            order.dispatch(ordering_request("session-a", command_id), move |request| {
3453                let RemoteSessionRequest::Submit { command_id, .. } = request else {
3454                    unreachable!("the fixture only submits")
3455                };
3456                async move {
3457                    // Only the first request waits. If the order were lost,
3458                    // the other two would finish while it is held.
3459                    if command_id == "first" {
3460                        release.notified().await;
3461                    }
3462                    observed.lock().unwrap().push(command_id);
3463                }
3464            });
3465        }
3466
3467        // Nothing may run while the first request is held. Yield generously:
3468        // the point is that the later requests never get to run, not that
3469        // they have not been polled yet.
3470        for _ in 0..64 {
3471            tokio::task::yield_now().await;
3472        }
3473        assert!(
3474            observed.lock().unwrap().is_empty(),
3475            "a later request overtook the one being held: {:?}",
3476            observed.lock().unwrap()
3477        );
3478
3479        release.notify_one();
3480        tokio::time::timeout(std::time::Duration::from_secs(5), async {
3481            while observed.lock().unwrap().len() < 3 {
3482                tokio::task::yield_now().await;
3483            }
3484        })
3485        .await
3486        .expect("every request ran");
3487        assert_eq!(*observed.lock().unwrap(), ["first", "second", "third"]);
3488    }
3489
3490    /// Ordering is per session: one session waiting on a slow relay must not
3491    /// hold up another session's prompt.
3492    #[tokio::test]
3493    async fn different_sessions_still_overlap() {
3494        let finished = Arc::new(Mutex::new(Vec::new()));
3495        let release = Arc::new(tokio::sync::Notify::new());
3496        let mut order = SessionRequestOrder::new();
3497
3498        let held = Arc::clone(&release);
3499        let recorder = Arc::clone(&finished);
3500        order.dispatch(ordering_request("session-a", "slow"), move |_| async move {
3501            held.notified().await;
3502            recorder.lock().unwrap().push("slow");
3503        });
3504        let recorder = Arc::clone(&finished);
3505        order.dispatch(ordering_request("session-b", "fast"), move |_| async move {
3506            recorder.lock().unwrap().push("fast");
3507        });
3508
3509        tokio::time::timeout(std::time::Duration::from_secs(5), async {
3510            while finished.lock().unwrap().is_empty() {
3511                tokio::task::yield_now().await;
3512            }
3513        })
3514        .await
3515        .expect("the other session ran while the first was held");
3516        assert_eq!(*finished.lock().unwrap(), ["fast"]);
3517
3518        release.notify_one();
3519        tokio::time::timeout(std::time::Duration::from_secs(5), async {
3520            while finished.lock().unwrap().len() < 2 {
3521                tokio::task::yield_now().await;
3522            }
3523        })
3524        .await
3525        .expect("the held request ran once released");
3526    }
3527
3528    #[tokio::test]
3529    async fn slow_reviewer_does_not_delay_primary_or_another_role_at_the_bridge() {
3530        let mut order = SessionRequestOrder::new();
3531        let release = Arc::new(tokio::sync::Notify::new());
3532        let (done, mut received) = mpsc::unbounded_channel();
3533        let reviewer = |role: &str| RemoteSessionRequest::Reviewer {
3534            session_id: "one-session".to_owned(),
3535            role: Some(role.to_owned()),
3536            action: ReviewerAction::Pause,
3537            reply: oneshot::channel().0,
3538        };
3539        let held = release.clone();
3540        order.dispatch(
3541            reviewer("slow"),
3542            move |_| async move { held.notified().await },
3543        );
3544        let done_role = done.clone();
3545        order.dispatch(reviewer("other"), move |_| async move {
3546            done_role.send("other").unwrap();
3547        });
3548        order.dispatch(
3549            ordering_request("one-session", "prompt"),
3550            move |_| async move {
3551                done.send("primary").unwrap();
3552            },
3553        );
3554        let first = tokio::time::timeout(Duration::from_secs(2), received.recv())
3555            .await
3556            .unwrap()
3557            .unwrap();
3558        let second = tokio::time::timeout(Duration::from_secs(2), received.recv())
3559            .await
3560            .unwrap()
3561            .unwrap();
3562        assert_ne!(first, second);
3563        release.notify_one();
3564    }
3565
3566    /// A session that has gone quiet must not leave a handle behind for ever:
3567    /// a long-lived daemon serves many sessions.
3568    #[tokio::test]
3569    async fn finished_sessions_are_forgotten() {
3570        let mut order = SessionRequestOrder::new();
3571        for index in 0..8 {
3572            order.dispatch(
3573                ordering_request(&format!("session-{index}"), "only"),
3574                |_| async {},
3575            );
3576            tokio::time::timeout(std::time::Duration::from_secs(5), async {
3577                while order.latest.values().any(|handle| !handle.is_finished()) {
3578                    tokio::task::yield_now().await;
3579                }
3580            })
3581            .await
3582            .expect("the request finished");
3583        }
3584        // The next dispatch prunes what has finished, so the map tracks live
3585        // work rather than every session ever seen.
3586        order.dispatch(ordering_request("session-last", "only"), |_| async {});
3587        assert_eq!(order.latest.len(), 1);
3588    }
3589
3590    /// A reviewer action reaches a remote controller daemon as JSON, so both
3591    /// halves of the exchange have to survive that round trip intact.
3592    #[test]
3593    fn reviewer_actions_and_outcomes_survive_the_daemon_wire() {
3594        let config = ReviewerLaunchConfig {
3595            profile_id: "claude".into(),
3596            harness: mj_core::config::HarnessKind::Claude,
3597            bridge_command: "npx".into(),
3598            bridge_args: vec!["claude-code-acp".into()],
3599            environment: BTreeMap::from([("EXTRA".into(), "1".into())]),
3600            execution_policy: mj_core::config::ExecutionPolicy::Unconstrained,
3601            model: Some("sonnet".into()),
3602            effort: Some("high".into()),
3603            generation: 2,
3604            mcp_servers: Vec::new(),
3605        };
3606        let actions = [
3607            ReviewerAction::Start {
3608                config: Box::new(config),
3609            },
3610            ReviewerAction::Submit {
3611                command_id: "review-1".into(),
3612                command: RelayCommand::Cancel,
3613            },
3614            ReviewerAction::Attach {
3615                after_ordinal: 4,
3616                after_digest: "digest".into(),
3617            },
3618            ReviewerAction::Acknowledge {
3619                through_ordinal: 4,
3620                through_digest: "digest".into(),
3621            },
3622            ReviewerAction::Status,
3623            ReviewerAction::Pause,
3624            ReviewerAction::CaptureDelta {
3625                baselines: BTreeMap::from([(std::path::PathBuf::from("/w/app"), "tree".into())]),
3626            },
3627            ReviewerAction::AdvanceBaseline {
3628                trees: BTreeMap::from([(std::path::PathBuf::from("/w/app"), "tree".into())]),
3629            },
3630            ReviewerAction::AnalyzeDelta {
3631                repositories: vec![mj_core::relay::AnalyzeDeltaRepository {
3632                    root: std::path::PathBuf::from("/w/app"),
3633                    baseline_tree: Some("base".into()),
3634                    current_tree: "target".into(),
3635                }],
3636            },
3637        ];
3638        for action in actions {
3639            let encoded = serde_json::to_string(&action).unwrap();
3640            let decoded: ReviewerAction = serde_json::from_str(&encoded).unwrap();
3641            assert_eq!(decoded, action);
3642        }
3643
3644        let outcome = ReviewerOutcome::Accepted { ordinal: 9 };
3645        let encoded = serde_json::to_string(&outcome).unwrap();
3646        let decoded: ReviewerOutcome = serde_json::from_str(&encoded).unwrap();
3647        assert!(matches!(decoded, ReviewerOutcome::Accepted { ordinal: 9 }));
3648
3649        let paused = serde_json::to_string(&ReviewerOutcome::Paused).unwrap();
3650        assert!(matches!(
3651            serde_json::from_str::<ReviewerOutcome>(&paused).unwrap(),
3652            ReviewerOutcome::Paused
3653        ));
3654
3655        let delta = ReviewerOutcome::Delta {
3656            repositories: vec![mj_core::relay::RepoDelta {
3657                root: std::path::PathBuf::from("/w/app"),
3658                baseline_tree: None,
3659                current_tree: "target".into(),
3660                patch: "diff --git a/a b/a\n".into(),
3661                diffstat: "1 file changed".into(),
3662                changed_lines: 1,
3663            }],
3664        };
3665        let encoded = serde_json::to_string(&delta).unwrap();
3666        let ReviewerOutcome::Delta { repositories } =
3667            serde_json::from_str::<ReviewerOutcome>(&encoded).unwrap()
3668        else {
3669            panic!("a captured delta must survive the daemon wire");
3670        };
3671        assert_eq!(repositories.len(), 1);
3672        assert_eq!(repositories[0].current_tree, "target");
3673    }
3674
3675    /// Every reviewer action names itself for the actor's logs and for the
3676    /// rejection path, so a stalled review can be traced to the step it stalled
3677    /// on.
3678    #[test]
3679    fn every_reviewer_action_names_its_operation() {
3680        let names = [
3681            ReviewerAction::Submit {
3682                command_id: String::new(),
3683                command: RelayCommand::Cancel,
3684            }
3685            .operation_name(),
3686            ReviewerAction::Attach {
3687                after_ordinal: 0,
3688                after_digest: String::new(),
3689            }
3690            .operation_name(),
3691            ReviewerAction::Acknowledge {
3692                through_ordinal: 0,
3693                through_digest: String::new(),
3694            }
3695            .operation_name(),
3696            ReviewerAction::Status.operation_name(),
3697            ReviewerAction::Pause.operation_name(),
3698        ];
3699        assert_eq!(
3700            names,
3701            [
3702                "reviewer_submit",
3703                "reviewer_attach",
3704                "reviewer_acknowledge",
3705                "reviewer_status",
3706                "reviewer_pause",
3707            ]
3708        );
3709        assert!(names.iter().all(|name| name.starts_with("reviewer_")));
3710    }
3711
3712    #[test]
3713    fn reconnect_delay_backs_off_and_stops_at_the_ceiling() {
3714        assert_eq!(reconnect_delay(1), RECONNECT_INTERVAL);
3715        assert_eq!(reconnect_delay(2), Duration::from_secs(2));
3716        assert_eq!(reconnect_delay(4), Duration::from_secs(8));
3717        assert_eq!(reconnect_delay(6), RECONNECT_BACKOFF_CEILING);
3718        assert_eq!(reconnect_delay(u32::MAX), RECONNECT_BACKOFF_CEILING);
3719    }
3720
3721    #[test]
3722    fn only_dead_worker_connection_failures_request_a_restart() {
3723        // The wording is deliberately unlike anything a matcher could have
3724        // been written against: the marker, not the message, decides.
3725        let reworded = anyhow::Error::new(RelayTransportDead::new(
3726            "the session proxy vanished mid-conversation",
3727        ))
3728        .context("connect to the session worker for checkpoint");
3729        assert!(worker_connect_needs_restart(&reworded), "{reworded:#}");
3730        assert!(!worker_connect_allows_live_restart(&reworded));
3731
3732        // Text alone proves nothing now, not even the exact text the producing
3733        // sites still use: an unmarked failure must never restart a worker.
3734        for detail in [
3735            "relay proxy disconnected during hello",
3736            "Connection refused (os error 111)",
3737            "relay negotiated unsupported protocol 9",
3738            "controller projection is corrupt",
3739        ] {
3740            assert!(!worker_connect_needs_restart(&anyhow::anyhow!(detail)));
3741        }
3742    }
3743
3744    /// The producing side of the same contract: a proxy that dies without
3745    /// serving the handshake must ask for a worker restart, whatever its
3746    /// failure happens to read like.
3747    #[cfg(unix)]
3748    #[tokio::test]
3749    async fn a_proxy_that_dies_before_hello_requests_a_worker_restart() {
3750        let mut dead = target("sh");
3751        dead.spec = CommandSpec::new("sh", ["-c", "exit 1"]).purpose("dead relay proxy fixture");
3752
3753        let error = StandaloneSession::connect(&dead)
3754            .await
3755            .err()
3756            .expect("a proxy that exits cannot serve a session");
3757
3758        assert!(worker_connect_needs_restart(&error), "{error:#}");
3759        assert!(worker_connect_allows_live_restart(&error));
3760    }
3761
3762    /// A lease answer crosses a channel. Formatting the failure into a string
3763    /// there would strip the cause and silently cost the checkpoint path its
3764    /// restart decision, so prove the typed cause survives the handoff.
3765    #[cfg(unix)]
3766    #[tokio::test]
3767    async fn a_failed_lease_keeps_the_cause_that_decides_a_worker_restart() {
3768        let (commands_tx, commands_rx) = mpsc::channel(4);
3769        let (_releases_tx, releases_rx) = mpsc::unbounded_channel();
3770        let (_retirement_tx, retirement_rx) = watch::channel(false);
3771        let (view_tx, _view_rx) = watch::channel(ManagedSessionView::default());
3772        let (updates_tx, _updates_rx) = coalesced_update_channel();
3773        let mut dead = target("sh");
3774        dead.spec = CommandSpec::new("sh", ["-c", "exit 1"]).purpose("dead relay proxy fixture");
3775        tokio::spawn(run_session_actor(
3776            dead,
3777            commands_rx,
3778            releases_rx,
3779            retirement_rx,
3780            view_tx,
3781            updates_tx,
3782        ));
3783
3784        let (reply, response) = oneshot::channel();
3785        commands_tx
3786            .send(ActorCommand::Lease { reply })
3787            .await
3788            .unwrap();
3789        let error = response
3790            .await
3791            .expect("actor answered the lease request")
3792            .err()
3793            .expect("a dead proxy cannot be leased");
3794
3795        assert!(worker_connect_needs_restart(&error), "{error:#}");
3796    }
3797
3798    #[tokio::test]
3799    async fn recovery_restarts_a_live_worker_only_after_a_failed_handshake() {
3800        let directory = tempfile::tempdir().unwrap();
3801        let restarted = directory.path().join("restarted");
3802        let recovery = |liveness: &str| WorkerRecoveryPlan {
3803            source_target: recovery_source_target(),
3804            target: None,
3805            workspace: Some(WorkerWorkspace {
3806                target: mj_core::state::ManagedWorktreeTarget::Local,
3807                directory: directory.path().to_path_buf(),
3808            }),
3809            liveness_probe: CommandSpec::new("printf", [format!("{liveness}\n")])
3810                .purpose("probe test worker liveness"),
3811            binary_refresh: None,
3812            launch_refresh: None,
3813            restart: CommandPlan {
3814                description: "restart test worker".into(),
3815                commands: vec![
3816                    CommandSpec::new("touch", [restarted.to_string_lossy().into_owned()])
3817                        .purpose("restart test worker"),
3818                ],
3819            },
3820        };
3821
3822        assert_eq!(
3823            recover_worker(recovery("alive"), false).await.unwrap(),
3824            WorkerRecoveryOutcome::Alive
3825        );
3826        assert!(!restarted.exists(), "a live worker must not be restarted");
3827
3828        assert_eq!(
3829            recover_worker(recovery("starting"), true).await.unwrap(),
3830            WorkerRecoveryOutcome::Starting
3831        );
3832        assert!(
3833            !restarted.exists(),
3834            "a worker recovering its journal must not be restarted"
3835        );
3836
3837        assert_eq!(
3838            recover_worker(recovery("alive"), true).await.unwrap(),
3839            WorkerRecoveryOutcome::RestartedUnresponsive
3840        );
3841        assert!(
3842            restarted.exists(),
3843            "a worker that cannot serve a fresh handshake is restarted"
3844        );
3845        std::fs::remove_file(&restarted).unwrap();
3846
3847        assert_eq!(
3848            recover_worker(recovery("dead"), false).await.unwrap(),
3849            WorkerRecoveryOutcome::RestartedDead
3850        );
3851        assert!(restarted.exists(), "a confirmed dead worker is restarted");
3852    }
3853
3854    #[tokio::test]
3855    async fn recovery_reports_a_missing_bare_workspace_without_restarting() {
3856        let directory = tempfile::tempdir().unwrap();
3857        let missing = directory.path().join("removed-worktree");
3858        let restarted = directory.path().join("worker-restarted");
3859        let plan = WorkerRecoveryPlan {
3860            source_target: recovery_source_target(),
3861            target: None,
3862            workspace: Some(WorkerWorkspace {
3863                target: mj_core::state::ManagedWorktreeTarget::Local,
3864                directory: missing.clone(),
3865            }),
3866            liveness_probe: CommandSpec::new("printf", ["dead\n"])
3867                .purpose("probe test worker liveness"),
3868            binary_refresh: None,
3869            launch_refresh: None,
3870            restart: CommandPlan {
3871                description: "must not restart missing workspace worker".into(),
3872                commands: vec![
3873                    CommandSpec::new("touch", [restarted.to_string_lossy().into_owned()])
3874                        .purpose("restart test worker"),
3875                ],
3876            },
3877        };
3878
3879        assert_eq!(
3880            recover_worker(plan, false).await.unwrap(),
3881            WorkerRecoveryOutcome::WorkspaceMissing(missing.clone())
3882        );
3883        assert!(
3884            !restarted.exists(),
3885            "a missing workspace must not be restarted"
3886        );
3887    }
3888
3889    #[tokio::test]
3890    async fn recovery_replaces_only_a_stale_worker_binary_before_restart() {
3891        let directory = tempfile::tempdir().unwrap();
3892        let source = directory.path().join("current-worker");
3893        let refreshed = directory.path().join("worker-refreshed");
3894        let restarted = directory.path().join("worker-restarted");
3895        std::fs::write(&source, b"current worker binary").unwrap();
3896        let current_digest = format!("{:x}", sha2::Sha256::digest(b"current worker binary"));
3897        let recovery = |installed_digest: &str, require_refresh: bool| {
3898            let mut restart = if require_refresh {
3899                CommandSpec::new(
3900                    "sh",
3901                    [
3902                        "-c",
3903                        "test -f \"$MJ_TEST_REFRESHED\" && touch -- \"$MJ_TEST_RESTARTED\"",
3904                    ],
3905                )
3906            } else {
3907                CommandSpec::new("touch", [restarted.to_string_lossy().into_owned()])
3908            }
3909            .purpose("restart test worker");
3910            restart.env.insert(
3911                "MJ_TEST_REFRESHED".into(),
3912                refreshed.to_string_lossy().into_owned(),
3913            );
3914            restart.env.insert(
3915                "MJ_TEST_RESTARTED".into(),
3916                restarted.to_string_lossy().into_owned(),
3917            );
3918            WorkerRecoveryPlan {
3919                source_target: recovery_source_target(),
3920                target: None,
3921                workspace: None,
3922                liveness_probe: CommandSpec::new("printf", ["dead\n"])
3923                    .purpose("probe test worker liveness"),
3924                binary_refresh: Some(WorkerBinaryRefresh::Prepared(WorkerBinaryRefreshPlan {
3925                    source: source.clone(),
3926                    installed_digest: CommandSpec::new(
3927                        "printf",
3928                        [format!("{installed_digest}  /worker/hel\n")],
3929                    )
3930                    .purpose("identify test worker binary"),
3931                    replace: CommandPlan {
3932                        description: "refresh test worker".into(),
3933                        commands: vec![
3934                            CommandSpec::new("touch", [refreshed.to_string_lossy().into_owned()])
3935                                .purpose("refresh test worker"),
3936                        ],
3937                    },
3938                })),
3939                launch_refresh: None,
3940                restart: CommandPlan {
3941                    description: "restart test worker".into(),
3942                    commands: vec![restart],
3943                },
3944            }
3945        };
3946
3947        assert_eq!(
3948            recover_worker(recovery(&current_digest, false), false)
3949                .await
3950                .unwrap(),
3951            WorkerRecoveryOutcome::RestartedDead
3952        );
3953        assert!(!refreshed.exists(), "a current binary must not be copied");
3954        assert!(restarted.exists());
3955
3956        std::fs::remove_file(&restarted).unwrap();
3957        assert_eq!(
3958            recover_worker(recovery(&"0".repeat(64), true), false)
3959                .await
3960                .unwrap(),
3961            WorkerRecoveryOutcome::RestartedDead
3962        );
3963        assert!(refreshed.exists(), "a stale binary must be refreshed");
3964        assert!(restarted.exists(), "refresh must finish before restart");
3965    }
3966
3967    #[tokio::test]
3968    async fn recovery_refreshes_a_stale_launch_config_before_restart() {
3969        let directory = tempfile::tempdir().unwrap();
3970        let refreshed = directory.path().join("launch-refreshed");
3971        let restarted = directory.path().join("worker-restarted");
3972        let mut restart = CommandSpec::new(
3973            "sh",
3974            [
3975                "-c",
3976                "test -f \"$MJ_TEST_REFRESHED\" && touch -- \"$MJ_TEST_RESTARTED\"",
3977            ],
3978        )
3979        .purpose("restart test worker");
3980        restart.env.insert(
3981            "MJ_TEST_REFRESHED".into(),
3982            refreshed.to_string_lossy().into_owned(),
3983        );
3984        restart.env.insert(
3985            "MJ_TEST_RESTARTED".into(),
3986            restarted.to_string_lossy().into_owned(),
3987        );
3988        let outcome = recover_worker(
3989            WorkerRecoveryPlan {
3990                source_target: recovery_source_target(),
3991                target: None,
3992                workspace: None,
3993                liveness_probe: CommandSpec::new("printf", ["dead\n"])
3994                    .purpose("probe test worker liveness"),
3995                binary_refresh: None,
3996                launch_refresh: Some(WorkerLaunchRefreshPlan {
3997                    expected_sha256: "a".repeat(64),
3998                    installed_digest: CommandSpec::new(
3999                        "printf",
4000                        [format!("{}  /worker/launch.json\n", "b".repeat(64))],
4001                    )
4002                    .purpose("identify test launch config"),
4003                    replace: CommandPlan {
4004                        description: "refresh test launch config".into(),
4005                        commands: vec![
4006                            CommandSpec::new("touch", [refreshed.to_string_lossy().into_owned()])
4007                                .purpose("refresh test launch config"),
4008                        ],
4009                    },
4010                }),
4011                restart: CommandPlan {
4012                    description: "restart test worker".into(),
4013                    commands: vec![restart],
4014                },
4015            },
4016            false,
4017        )
4018        .await
4019        .unwrap();
4020
4021        assert_eq!(outcome, WorkerRecoveryOutcome::RestartedDead);
4022        assert!(refreshed.exists());
4023        assert!(
4024            restarted.exists(),
4025            "config refresh must finish before restart"
4026        );
4027    }
4028
4029    #[tokio::test]
4030    async fn recovery_starts_a_stopped_target_before_probing_its_worker() {
4031        let directory = tempfile::tempdir().unwrap();
4032        let target_started = directory.path().join("target-started");
4033        let worker_restarted = directory.path().join("worker-restarted");
4034        let inspection = |status: &str| {
4035            serde_json::to_string(&serde_json::json!([{
4036                "Config": { "Labels": {
4037                    (crate::targets::MANAGED_LABEL): "true",
4038                    (crate::targets::SESSION_LABEL): "session-1",
4039                }},
4040                "State": { "Status": status },
4041            }]))
4042            .unwrap()
4043        };
4044        let mut inspect = CommandSpec::new(
4045            "sh",
4046            [
4047                "-c",
4048                "if [ -f \"$MJ_TEST_TARGET_STARTED\" ]; then printf '%s\\n' \"$MJ_TEST_RUNNING\"; else printf '%s\\n' \"$MJ_TEST_EXITED\"; fi",
4049            ],
4050        )
4051        .purpose("inspect test target");
4052        inspect.env.insert(
4053            "MJ_TEST_TARGET_STARTED".into(),
4054            target_started.to_string_lossy().into_owned(),
4055        );
4056        inspect
4057            .env
4058            .insert("MJ_TEST_RUNNING".into(), inspection("running"));
4059        inspect
4060            .env
4061            .insert("MJ_TEST_EXITED".into(), inspection("exited"));
4062        let mut start = CommandSpec::new("sh", ["-c", "touch -- \"$MJ_TEST_TARGET_STARTED\""])
4063            .purpose("start test target");
4064        start.env.insert(
4065            "MJ_TEST_TARGET_STARTED".into(),
4066            target_started.to_string_lossy().into_owned(),
4067        );
4068        let mut liveness = CommandSpec::new(
4069            "sh",
4070            [
4071                "-c",
4072                "test -f \"$MJ_TEST_TARGET_STARTED\" && printf 'dead\\n'",
4073            ],
4074        )
4075        .purpose("probe test worker after target start");
4076        liveness.env.insert(
4077            "MJ_TEST_TARGET_STARTED".into(),
4078            target_started.to_string_lossy().into_owned(),
4079        );
4080
4081        let outcome = recover_worker(
4082            WorkerRecoveryPlan {
4083                source_target: recovery_source_target(),
4084                target: Some(TargetRecoveryPlan {
4085                    exists: CommandSpec::new("true", std::iter::empty::<&str>())
4086                        .purpose("check test target"),
4087                    inspect,
4088                    start,
4089                    session_id: "session-1".into(),
4090                }),
4091                workspace: None,
4092                liveness_probe: liveness,
4093                binary_refresh: None,
4094                launch_refresh: None,
4095                restart: CommandPlan {
4096                    description: "restart test worker".into(),
4097                    commands: vec![
4098                        CommandSpec::new(
4099                            "touch",
4100                            [worker_restarted.to_string_lossy().into_owned()],
4101                        )
4102                        .purpose("restart test worker"),
4103                    ],
4104                },
4105            },
4106            false,
4107        )
4108        .await
4109        .unwrap();
4110
4111        assert_eq!(outcome, WorkerRecoveryOutcome::RestartedDead);
4112        assert!(target_started.exists());
4113        assert!(worker_restarted.exists());
4114    }
4115
4116    #[tokio::test]
4117    async fn recovery_reports_a_missing_target_without_running_worker_commands() {
4118        let unreachable = CommandSpec::new("false", std::iter::empty::<&str>());
4119        let outcome = recover_worker(
4120            WorkerRecoveryPlan {
4121                source_target: recovery_source_target(),
4122                target: Some(TargetRecoveryPlan {
4123                    exists: unreachable,
4124                    inspect: CommandSpec::new("false", std::iter::empty::<&str>()),
4125                    start: CommandSpec::new("false", std::iter::empty::<&str>()),
4126                    session_id: "session-1".into(),
4127                }),
4128                workspace: None,
4129                liveness_probe: CommandSpec::new("false", std::iter::empty::<&str>()),
4130                binary_refresh: None,
4131                launch_refresh: None,
4132                restart: CommandPlan {
4133                    description: "must not restart".into(),
4134                    commands: vec![CommandSpec::new("false", std::iter::empty::<&str>())],
4135                },
4136            },
4137            true,
4138        )
4139        .await
4140        .unwrap();
4141
4142        assert_eq!(outcome, WorkerRecoveryOutcome::TargetMissing);
4143    }
4144
4145    fn target(program: &str) -> RelaySessionTarget {
4146        RelaySessionTarget {
4147            session_id: "session-1".to_owned(),
4148            spec: CommandSpec::new(program, std::iter::empty::<&str>()),
4149            worker_recovery: None,
4150            project_memory: None,
4151        }
4152    }
4153
4154    /// A connected view carrying a conversation, so republishing it exercises
4155    /// the case a whole-transcript comparison would have to walk.
4156    fn view_at_ordinal(ordinal: u64) -> ManagedSessionView {
4157        let digest = "a".repeat(64);
4158        let mut materialized = MaterializedSession::empty("session-1");
4159        materialized.applied_event_ordinal = ordinal;
4160        materialized.applied_event_digest = digest.clone();
4161        materialized.transcript = (1..=200)
4162            .map(|position| {
4163                Arc::new(mj_core::state::TranscriptItem {
4164                    stable_id: format!("system:{position}"),
4165                    position,
4166                    latest_content_event_ordinal: None,
4167                    created_at_ms: 1,
4168                    last_changed_at_ms: 1,
4169                    body: mj_core::state::TranscriptBody::System {
4170                        text: format!("event {position}"),
4171                    },
4172                })
4173            })
4174            .collect();
4175        ManagedSessionView {
4176            snapshot: Some(ManagedSessionSnapshot {
4177                subagent_requests: Vec::new(),
4178                subagent_results: Vec::new(),
4179                window: mj_core::state::ProjectionWindow::of(&materialized),
4180                materialized,
4181                operational: RelayOperationalState {
4182                    goal: Default::default(),
4183                    capacity_retry: None,
4184                    activity_turn_started_at_ms: None,
4185                    store_id: None,
4186                    idle_since_ms: None,
4187                    session_id: "session-1".into(),
4188                    execution: mj_core::relay::RelayExecutionState::Idle,
4189                    latest_ordinal: ordinal,
4190                    latest_digest: digest.clone(),
4191                    acknowledged_through: ordinal,
4192                    acknowledged_digest: digest,
4193                    recovery_floor_ordinal: 0,
4194                    recovery_floor_digest: mj_core::relay::RELAY_EVENT_GENESIS_DIGEST.into(),
4195                    native_session_id: None,
4196                    checkpoint_only: false,
4197                    acp_ready: None,
4198                    agent_capabilities: None,
4199                    agent_info: None,
4200                    steering_supported: None,
4201                    config_options: Vec::new(),
4202                    modes: None,
4203                    available_commands: Vec::new(),
4204                    config: BTreeMap::new(),
4205                    active_prompt: None,
4206                    queued_prompts: Vec::new(),
4207                    active_user_shells: Vec::new(),
4208                    active_agent_terminals: Vec::new(),
4209                    checkpoint_barrier: None,
4210                    checkpoint_ready: None,
4211                    last_acp_activity_at_ms: None,
4212                    current_step_started_at_ms: None,
4213                    foreground_tool_started_at_ms: None,
4214                    harness_turn: None,
4215                    last_harness_turn_started_ordinal: None,
4216                    background_commands: Vec::new(),
4217                    background_work_known: None,
4218                },
4219                latest_credential_sync_signal: None,
4220                worker_build: None,
4221            }),
4222            connected: true,
4223            error: None,
4224        }
4225    }
4226
4227    #[test]
4228    fn republishing_an_unchanged_view_notifies_nobody() {
4229        let (view_tx, mut view_rx) = watch::channel(ManagedSessionView::default());
4230        let (updates_tx, mut updates_rx) = coalesced_update_channel();
4231
4232        publish_view("session-1", view_at_ordinal(7), &view_tx, &updates_tx);
4233        assert!(view_rx.has_changed().expect("watch stays open"));
4234        assert_eq!(
4235            updates_rx.try_recv().expect("the first view is news").view,
4236            view_at_ordinal(7)
4237        );
4238        let _ = view_rx.borrow_and_update();
4239
4240        publish_view("session-1", view_at_ordinal(7), &view_tx, &updates_tx);
4241
4242        assert!(
4243            !view_rx.has_changed().expect("watch stays open"),
4244            "a sync tick that moved nothing must not wake the dashboard"
4245        );
4246        assert!(updates_rx.try_recv().is_err());
4247    }
4248
4249    #[test]
4250    fn publishing_an_advanced_event_frontier_notifies_watchers() {
4251        let (view_tx, mut view_rx) = watch::channel(ManagedSessionView::default());
4252        let (updates_tx, mut updates_rx) = coalesced_update_channel();
4253        publish_view("session-1", view_at_ordinal(7), &view_tx, &updates_tx);
4254        let _ = updates_rx.try_recv();
4255        let _ = view_rx.borrow_and_update();
4256
4257        publish_view("session-1", view_at_ordinal(8), &view_tx, &updates_tx);
4258
4259        assert!(view_rx.has_changed().expect("watch stays open"));
4260        let update = updates_rx.try_recv().expect("the advance is news");
4261        assert_eq!(update.session_id, "session-1");
4262        assert_eq!(
4263            update
4264                .view
4265                .snapshot
4266                .expect("published snapshot")
4267                .materialized
4268                .applied_event_ordinal,
4269            8
4270        );
4271    }
4272
4273    #[test]
4274    fn publishing_relay_state_that_moved_without_the_frontier_notifies_watchers() {
4275        let (view_tx, mut view_rx) = watch::channel(ManagedSessionView::default());
4276        let (updates_tx, mut updates_rx) = coalesced_update_channel();
4277        publish_view("session-1", view_at_ordinal(7), &view_tx, &updates_tx);
4278        let _ = updates_rx.try_recv();
4279        let _ = view_rx.borrow_and_update();
4280
4281        let mut view = view_at_ordinal(7);
4282        view.snapshot
4283            .as_mut()
4284            .expect("published snapshot")
4285            .operational
4286            .execution = mj_core::relay::RelayExecutionState::Running;
4287        publish_view("session-1", view, &view_tx, &updates_tx);
4288
4289        assert!(view_rx.has_changed().expect("watch stays open"));
4290        assert!(updates_rx.try_recv().is_ok());
4291    }
4292
4293    #[test]
4294    fn losing_the_relay_republishes_the_same_snapshot_as_disconnected() {
4295        let (view_tx, mut view_rx) = watch::channel(ManagedSessionView::default());
4296        let (updates_tx, mut updates_rx) = coalesced_update_channel();
4297        publish_view("session-1", view_at_ordinal(7), &view_tx, &updates_tx);
4298        let _ = updates_rx.try_recv();
4299        let _ = view_rx.borrow_and_update();
4300
4301        let mut view = view_at_ordinal(7);
4302        view.connected = false;
4303        view.error = Some(ViewError::Unreachable("relay is unreachable".into()));
4304        publish_view("session-1", view, &view_tx, &updates_tx);
4305
4306        assert!(view_rx.has_changed().expect("watch stays open"));
4307        assert!(updates_rx.try_recv().is_ok());
4308    }
4309
4310    #[test]
4311    fn command_ids_are_namespaced_and_unique() {
4312        let first = new_command_id("prompt").unwrap();
4313        let second = new_command_id("prompt").unwrap();
4314        assert!(first.starts_with("prompt-"));
4315        assert_ne!(first, second);
4316    }
4317
4318    #[test]
4319    fn leased_actor_defers_replacement_and_uses_latest_queued_target() {
4320        let original = target("relay-v1");
4321        let intermediate = target("relay-v2");
4322        let latest = target("relay-v3");
4323        let mut lifecycle = ActorLifecycle::default();
4324        lifecycle.activate_lease(7);
4325
4326        assert_eq!(
4327            reconcile_action(Some(&original), Some(&intermediate)),
4328            ReconcileAction::Retire
4329        );
4330        lifecycle.set_retirement_requested(true);
4331        assert!(!lifecycle.accepts_new_work());
4332        assert!(!lifecycle.should_stop());
4333
4334        assert_eq!(
4335            reconcile_action(Some(&original), Some(&latest)),
4336            ReconcileAction::Retire
4337        );
4338        assert!(lifecycle.return_lease(7));
4339        assert!(lifecycle.should_stop());
4340
4341        assert_eq!(
4342            reconcile_action(None, Some(&latest)),
4343            ReconcileAction::Spawn
4344        );
4345    }
4346
4347    #[test]
4348    fn leased_actor_defers_removal_until_its_connection_returns() {
4349        let original = target("relay-v1");
4350        let mut lifecycle = ActorLifecycle::default();
4351        lifecycle.activate_lease(11);
4352
4353        assert_eq!(
4354            reconcile_action(Some(&original), None),
4355            ReconcileAction::Retire
4356        );
4357        lifecycle.set_retirement_requested(true);
4358        assert!(!lifecycle.should_stop());
4359        assert!(!lifecycle.return_lease(10));
4360        assert!(!lifecycle.should_stop());
4361        assert!(lifecycle.return_lease(11));
4362        assert!(lifecycle.should_stop());
4363        assert_eq!(reconcile_action(None, None), ReconcileAction::Idle);
4364    }
4365
4366    #[test]
4367    fn queued_change_back_to_current_target_cancels_retirement() {
4368        let original = target("relay-v1");
4369        let replacement = target("relay-v2");
4370        let mut lifecycle = ActorLifecycle::default();
4371        lifecycle.activate_lease(3);
4372
4373        assert_eq!(
4374            reconcile_action(Some(&original), Some(&replacement)),
4375            ReconcileAction::Retire
4376        );
4377        lifecycle.set_retirement_requested(true);
4378        assert_eq!(
4379            reconcile_action(Some(&original), Some(&original)),
4380            ReconcileAction::Keep
4381        );
4382        lifecycle.set_retirement_requested(false);
4383
4384        assert!(lifecycle.return_lease(3));
4385        assert!(!lifecycle.should_stop());
4386        assert!(lifecycle.accepts_new_work());
4387    }
4388
4389    #[tokio::test]
4390    async fn stopped_actor_is_replaced_without_late_completion_removing_replacement() {
4391        let desired = target("sh");
4392        let desired_targets = target_map(std::slice::from_ref(&desired));
4393        let mut actors = BTreeMap::new();
4394        let mut tasks = tokio::task::JoinSet::new();
4395        let (commands, commands_rx) = mpsc::channel(1);
4396        drop(commands_rx);
4397        let (releases, _releases_rx) = mpsc::unbounded_channel();
4398        let (retirement, _retirement_rx) = watch::channel(false);
4399        let (_view_tx, view) = watch::channel(ManagedSessionView::default());
4400        let old_abort = tasks.spawn(async { "session-1".to_owned() });
4401        let old_task_id = old_abort.id();
4402        actors.insert(
4403            "session-1".to_owned(),
4404            ActorRegistration {
4405                target: desired.clone(),
4406                commands,
4407                releases,
4408                retirement,
4409                view,
4410                abort: old_abort,
4411            },
4412        );
4413        let (updates, _updates_rx) = coalesced_update_channel();
4414
4415        reconcile_actors(&desired_targets, &mut actors, &mut tasks, &updates);
4416
4417        let replacement_task_id = actors["session-1"].abort.id();
4418        assert_ne!(replacement_task_id, old_task_id);
4419        assert!(!actors["session-1"].commands.is_closed());
4420        assert_eq!(remove_actor_task(&mut actors, old_task_id), None);
4421        assert_eq!(actors["session-1"].abort.id(), replacement_task_id);
4422        tasks.abort_all();
4423    }
4424
4425    const UNREACHABLE_VIEW_TEST_CHILD: &str = "MJ_TEST_UNREACHABLE_RELAY_CHILD";
4426
4427    #[tokio::test(start_paused = true)]
4428    async fn unreachable_relay_publishes_error_view() {
4429        // MJ_DATA_DIR is process-global, so run the database-backed half in
4430        // an exact child test instead of racing unrelated tests in this
4431        // process.
4432        if std::env::var_os(UNREACHABLE_VIEW_TEST_CHILD).is_none() {
4433            let directory = tempfile::tempdir().unwrap();
4434            let test_name = format!(
4435                "{}::unreachable_relay_publishes_error_view",
4436                module_path!()
4437                    .strip_prefix("mj_controller::")
4438                    .unwrap_or(module_path!())
4439            );
4440            let output = std::process::Command::new(std::env::current_exe().unwrap())
4441                .args(["--exact", &test_name, "--nocapture"])
4442                .env(UNREACHABLE_VIEW_TEST_CHILD, "1")
4443                .env("MJ_DATA_DIR", directory.path())
4444                .env("MJ_CONFIG_DIR", directory.path().join("config"))
4445                .output()
4446                .unwrap();
4447            assert!(
4448                output.status.success(),
4449                "isolated unreachable relay test failed\nstdout:\n{}\nstderr:\n{}",
4450                String::from_utf8_lossy(&output.stdout),
4451                String::from_utf8_lossy(&output.stderr)
4452            );
4453            return;
4454        }
4455
4456        // A regression in the publish path deadlocks the actor instead of
4457        // returning an error, so convert a hang into a hard failure.
4458        std::thread::spawn(|| {
4459            std::thread::sleep(Duration::from_secs(60));
4460            eprintln!("unreachable relay error view was never published");
4461            std::process::exit(101);
4462        });
4463
4464        let (_commands_tx, commands_rx) = mpsc::channel(4);
4465        let (_releases_tx, releases_rx) = mpsc::unbounded_channel();
4466        let (_retirement_tx, retirement_rx) = watch::channel(false);
4467        let (view_tx, mut view_rx) = watch::channel(ManagedSessionView::default());
4468        let (updates_tx, mut updates_rx) = coalesced_update_channel();
4469        tokio::spawn(run_session_actor(
4470            target("hel-relay-program-that-does-not-exist"),
4471            commands_rx,
4472            releases_rx,
4473            retirement_rx,
4474            view_tx,
4475            updates_tx,
4476        ));
4477
4478        loop {
4479            view_rx.changed().await.unwrap();
4480            let view = view_rx.borrow_and_update().clone();
4481            if !view.connected {
4482                let error = view
4483                    .error
4484                    .expect("unreachable view carries the connect error");
4485                assert!(
4486                    error.detail().contains("session relay proxy"),
4487                    "unexpected error: {error:?}"
4488                );
4489                break;
4490            }
4491        }
4492        let update = updates_rx
4493            .recv()
4494            .await
4495            .expect("dashboard feed received the error view");
4496        assert_eq!(update.session_id, "session-1");
4497        assert!(!update.view.connected);
4498    }
4499
4500    const UNREADABLE_PROJECTION_TEST_CHILD: &str = "MJ_TEST_UNREADABLE_PROJECTION_CHILD";
4501
4502    #[tokio::test]
4503    async fn connecting_to_an_absent_worker_never_reads_the_projection() {
4504        // MJ_DATA_DIR is process-global, so run the database-backed half in
4505        // an exact child test instead of racing unrelated tests in this
4506        // process.
4507        if std::env::var_os(UNREADABLE_PROJECTION_TEST_CHILD).is_none() {
4508            let directory = tempfile::tempdir().unwrap();
4509            // A directory where the database file belongs makes every
4510            // projection read fail, so a read that happens at all shows up in
4511            // the reported error.
4512            std::fs::create_dir(directory.path().join("mj.sqlite3")).unwrap();
4513            let output = std::process::Command::new(std::env::current_exe().unwrap())
4514                .args([
4515                    "--exact",
4516                    &format!(
4517                        "{}::connecting_to_an_absent_worker_never_reads_the_projection",
4518                        module_path!()
4519                            .strip_prefix("mj_controller::")
4520                            .unwrap_or(module_path!())
4521                    ),
4522                    "--nocapture",
4523                ])
4524                .env(UNREADABLE_PROJECTION_TEST_CHILD, "1")
4525                .env("MJ_DATA_DIR", directory.path())
4526                .env("MJ_CONFIG_DIR", directory.path().join("config"))
4527                .output()
4528                .unwrap();
4529            assert!(
4530                output.status.success(),
4531                "isolated projection ordering test failed\nstdout:\n{}\nstderr:\n{}",
4532                String::from_utf8_lossy(&output.stdout),
4533                String::from_utf8_lossy(&output.stderr)
4534            );
4535            return;
4536        }
4537
4538        assert!(
4539            crate::database::load_materialized_session("session-1").is_err(),
4540            "this store must fail every projection read for the test to mean anything"
4541        );
4542        let connected =
4543            StandaloneSession::connect(&target("hel-relay-program-that-does-not-exist")).await;
4544        let error = match connected {
4545            Ok(_) => panic!("a relay program that does not exist cannot connect"),
4546            Err(error) => error,
4547        };
4548        let detail = format!("{error:#}");
4549        assert!(
4550            detail.contains("session relay proxy"),
4551            "unexpected error: {detail}"
4552        );
4553        assert!(
4554            !detail.contains("Mjolnir database"),
4555            "connect read the projection before it reached the relay: {detail}"
4556        );
4557    }
4558
4559    const LEASED_RELAY_ROOT: &str = "MJ_TEST_LEASED_RELAY_ROOT";
4560    #[cfg(unix)]
4561    const AUTO_RESTART_TEST_CHILD: &str = "MJ_TEST_AUTO_RESTART_CHILD";
4562    #[cfg(unix)]
4563    const AUTO_RESTART_MARKER: &str = "MJ_TEST_AUTO_RESTART_MARKER";
4564    #[cfg(unix)]
4565    const DEFERRED_SUBMIT_TEST_CHILD: &str = "MJ_TEST_DEFERRED_SUBMIT_CHILD";
4566    #[cfg(unix)]
4567    const RETIRED_SUBMIT_TEST_CHILD: &str = "MJ_TEST_RETIRED_SUBMIT_CHILD";
4568    #[cfg(unix)]
4569    const RETURNED_LEASE_VIEW_TEST_CHILD: &str = "MJ_TEST_RETURNED_LEASE_VIEW_CHILD";
4570    #[cfg(unix)]
4571    const EXPLICIT_MEMORY_SYNC_TEST_CHILD: &str = "MJ_TEST_EXPLICIT_MEMORY_SYNC_CHILD";
4572    #[cfg(unix)]
4573    const SUBMIT_WITHOUT_SYNC_TEST_CHILD: &str = "MJ_TEST_SUBMIT_WITHOUT_SYNC_CHILD";
4574    #[cfg(unix)]
4575    const MANAGER_SHUTDOWN_TEST_CHILD: &str = "MJ_TEST_MANAGER_SHUTDOWN_CHILD";
4576    const LEASED_RELAY_SESSION: &str = "018f9dd2-a3b4-7c8d-9000-123456789abc";
4577
4578    /// Relay server half of the leased-submission tests. It does nothing unless
4579    /// a parent test points it at a relay journal root.
4580    #[test]
4581    fn leased_relay_child_serves_stdio() {
4582        let Some(root) = std::env::var_os(LEASED_RELAY_ROOT) else {
4583            return;
4584        };
4585        // With `--nocapture` libtest writes `test <name> ... ` without a
4586        // trailing newline before the body runs. End that line first so it
4587        // cannot glue itself onto the first protocol frame.
4588        println!();
4589        let mut relay = mj_worker::relay::DurableRelay::open(
4590            std::path::Path::new(&root),
4591            LEASED_RELAY_SESSION,
4592            "1.0.0",
4593        )
4594        .expect("open the test relay journal");
4595        if let Some(marker) = std::env::var_os("MJ_TEST_BLOCKED_REVIEWER") {
4596            let mut input = std::io::stdin().lock();
4597            let mut output = std::io::stdout().lock();
4598            while let Some(request) = mj_core::relay::read_relay_frame(&mut input).unwrap() {
4599                let response =
4600                    if let mj_core::relay::RelayRequest::Reviewer { role, .. } = &request.request {
4601                        if role.as_deref() == Some("slow") {
4602                            std::fs::write(&marker, b"started").unwrap();
4603                            std::io::copy(&mut input, &mut std::io::sink()).unwrap();
4604                            std::fs::write(&marker, b"disconnected").unwrap();
4605                            return;
4606                        }
4607                        mj_core::relay::RelayResponseEnvelope {
4608                            request_id: request.request_id,
4609                            protocol_version: request.protocol_version,
4610                            body: mj_core::relay::RelayResponseBody::Ok {
4611                                payload: mj_core::relay::RelayResponsePayload::ReviewerPaused,
4612                            },
4613                        }
4614                    } else {
4615                        relay.handle(request)
4616                    };
4617                mj_core::relay::write_relay_frame(&mut output, &response).unwrap();
4618            }
4619            return;
4620        }
4621        mj_worker::relay::serve_relay_json_lines(
4622            &mut std::io::stdin().lock(),
4623            &mut std::io::stdout().lock(),
4624            &mut relay,
4625        )
4626        .expect("serve relay frames until the controller disconnects");
4627    }
4628
4629    #[cfg(unix)]
4630    fn exact_test_name(test: &str) -> String {
4631        format!(
4632            "{}::{test}",
4633            module_path!()
4634                .strip_prefix("mj_controller::")
4635                .unwrap_or(module_path!())
4636        )
4637    }
4638
4639    /// MJ_DATA_DIR is process-global, so every test that reaches the
4640    /// controller database runs in an exact child with its own data directory.
4641    #[cfg(unix)]
4642    fn run_in_isolated_child(marker: &str, test: &str) {
4643        let directory = tempfile::tempdir().unwrap();
4644        let output = std::process::Command::new(std::env::current_exe().unwrap())
4645            .args(["--exact", &exact_test_name(test), "--nocapture"])
4646            .env(marker, "1")
4647            .env("MJ_DATA_DIR", directory.path())
4648            .env("MJ_CONFIG_DIR", directory.path().join("config"))
4649            .output()
4650            .unwrap();
4651        assert!(
4652            output.status.success(),
4653            "isolated {test} failed\nstdout:\n{}\nstderr:\n{}",
4654            String::from_utf8_lossy(&output.stdout),
4655            String::from_utf8_lossy(&output.stderr)
4656        );
4657    }
4658
4659    #[cfg(unix)]
4660    #[tokio::test]
4661    async fn session_manager_shutdown_joins_a_live_relay_actor() {
4662        if std::env::var_os(MANAGER_SHUTDOWN_TEST_CHILD).is_none() {
4663            run_in_isolated_child(
4664                MANAGER_SHUTDOWN_TEST_CHILD,
4665                "session_manager_shutdown_joins_a_live_relay_actor",
4666            );
4667            return;
4668        }
4669        // Alone in this child process, so it installs the one writer.
4670        let _writer = crate::database::install_isolated_test_writer();
4671        register_leased_relay_session();
4672        let relay_root = tempfile::tempdir().unwrap();
4673        let SessionManagerChannels {
4674            targets,
4675            control,
4676            updates: _updates,
4677            shutdown,
4678        } = spawn_session_manager().expect("spawn the session manager");
4679        targets.send_replace(vec![leased_relay_target(relay_root.path())]);
4680        let session = control
4681            .wait_for_session(LEASED_RELAY_SESSION, Duration::from_secs(2))
4682            .await
4683            .expect("manager registered the relay actor");
4684        session
4685            .sync_now()
4686            .await
4687            .expect("relay actor established a live connection");
4688        assert!(session.view().connected);
4689
4690        tokio::time::timeout(Duration::from_secs(2), shutdown.shutdown())
4691            .await
4692            .expect("manager shutdown stayed within its deadline")
4693            .expect("manager shutdown task completed cleanly");
4694    }
4695
4696    #[cfg(unix)]
4697    #[tokio::test]
4698    async fn a_blocked_reviewer_keeps_primary_responsive_and_disconnects_on_cancellation() {
4699        const CHILD: &str = "MJ_TEST_REVIEWER_CANCELLATION_CHILD";
4700        if std::env::var_os(CHILD).is_none() {
4701            run_in_isolated_child(
4702                CHILD,
4703                "a_blocked_reviewer_keeps_primary_responsive_and_disconnects_on_cancellation",
4704            );
4705            return;
4706        }
4707        let _writer = crate::database::install_isolated_test_writer();
4708        register_leased_relay_session();
4709        let directory = tempfile::tempdir().unwrap();
4710        let marker = directory.path().join("reviewer-status");
4711        let mut target = leased_relay_target(directory.path());
4712        target.spec.env.insert(
4713            "MJ_TEST_BLOCKED_REVIEWER".into(),
4714            marker.to_string_lossy().into_owned(),
4715        );
4716        let manager = spawn_session_manager().unwrap();
4717        manager.targets.send_replace(vec![target]);
4718        let session = manager
4719            .control
4720            .wait_for_session(LEASED_RELAY_SESSION, Duration::from_secs(5))
4721            .await
4722            .unwrap();
4723        session.sync_now().await.unwrap();
4724        let slow = session.clone();
4725        let blocked = tokio::spawn(async move {
4726            slow.reviewer_as(Some("slow".into()), ReviewerAction::Pause)
4727                .await
4728        });
4729        tokio::time::timeout(Duration::from_secs(5), async {
4730            while !marker.exists() {
4731                tokio::time::sleep(Duration::from_millis(10)).await;
4732            }
4733        })
4734        .await
4735        .expect("slow reviewer started");
4736        tokio::time::timeout(Duration::from_secs(2), session.sync_now())
4737            .await
4738            .expect("primary sync must not wait for reviewer")
4739            .unwrap();
4740        tokio::time::timeout(
4741            Duration::from_secs(2),
4742            session.reviewer_as(Some("other".into()), ReviewerAction::Pause),
4743        )
4744        .await
4745        .expect("another role must remain responsive")
4746        .unwrap();
4747        blocked.abort();
4748        assert!(blocked.await.unwrap_err().is_cancelled());
4749        tokio::time::timeout(Duration::from_secs(5), async {
4750            while std::fs::read(&marker).unwrap() != b"disconnected" {
4751                tokio::time::sleep(Duration::from_millis(10)).await;
4752            }
4753        })
4754        .await
4755        .expect("cancelling the caller must disconnect its in-flight reviewer proxy");
4756        tokio::time::timeout(Duration::from_secs(2), manager.shutdown.shutdown())
4757            .await
4758            .unwrap()
4759            .unwrap();
4760    }
4761
4762    #[cfg(unix)]
4763    #[tokio::test]
4764    async fn relay_attach_does_not_probe_or_install_project_memory() {
4765        if std::env::var_os(EXPLICIT_MEMORY_SYNC_TEST_CHILD).is_none() {
4766            run_in_isolated_child(
4767                EXPLICIT_MEMORY_SYNC_TEST_CHILD,
4768                "relay_attach_does_not_probe_or_install_project_memory",
4769            );
4770            return;
4771        }
4772        // Alone in this child process, so it installs the one writer.
4773        let _writer = crate::database::install_isolated_test_writer();
4774        register_leased_relay_session();
4775        let relay_root = tempfile::tempdir().unwrap();
4776        let canonical = tempfile::tempdir().unwrap();
4777        let mut target = leased_relay_target(relay_root.path());
4778        target.project_memory = Some(ProjectMemorySyncTarget {
4779            canonical_root: canonical.path().to_path_buf(),
4780        });
4781
4782        let mut connection = StandaloneSession::connect(&target)
4783            .await
4784            .expect("relay attach must not depend on its memory endpoint");
4785        assert!(
4786            connection.project_memory.is_some(),
4787            "attach must leave memory pending for an explicit checkpoint sync"
4788        );
4789
4790        connection
4791            .sync_project_memory()
4792            .await
4793            .expect("an explicit sync may detect a legacy memory endpoint");
4794        assert!(
4795            connection.project_memory.is_none(),
4796            "the explicit sync reached the relay and disabled its unavailable endpoint"
4797        );
4798    }
4799
4800    /// Catching the local projection up to an accepted command is the
4801    /// expensive half of a submit, and a caller waiting to hear that the relay
4802    /// took the command should not wait for it. The two are separate calls, so
4803    /// the cheap one can answer first.
4804    #[cfg(unix)]
4805    #[tokio::test]
4806    async fn submitting_does_not_catch_the_projection_up_until_asked() {
4807        if std::env::var_os(SUBMIT_WITHOUT_SYNC_TEST_CHILD).is_none() {
4808            run_in_isolated_child(
4809                SUBMIT_WITHOUT_SYNC_TEST_CHILD,
4810                "submitting_does_not_catch_the_projection_up_until_asked",
4811            );
4812            return;
4813        }
4814        // Alone in this child process, so it installs the one writer.
4815        let _writer = crate::database::install_isolated_test_writer();
4816        register_leased_relay_session();
4817        let relay_root = tempfile::tempdir().unwrap();
4818        let mut connection = StandaloneSession::connect(&leased_relay_target(relay_root.path()))
4819            .await
4820            .expect("connect to the live test relay");
4821        let before = connection.materialized.applied_event_ordinal;
4822
4823        let ordinal = connection
4824            .submit_accepted(
4825                new_command_id("prompt").unwrap(),
4826                RelayCommand::Prompt {
4827                    prompt: vec![ContentBlock::Text(TextContent::new("hello"))],
4828                },
4829            )
4830            .await
4831            .expect("the relay accepted the command");
4832        assert!(ordinal > before, "the relay reported where it accepted it");
4833        assert_eq!(
4834            connection.materialized.applied_event_ordinal, before,
4835            "the caller was answered without paying for the catch-up"
4836        );
4837
4838        connection.sync().await.expect("catch the projection up");
4839        assert!(
4840            connection.materialized.applied_event_ordinal > before,
4841            "the catch-up is what advances the projection"
4842        );
4843    }
4844
4845    #[cfg(unix)]
4846    #[test]
4847    fn stale_recovery_checks_durable_state_under_target_ownership() {
4848        const CHILD: &str = "MJ_STALE_RECOVERY_TEST_CHILD";
4849        if std::env::var_os(CHILD).is_none() {
4850            run_in_isolated_child(
4851                CHILD,
4852                "stale_recovery_checks_durable_state_under_target_ownership",
4853            );
4854            return;
4855        }
4856        let _writer = crate::database::install_isolated_test_writer();
4857        register_leased_relay_session();
4858        let mut record =
4859            crate::database::load_state().unwrap().sessions[LEASED_RELAY_SESSION].clone();
4860        record.target = Some(recovery_source_target());
4861        let plan = WorkerRecoveryPlan {
4862            source_target: recovery_source_target(),
4863            target: None,
4864            workspace: None,
4865            liveness_probe: CommandSpec::new("probe", std::iter::empty::<&str>()),
4866            binary_refresh: None,
4867            launch_refresh: None,
4868            restart: CommandPlan {
4869                description: "restart".into(),
4870                commands: vec![CommandSpec::new("restart", std::iter::empty::<&str>())],
4871            },
4872        };
4873        #[derive(Default)]
4874        struct RecordingExecutor(Mutex<Vec<String>>);
4875        impl CommandExecutor for RecordingExecutor {
4876            fn execute(&self, command: &CommandSpec) -> Result<crate::targets::CommandOutput> {
4877                self.0.lock().unwrap().push(command.program.clone());
4878                Ok(crate::targets::CommandOutput {
4879                    status: 0,
4880                    stdout: b"dead\n".to_vec(),
4881                    stderr: Vec::new(),
4882                })
4883            }
4884        }
4885        let executor = RecordingExecutor::default();
4886        use mj_core::state::SessionState;
4887        for state in [
4888            SessionState::Destroying,
4889            SessionState::Stopped,
4890            SessionState::Lost,
4891            SessionState::Error,
4892            SessionState::Provisioning,
4893            SessionState::DestroyedWithDataLoss,
4894        ] {
4895            record.state = state;
4896            record.last_error = Some("cleanup is safely retryable".into());
4897            crate::database::save_session(&record).unwrap();
4898            assert_eq!(
4899                recover_worker_controlled(plan.clone(), true, Some(&record.id), &executor).unwrap(),
4900                WorkerRecoveryOutcome::Suppressed
4901            );
4902        }
4903        record.state = SessionState::Running;
4904        for target in [
4905            None,
4906            Some(mj_core::state::TargetLocator::LocalBare {
4907                worker_root: PathBuf::from("/replacement-worker").join(LEASED_RELAY_SESSION),
4908            }),
4909        ] {
4910            record.target = target;
4911            crate::database::save_session(&record).unwrap();
4912            assert_eq!(
4913                recover_worker_controlled(plan.clone(), true, Some(&record.id), &executor).unwrap(),
4914                WorkerRecoveryOutcome::Suppressed
4915            );
4916        }
4917        assert_eq!(
4918            recover_worker_controlled(plan.clone(), true, Some("removed-session"), &executor)
4919                .unwrap(),
4920            WorkerRecoveryOutcome::Suppressed
4921        );
4922        assert!(executor.0.lock().unwrap().is_empty());
4923
4924        record.target = Some(recovery_source_target());
4925        for state in [
4926            SessionState::Running,
4927            SessionState::Disconnected,
4928            SessionState::Checkpointing,
4929            SessionState::Closing,
4930        ] {
4931            record.state = state;
4932            crate::database::save_session(&record).unwrap();
4933            assert_eq!(
4934                recover_worker_controlled(plan.clone(), true, Some(&record.id), &executor).unwrap(),
4935                WorkerRecoveryOutcome::RestartedDead
4936            );
4937        }
4938        executor.0.lock().unwrap().clear();
4939
4940        // A plan queued while Closing must read Destroying only after cleanup
4941        // releases ownership, rather than use the actor's old observation.
4942        let mutex = crate::recovery_gate::worker_target_mutex(&record.id);
4943        let guard = mutex.lock().unwrap();
4944        std::thread::scope(|scope| {
4945            let pending = scope.spawn(|| {
4946                recover_worker_controlled(plan.clone(), true, Some(&record.id), &executor)
4947            });
4948            let mut destroying = record.clone();
4949            destroying.state = SessionState::Destroying;
4950            crate::database::save_session(&destroying).unwrap();
4951            drop(guard);
4952            assert_eq!(
4953                pending.join().unwrap().unwrap(),
4954                WorkerRecoveryOutcome::Suppressed
4955            );
4956        });
4957        assert!(executor.0.lock().unwrap().is_empty());
4958
4959        // A storage failure also refuses recovery before touching the target.
4960        let connection = rusqlite::Connection::open(crate::database::database_path()).unwrap();
4961        connection.execute("DROP TABLE sessions", []).unwrap();
4962        let error = recover_worker_controlled(plan, true, Some(&record.id), &executor).unwrap_err();
4963        assert!(format!("{error:#}").contains("read durable session before worker recovery"));
4964        assert!(executor.0.lock().unwrap().is_empty());
4965    }
4966
4967    #[cfg(unix)]
4968    #[tokio::test]
4969    async fn unresponsive_live_relay_worker_is_restarted_and_reconnected() {
4970        if std::env::var_os(AUTO_RESTART_TEST_CHILD).is_none() {
4971            run_in_isolated_child(
4972                AUTO_RESTART_TEST_CHILD,
4973                "unresponsive_live_relay_worker_is_restarted_and_reconnected",
4974            );
4975            return;
4976        }
4977        // Alone in this child process, so it installs the one writer.
4978        let _writer = crate::database::install_isolated_test_writer();
4979        fail_if_the_actor_stalls("unresponsive live relay worker was never restarted");
4980        register_leased_relay_session();
4981        let mut record =
4982            crate::database::load_state().unwrap().sessions[LEASED_RELAY_SESSION].clone();
4983        record.target = Some(recovery_source_target());
4984        crate::database::save_session(&record).unwrap();
4985        let relay_root = tempfile::tempdir().unwrap();
4986        let restarted = relay_root.path().join("worker-restarted");
4987        let script = format!(
4988            "if [ ! -f \"${AUTO_RESTART_MARKER}\" ]; then IFS= read -r _; exit 0; fi; \
4989             \"$0\" --exact {} --nocapture | grep --line-buffered '^{{'",
4990            exact_test_name("leased_relay_child_serves_stdio")
4991        );
4992        let mut spec = CommandSpec::new(
4993            "sh",
4994            [
4995                "-c".to_owned(),
4996                script,
4997                std::env::current_exe()
4998                    .unwrap()
4999                    .to_string_lossy()
5000                    .into_owned(),
5001            ],
5002        )
5003        .purpose("test restartable relay");
5004        spec.env.insert(
5005            LEASED_RELAY_ROOT.to_owned(),
5006            relay_root.path().to_string_lossy().into_owned(),
5007        );
5008        spec.env.insert(
5009            AUTO_RESTART_MARKER.to_owned(),
5010            restarted.to_string_lossy().into_owned(),
5011        );
5012        let worker_recovery = WorkerRecoveryPlan {
5013            source_target: recovery_source_target(),
5014            target: None,
5015            workspace: None,
5016            liveness_probe: CommandSpec::new("printf", ["alive\n"])
5017                .purpose("probe test relay worker"),
5018            binary_refresh: None,
5019            launch_refresh: None,
5020            restart: CommandPlan {
5021                description: "restart test relay worker".into(),
5022                commands: vec![
5023                    CommandSpec::new("touch", [restarted.to_string_lossy().into_owned()])
5024                        .purpose("restart test relay worker"),
5025                ],
5026            },
5027        };
5028        let target = RelaySessionTarget {
5029            session_id: LEASED_RELAY_SESSION.to_owned(),
5030            spec,
5031            worker_recovery: Some(worker_recovery),
5032            project_memory: None,
5033        };
5034        let (_commands_tx, commands_rx) = mpsc::channel(4);
5035        let (_releases_tx, releases_rx) = mpsc::unbounded_channel();
5036        let (_retirement_tx, retirement_rx) = watch::channel(false);
5037        let (view_tx, mut view_rx) = watch::channel(ManagedSessionView::default());
5038        let (updates_tx, _updates_rx) = coalesced_update_channel();
5039        tokio::spawn(run_session_actor(
5040            target,
5041            commands_rx,
5042            releases_rx,
5043            retirement_rx,
5044            view_tx,
5045            updates_tx,
5046        ));
5047
5048        tokio::time::timeout(Duration::from_secs(20), async {
5049            loop {
5050                view_rx.changed().await.unwrap();
5051                let view = view_rx.borrow_and_update().clone();
5052                if view.connected {
5053                    assert!(restarted.exists(), "the restart plan did not run");
5054                    assert!(view.error.is_none());
5055                    return;
5056                }
5057            }
5058        })
5059        .await
5060        .unwrap_or_else(|_| panic!("relay stayed disconnected: {:?}", view_rx.borrow().error));
5061    }
5062
5063    /// A deferred submission that is never answered would hang the suite
5064    /// instead of failing it, so turn a stall into a hard error.
5065    #[cfg(unix)]
5066    fn fail_if_the_actor_stalls(reason: &'static str) {
5067        std::thread::spawn(move || {
5068            std::thread::sleep(Duration::from_secs(60));
5069            eprintln!("{reason}");
5070            std::process::exit(101);
5071        });
5072    }
5073
5074    /// A relay target served by this test binary over stdio.
5075    #[cfg(unix)]
5076    fn leased_relay_target(relay_root: &std::path::Path) -> RelaySessionTarget {
5077        // `RelayClient` parses every stdout line as JSON, so libtest's own
5078        // progress lines are dropped before they reach the protocol reader.
5079        let script = format!(
5080            "\"$0\" --exact {} --nocapture | grep --line-buffered '^{{'",
5081            exact_test_name("leased_relay_child_serves_stdio")
5082        );
5083        let mut spec = CommandSpec::new(
5084            "sh",
5085            [
5086                "-c".to_owned(),
5087                script,
5088                std::env::current_exe()
5089                    .unwrap()
5090                    .to_string_lossy()
5091                    .into_owned(),
5092            ],
5093        )
5094        .purpose("test leased relay");
5095        spec.env.insert(
5096            LEASED_RELAY_ROOT.to_owned(),
5097            relay_root.to_string_lossy().into_owned(),
5098        );
5099        RelaySessionTarget {
5100            session_id: LEASED_RELAY_SESSION.to_owned(),
5101            spec,
5102            worker_recovery: None,
5103            project_memory: None,
5104        }
5105    }
5106
5107    /// Register the session the projection writes to. `apply_projection_event`
5108    /// rejects events for sessions the controller database does not know.
5109    #[cfg(unix)]
5110    fn register_leased_relay_session() {
5111        crate::database::save_session(&mj_core::state::SessionRecord {
5112            mjolnir_subagents: None,
5113            create_managed_worktree: None,
5114            workspace_id: mj_core::workspace::DEFAULT_WORKSPACE_ID.to_owned(),
5115            archived: false,
5116            container_cpus: None,
5117            container_memory: None,
5118            id: LEASED_RELAY_SESSION.into(),
5119            title: "leased relay".into(),
5120            harness_kind: mj_core::config::HarnessKind::Codex,
5121            last_profile: "codex".into(),
5122            bundle_id: "project".into(),
5123            project_directory: None,
5124            managed_worktree: None,
5125            target_template_id: "podman".into(),
5126            resource_allocation: None,
5127            additional_mounts: Vec::new(),
5128            state: mj_core::state::SessionState::Running,
5129            target: None,
5130            native_session_id: None,
5131            acp_session_title: None,
5132            session_title_override: None,
5133            created_at: "2026-08-12T00:00:00Z".into(),
5134            updated_at: "2026-08-12T00:00:00Z".into(),
5135            viewed_through_event_ordinal: 0,
5136            draft_input: String::new(),
5137            last_error: None,
5138            last_checkpoint_error: None,
5139            checkpoint: None,
5140        })
5141        .expect("register the test session");
5142    }
5143
5144    #[cfg(unix)]
5145    struct LeasedActor {
5146        commands: mpsc::Sender<ActorCommand>,
5147        releases: mpsc::UnboundedSender<ReturnedConnection>,
5148        retirement: watch::Sender<bool>,
5149        _views: watch::Receiver<ManagedSessionView>,
5150        _updates: SessionManagerUpdates,
5151        _relay_root: tempfile::TempDir,
5152    }
5153
5154    /// Start an actor against a live relay and take its connection under lease.
5155    #[cfg(unix)]
5156    async fn lease_a_live_actor() -> (LeasedActor, u64, StandaloneSession) {
5157        register_leased_relay_session();
5158        let relay_root = tempfile::tempdir().unwrap();
5159        let (commands_tx, commands_rx) = mpsc::channel(4);
5160        let (releases_tx, releases_rx) = mpsc::unbounded_channel();
5161        let (retirement_tx, retirement_rx) = watch::channel(false);
5162        let (view_tx, view_rx) = watch::channel(ManagedSessionView::default());
5163        let (updates_tx, updates_rx) = coalesced_update_channel();
5164        tokio::spawn(run_session_actor(
5165            leased_relay_target(relay_root.path()),
5166            commands_rx,
5167            releases_rx,
5168            retirement_rx,
5169            view_tx,
5170            updates_tx,
5171        ));
5172
5173        let (reply, response) = oneshot::channel();
5174        commands_tx
5175            .send(ActorCommand::Lease { reply })
5176            .await
5177            .unwrap();
5178        let (lease_id, connection) = response
5179            .await
5180            .expect("actor answered the lease request")
5181            .expect("actor leased its relay connection");
5182        (
5183            LeasedActor {
5184                commands: commands_tx,
5185                releases: releases_tx,
5186                retirement: retirement_tx,
5187                _views: view_rx,
5188                _updates: updates_rx,
5189                _relay_root: relay_root,
5190            },
5191            lease_id,
5192            connection,
5193        )
5194    }
5195
5196    #[cfg(unix)]
5197    async fn submit_a_deferred_prompt(
5198        actor: &LeasedActor,
5199    ) -> oneshot::Receiver<std::result::Result<u64, String>> {
5200        let (reply, mut response) = oneshot::channel();
5201        actor
5202            .commands
5203            .send(ActorCommand::Submit {
5204                command_id: new_command_id("prompt").unwrap(),
5205                command: RelayCommand::Prompt {
5206                    prompt: vec![ContentBlock::Text(TextContent::new("hello"))],
5207                },
5208                admission: None,
5209                reply,
5210            })
5211            .await
5212            .unwrap();
5213        assert!(
5214            tokio::time::timeout(Duration::from_millis(300), &mut response)
5215                .await
5216                .is_err(),
5217            "a leased actor must hold the prompt instead of answering it"
5218        );
5219        response
5220    }
5221
5222    #[cfg(unix)]
5223    #[tokio::test]
5224    async fn prompt_submitted_during_lease_is_delivered_after_release() {
5225        if std::env::var_os(DEFERRED_SUBMIT_TEST_CHILD).is_none() {
5226            run_in_isolated_child(
5227                DEFERRED_SUBMIT_TEST_CHILD,
5228                "prompt_submitted_during_lease_is_delivered_after_release",
5229            );
5230            return;
5231        }
5232        // Alone in this child process, so it installs the one writer.
5233        let _writer = crate::database::install_isolated_test_writer();
5234        fail_if_the_actor_stalls("prompt deferred during a lease was never delivered");
5235
5236        let (actor, lease_id, connection) = lease_a_live_actor().await;
5237        let response = submit_a_deferred_prompt(&actor).await;
5238
5239        actor
5240            .releases
5241            .send(ReturnedConnection {
5242                lease_id,
5243                connection: Some(connection),
5244            })
5245            .unwrap();
5246
5247        let ordinal = response
5248            .await
5249            .expect("actor answered the deferred prompt")
5250            .expect("deferred prompt reached the relay");
5251        assert!(
5252            ordinal > 0,
5253            "relay accepted the prompt at ordinal {ordinal}"
5254        );
5255    }
5256
5257    #[cfg(unix)]
5258    #[tokio::test]
5259    async fn returned_lease_publishes_what_it_learned_while_it_held_the_connection() {
5260        if std::env::var_os(RETURNED_LEASE_VIEW_TEST_CHILD).is_none() {
5261            run_in_isolated_child(
5262                RETURNED_LEASE_VIEW_TEST_CHILD,
5263                "returned_lease_publishes_what_it_learned_while_it_held_the_connection",
5264            );
5265            return;
5266        }
5267        // Alone in this child process, so it installs the one writer.
5268        let _writer = crate::database::install_isolated_test_writer();
5269        fail_if_the_actor_stalls("a returned lease never republished its session");
5270
5271        let (actor, lease_id, mut connection) = lease_a_live_actor().await;
5272        let mut views = actor._views.clone();
5273        // The lease applies these events itself, so the actor's own next sync
5274        // has nothing left to catch up on.
5275        let ordinal = connection
5276            .submit(
5277                new_command_id("prompt").unwrap(),
5278                RelayCommand::Prompt {
5279                    prompt: vec![ContentBlock::Text(TextContent::new("hello"))],
5280                },
5281            )
5282            .await
5283            .unwrap();
5284        assert!(views.borrow_and_update().snapshot.is_none());
5285
5286        actor
5287            .releases
5288            .send(ReturnedConnection {
5289                lease_id,
5290                connection: Some(connection),
5291            })
5292            .unwrap();
5293
5294        views.changed().await.unwrap();
5295        let snapshot = views
5296            .borrow_and_update()
5297            .snapshot
5298            .clone()
5299            .expect("the returned connection republished its session");
5300        assert!(
5301            snapshot.materialized.applied_event_ordinal >= ordinal,
5302            "published frontier {} is behind the leased submission at {ordinal}",
5303            snapshot.materialized.applied_event_ordinal
5304        );
5305    }
5306
5307    #[cfg(unix)]
5308    #[tokio::test]
5309    async fn retirement_rejects_prompts_deferred_during_lease() {
5310        if std::env::var_os(RETIRED_SUBMIT_TEST_CHILD).is_none() {
5311            run_in_isolated_child(
5312                RETIRED_SUBMIT_TEST_CHILD,
5313                "retirement_rejects_prompts_deferred_during_lease",
5314            );
5315            return;
5316        }
5317        // Alone in this child process, so it installs the one writer.
5318        let _writer = crate::database::install_isolated_test_writer();
5319        fail_if_the_actor_stalls("prompt deferred during a lease was never answered");
5320
5321        let (actor, lease_id, connection) = lease_a_live_actor().await;
5322        let response = submit_a_deferred_prompt(&actor).await;
5323
5324        actor.retirement.send(true).unwrap();
5325        actor
5326            .releases
5327            .send(ReturnedConnection {
5328                lease_id,
5329                connection: Some(connection),
5330            })
5331            .unwrap();
5332
5333        let error = response
5334            .await
5335            .expect("actor answered the deferred prompt")
5336            .expect_err("a retiring actor must not deliver the prompt");
5337        assert!(
5338            error.contains("session target is changing"),
5339            "unexpected rejection: {error}"
5340        );
5341    }
5342
5343    #[test]
5344    fn projection_integrity_failure_is_detected_only_for_integrity_errors() {
5345        let integrity = anyhow::Error::from(ProjectionIntegrityError(
5346            "transcript item \"tool:call-1\" changed immutable identity fields".into(),
5347        ))
5348        .context("apply projection event");
5349        assert!(projection_integrity_failure(&integrity));
5350
5351        let concurrent = anyhow::Error::from(ProjectionAdvancedError { event_ordinal: 7 });
5352        assert!(!projection_integrity_failure(&concurrent));
5353
5354        let unreachable = anyhow::anyhow!("connection refused").context("connect relay proxy");
5355        assert!(!projection_integrity_failure(&unreachable));
5356    }
5357
5358    #[test]
5359    fn dashboard_updates_keep_only_the_latest_view_per_session() {
5360        let (sender, mut receiver) = coalesced_update_channel();
5361        for revision in 0..1_000 {
5362            sender.send(SessionManagerUpdate {
5363                session_id: "session-1".into(),
5364                view: ManagedSessionView {
5365                    error: Some(ViewError::Unreachable(format!("revision-{revision}"))),
5366                    ..ManagedSessionView::default()
5367                },
5368            });
5369        }
5370        sender.send(SessionManagerUpdate {
5371            session_id: "session-2".into(),
5372            view: ManagedSessionView {
5373                error: Some(ViewError::Unreachable("other".into())),
5374                ..ManagedSessionView::default()
5375            },
5376        });
5377
5378        assert_eq!(
5379            sender
5380                .pending
5381                .lock()
5382                .expect("session update coalescer poisoned")
5383                .len(),
5384            2
5385        );
5386        let updates = [receiver.try_recv().unwrap(), receiver.try_recv().unwrap()]
5387            .into_iter()
5388            .map(|update| (update.session_id, update.view.error.unwrap()))
5389            .collect::<BTreeMap<_, _>>();
5390        assert_eq!(updates["session-1"].detail(), "revision-999");
5391        assert_eq!(updates["session-2"].detail(), "other");
5392        assert!(receiver.try_recv().is_err());
5393    }
5394
5395    #[tokio::test]
5396    async fn remote_session_manager_fans_out_views_and_forwards_commands() {
5397        let mut remote = spawn_remote_session_manager().unwrap();
5398        remote.targets.send_replace(vec![target("unused")]);
5399        remote
5400            .publisher
5401            .publish("session-1".into(), view_at_ordinal(7))
5402            .await
5403            .unwrap();
5404
5405        let session = remote
5406            .control
5407            .wait_for_session("session-1", Duration::from_secs(1))
5408            .await
5409            .unwrap();
5410        assert_eq!(
5411            session
5412                .view()
5413                .snapshot
5414                .as_ref()
5415                .unwrap()
5416                .materialized
5417                .applied_event_ordinal,
5418            7
5419        );
5420
5421        let submitted = session
5422            .enqueue_submit("prompt-1".into(), RelayCommand::Cancel)
5423            .await
5424            .unwrap();
5425        let request = remote.requests.recv().await.unwrap();
5426        match request {
5427            RemoteSessionRequest::Submit {
5428                session_id,
5429                command_id,
5430                command: RelayCommand::Cancel,
5431                admission: None,
5432                reply,
5433            } => {
5434                assert_eq!(session_id, "session-1");
5435                assert_eq!(command_id, "prompt-1");
5436                reply.send(Ok(8)).unwrap();
5437            }
5438            _ => panic!("unexpected remote session request"),
5439        }
5440        assert_eq!(submitted.wait().await.unwrap(), 8);
5441        remote.shutdown.shutdown().await.unwrap();
5442    }
5443}