Skip to main content

mj_controller/
hel_session_manager.rs

1//! Multiplexed controller-side ownership of durable ACP relay sessions.
2
3use std::collections::{BTreeMap, VecDeque};
4use std::path::PathBuf;
5use std::sync::{Arc, Mutex};
6use std::time::{Duration, Instant};
7
8use anyhow::{Context, Result, bail, ensure};
9use tokio::sync::{mpsc, oneshot, watch};
10
11use crate::hel_worker_client::{
12    RelayAttachment, RelayClient, RelayEventPage, RelayRejected, RelayTransportDead,
13    StartedReviewer,
14};
15use hel::hel_archive::verify_archive_streaming;
16use hel::hel_credentials::{CredentialSyncSignal, relay_event_credential_sync_reason};
17use hel::hel_database::{
18    ProjectionApplyOutcome, ProjectionIntegrityError, apply_projection_page,
19    save_materialized_session,
20};
21use hel::hel_elicitation::ElicitationResponse;
22use hel::hel_projection::{
23    ProjectionIndex, apply_committed_projection_event_indexed, materialized_session_from_canonical,
24    project_relay_event_indexed,
25};
26use hel::hel_state::{ManagedSessionSnapshot, MaterializedSession};
27use hel::hel_targets::{
28    CancellableProcessExecutor, CommandExecutor, CommandPlan, CommandSpec, TargetLocator,
29    TargetRecoveryOutcome, TargetRecoveryPlan, ensure_recovery_target_running,
30};
31use hel::hel_worker::{RelayCommand, RelayCursor, RelayOperationalState};
32use hel::hel_worker_launch::ReviewerLaunchConfig;
33
34const SESSION_SYNC_INTERVAL: Duration = Duration::from_millis(150);
35/// Release SQLite's single writer between bounded pieces of a large relay
36/// catch-up. One transport page can contain thousands of terminal events and
37/// must not prevent every other session actor from publishing its view.
38const PROJECTION_TRANSACTION_EVENT_BUDGET: usize = 128;
39const RECONNECT_INTERVAL: Duration = Duration::from_secs(1);
40/// Ceiling for reconnect backoff. A worker that exited stays gone until the
41/// user acts, so retrying it every second only burns process spawns.
42const RECONNECT_BACKOFF_CEILING: Duration = Duration::from_secs(30);
43const UNREACHABLE_FAILURE_THRESHOLD: u32 = 2;
44const WORKER_RESTART_TIMEOUT: Duration = Duration::from_secs(30);
45const WORKER_RESTART_COOLDOWN: Duration = Duration::from_secs(60);
46const SESSION_MANAGER_SHUTDOWN_GRACE: Duration = Duration::from_millis(750);
47
48#[derive(Debug)]
49struct ProjectionAdvancedError {
50    event_ordinal: u64,
51}
52
53impl std::fmt::Display for ProjectionAdvancedError {
54    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
55        write!(
56            formatter,
57            "another projector committed relay event {} first",
58            self.event_ordinal
59        )
60    }
61}
62
63impl std::error::Error for ProjectionAdvancedError {}
64
65/// Delay before the next reconnect attempt after `failures` consecutive
66/// failures. Doubles from `RECONNECT_INTERVAL` up to the ceiling.
67fn reconnect_delay(failures: u32) -> Duration {
68    let doubling = failures.saturating_sub(1).min(u32::BITS - 1);
69    RECONNECT_INTERVAL
70        .saturating_mul(1_u32 << doubling)
71        .min(RECONNECT_BACKOFF_CEILING)
72}
73
74#[derive(Debug, Clone, PartialEq, Eq)]
75pub struct RelaySessionTarget {
76    pub session_id: String,
77    pub spec: CommandSpec,
78    /// Prove the exact worker is absent before restarting it in place. Direct
79    /// relay clients omit recovery; controller-managed sessions self-heal
80    /// without turning a shared transport outage into destructive restarts.
81    pub worker_recovery: Option<WorkerRecoveryPlan>,
82    pub project_memory: Option<ProjectMemorySyncTarget>,
83}
84
85#[derive(Debug, Clone, PartialEq, Eq)]
86pub struct ProjectMemorySyncTarget {
87    pub canonical_root: std::path::PathBuf,
88}
89
90#[derive(Debug, Clone, PartialEq, Eq)]
91pub struct WorkerRecoveryPlan {
92    pub target: Option<TargetRecoveryPlan>,
93    pub liveness_probe: CommandSpec,
94    /// Refresh a stale installed worker before restarting it. The digest is
95    /// computed inside the recovery task so hashing a large binary never
96    /// blocks a controller UI loop.
97    pub binary_refresh: Option<WorkerBinaryRefresh>,
98    /// Keep the worker executable and its launch schema paired. Configuration
99    /// bytes travel through redacted stdin only when their digest is stale.
100    pub launch_refresh: Option<WorkerLaunchRefreshPlan>,
101    pub restart: CommandPlan,
102}
103
104/// How recovery refreshes a stale installed worker binary before restarting.
105///
106/// Local targets resolve the source and the copy at plan-build time, which is
107/// cheap. Remote targets cannot: choosing the binary needs the target's
108/// architecture, and that probe plus hashing the remote binary are blocking
109/// ssh round-trips that must not run on the plan-build/UI path. So a remote
110/// refresh carries only what is cheap to compute and resolves the rest inside
111/// the recovery task.
112#[derive(Debug, Clone, PartialEq, Eq)]
113pub enum WorkerBinaryRefresh {
114    Prepared(WorkerBinaryRefreshPlan),
115    Remote(RemoteWorkerBinaryRefresh),
116}
117
118#[derive(Debug, Clone, PartialEq, Eq)]
119pub struct WorkerBinaryRefreshPlan {
120    pub source: PathBuf,
121    pub installed_digest: CommandSpec,
122    pub replace: CommandPlan,
123}
124
125/// A remote worker refresh resolved at recovery time: select the worker binary
126/// for the target's own architecture, compare it to the installed one, and
127/// copy only when they differ.
128#[derive(Debug, Clone, PartialEq, Eq)]
129pub struct RemoteWorkerBinaryRefresh {
130    pub locator: TargetLocator,
131    pub session_id: String,
132    pub installed_digest: CommandSpec,
133}
134
135#[derive(Debug, Clone, PartialEq, Eq)]
136pub struct WorkerLaunchRefreshPlan {
137    pub expected_sha256: String,
138    pub installed_digest: CommandSpec,
139    pub replace: CommandPlan,
140}
141
142/// Whether a relay failure means the transport to the worker is gone, so
143/// restarting that worker is the only recovery left.
144///
145/// Every failure that proves it is marked with [`RelayTransportDead`] where it
146/// is produced, and this decision downcasts for that marker. Message text is
147/// never read: a reworded diagnostic must not be able to disable auto-restart.
148pub(crate) fn worker_connect_needs_restart(error: &anyhow::Error) -> bool {
149    RelayTransportDead::marks(error)
150}
151
152fn worker_connect_allows_live_restart(error: &anyhow::Error) -> bool {
153    RelayTransportDead::marks_failed_handshake(error)
154}
155
156#[derive(Debug, Clone, Copy, PartialEq, Eq)]
157enum WorkerRecoveryOutcome {
158    Alive,
159    Starting,
160    TargetMissing,
161    RestartedDead,
162    RestartedUnresponsive,
163}
164
165fn refresh_worker_binary_if_stale(
166    executor: &impl CommandExecutor,
167    refresh: Option<&WorkerBinaryRefresh>,
168) -> Result<()> {
169    match refresh {
170        None => Ok(()),
171        Some(WorkerBinaryRefresh::Prepared(plan)) => {
172            let expected = hel::hel_worker_launch::worker_executable_digest(&plan.source)?;
173            if installed_digest_matches(executor, &plan.installed_digest, &expected) {
174                return Ok(());
175            }
176            plan.replace
177                .execute(executor)
178                .context("replace stale relay worker binary")?;
179            Ok(())
180        }
181        // Remote: pick the binary for the target's architecture and copy only
182        // if it differs. Runs here in the recovery task, never on the UI path.
183        Some(WorkerBinaryRefresh::Remote(refresh)) => {
184            crate::hel_controller::refresh_remote_worker_binary_if_stale(executor, refresh)
185        }
186    }
187}
188
189fn installed_digest_matches(
190    executor: &impl CommandExecutor,
191    command: &CommandSpec,
192    expected: &str,
193) -> bool {
194    executor.execute(command).as_ref().is_ok_and(|output| {
195        output.status == 0
196            && String::from_utf8_lossy(&output.stdout)
197                .split_whitespace()
198                .next()
199                .is_some_and(|digest| digest.eq_ignore_ascii_case(expected))
200    })
201}
202
203fn refresh_worker_launch_if_stale(
204    executor: &impl CommandExecutor,
205    plan: Option<&WorkerLaunchRefreshPlan>,
206) -> Result<()> {
207    let Some(plan) = plan else {
208        return Ok(());
209    };
210    if installed_digest_matches(executor, &plan.installed_digest, &plan.expected_sha256) {
211        return Ok(());
212    }
213    plan.replace
214        .execute(executor)
215        .context("replace stale relay worker launch config")?;
216    Ok(())
217}
218
219async fn recover_worker(
220    plan: WorkerRecoveryPlan,
221    restart_unresponsive: bool,
222) -> Result<WorkerRecoveryOutcome> {
223    tokio::task::spawn_blocking(move || {
224        let executor = CancellableProcessExecutor::with_timeout(WORKER_RESTART_TIMEOUT);
225        if ensure_recovery_target_running(&executor, plan.target.as_ref())
226            .context("restore relay worker target")?
227            == TargetRecoveryOutcome::Missing
228        {
229            return Ok(WorkerRecoveryOutcome::TargetMissing);
230        }
231        let output = executor
232            .execute(&plan.liveness_probe)
233            .context("probe relay worker liveness")?;
234        if output.status != 0 {
235            bail!(
236                "{} failed with status {}: {}",
237                plan.liveness_probe.purpose,
238                output.status,
239                String::from_utf8_lossy(&output.stderr).trim()
240            );
241        }
242        match String::from_utf8_lossy(&output.stdout).trim() {
243            "starting" => Ok(WorkerRecoveryOutcome::Starting),
244            "alive" if !restart_unresponsive => Ok(WorkerRecoveryOutcome::Alive),
245            "alive" => {
246                refresh_worker_binary_if_stale(&executor, plan.binary_refresh.as_ref())?;
247                refresh_worker_launch_if_stale(&executor, plan.launch_refresh.as_ref())?;
248                plan.restart.execute(&executor)?;
249                Ok(WorkerRecoveryOutcome::RestartedUnresponsive)
250            }
251            "dead" => {
252                refresh_worker_binary_if_stale(&executor, plan.binary_refresh.as_ref())?;
253                refresh_worker_launch_if_stale(&executor, plan.launch_refresh.as_ref())?;
254                plan.restart.execute(&executor)?;
255                Ok(WorkerRecoveryOutcome::RestartedDead)
256            }
257            output => bail!("worker liveness probe returned unexpected output {output:?}"),
258        }
259    })
260    .await
261    .context("worker recovery task failed")?
262}
263
264/// Why a managed session stopped producing fresh views. The kind matters to
265/// callers: an unreachable relay is worth retrying and diagnosing, while a
266/// projection integrity failure is deterministic and needs a different report.
267#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
268#[serde(tag = "kind", content = "detail", rename_all = "snake_case")]
269pub enum ViewError {
270    Unreachable(String),
271    TargetMissing(String),
272    ProjectionIntegrity(String),
273}
274
275impl ViewError {
276    pub fn detail(&self) -> &str {
277        match self {
278            Self::Unreachable(detail)
279            | Self::TargetMissing(detail)
280            | Self::ProjectionIntegrity(detail) => detail,
281        }
282    }
283}
284
285#[derive(Debug, Clone, PartialEq, Default)]
286pub struct ManagedSessionView {
287    pub snapshot: Option<ManagedSessionSnapshot>,
288    pub connected: bool,
289    pub error: Option<ViewError>,
290}
291
292#[derive(Debug, Clone)]
293pub struct SessionManagerUpdate {
294    pub session_id: String,
295    pub view: ManagedSessionView,
296}
297
298pub struct SessionManagerChannels {
299    pub targets: watch::Sender<Vec<RelaySessionTarget>>,
300    pub control: SessionManagerControl,
301    pub updates: SessionManagerUpdates,
302    pub shutdown: SessionManagerShutdown,
303}
304
305/// Client-side half of a remotely owned session manager.
306///
307/// The daemon remains the only process with relay connections. A control
308/// surface publishes the daemon's latest views here and forwards requests from
309/// [`RemoteSessionRequests`] over its authenticated transport.
310pub struct RemoteSessionManagerChannels {
311    pub targets: watch::Sender<Vec<RelaySessionTarget>>,
312    pub control: SessionManagerControl,
313    pub updates: SessionManagerUpdates,
314    pub shutdown: SessionManagerShutdown,
315    pub publisher: RemoteSessionPublisher,
316    pub requests: RemoteSessionRequests,
317}
318
319#[derive(Clone)]
320pub struct RemoteSessionPublisher {
321    updates: mpsc::UnboundedSender<RemoteManagerUpdate>,
322}
323
324impl RemoteSessionPublisher {
325    pub async fn publish(&self, session_id: String, view: ManagedSessionView) -> Result<()> {
326        self.updates
327            .send(RemoteManagerUpdate::Publish { session_id, view })
328            .context("remote session manager stopped")
329    }
330
331    pub fn try_publish(&self, session_id: String, view: ManagedSessionView) -> Result<()> {
332        self.updates
333            .send(RemoteManagerUpdate::Publish { session_id, view })
334            .context("remote session manager update queue is unavailable")
335    }
336}
337
338pub struct RemoteSessionRequests {
339    requests: mpsc::Receiver<RemoteSessionRequest>,
340}
341
342impl RemoteSessionRequests {
343    pub async fn recv(&mut self) -> Option<RemoteSessionRequest> {
344        self.requests.recv().await
345    }
346}
347
348/// What a caller asks of a session's second-opinion reviewer.
349///
350/// The reviewer is a sidecar of the session's worker, so every action travels
351/// the session's own relay connection rather than opening a second one.
352#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
353#[serde(rename_all = "snake_case")]
354pub enum ReviewerAction {
355    Start {
356        config: Box<ReviewerLaunchConfig>,
357    },
358    Submit {
359        command_id: String,
360        command: RelayCommand,
361    },
362    Attach {
363        after_ordinal: u64,
364        after_digest: String,
365    },
366    Acknowledge {
367        through_ordinal: u64,
368        through_digest: String,
369    },
370    Status,
371    /// Answer a form the reviewer's harness is waiting on. A reviewer left
372    /// waiting on one stalls the whole review.
373    RespondElicitation {
374        elicitation_id: String,
375        response: ElicitationResponse,
376    },
377    Pause,
378    /// Report what the workspace repositories changed since these baselines.
379    CaptureDelta {
380        baselines: std::collections::BTreeMap<std::path::PathBuf, String>,
381    },
382    /// Record the trees a completed review reviewed through.
383    AdvanceBaseline {
384        trees: std::collections::BTreeMap<std::path::PathBuf, String>,
385    },
386    /// Run Bifrost's semantic diff analysis over the captured trees.
387    AnalyzeDelta {
388        repositories: Vec<hel::hel_worker::AnalyzeDeltaRepository>,
389    },
390    /// Collect the specialist lanes the review supervisor asked for.
391    TakeLaneDispatches,
392}
393
394impl ReviewerAction {
395    pub const fn operation_name(&self) -> &'static str {
396        match self {
397            Self::Start { .. } => "reviewer_start",
398            Self::Submit { .. } => "reviewer_submit",
399            Self::Attach { .. } => "reviewer_attach",
400            Self::Acknowledge { .. } => "reviewer_acknowledge",
401            Self::Status => "reviewer_status",
402            Self::RespondElicitation { .. } => "reviewer_respond_elicitation",
403            Self::Pause => "reviewer_pause",
404            Self::CaptureDelta { .. } => "reviewer_capture_delta",
405            Self::AdvanceBaseline { .. } => "reviewer_advance_baseline",
406            Self::AnalyzeDelta { .. } => "reviewer_analyze_delta",
407            Self::TakeLaneDispatches => "reviewer_take_lane_dispatches",
408        }
409    }
410}
411
412/// What a [`ReviewerAction`] produced.
413#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
414#[serde(rename_all = "snake_case")]
415pub enum ReviewerOutcome {
416    Started(Box<StartedReviewer>),
417    Accepted {
418        ordinal: u64,
419    },
420    Attached(Box<RelayAttachment>),
421    Acknowledged(RelayCursor),
422    Status(Box<RelayOperationalState>),
423    ElicitationResolved,
424    Paused,
425    /// What every workspace repository changed since the stored baselines.
426    Delta {
427        repositories: Vec<hel::hel_worker::RepoDelta>,
428    },
429    BaselineAdvanced,
430    /// Bifrost's changed-callable packet for the captured trees.
431    ChangedFunctions {
432        packet: String,
433    },
434    /// Specialist lanes the review supervisor asked for.
435    LaneDispatches {
436        requests: Vec<hel::hel_review::lanes::ReviewSubagentRequest>,
437    },
438}
439
440pub enum RemoteSessionRequest {
441    Submit {
442        session_id: String,
443        command_id: String,
444        command: RelayCommand,
445        reply: oneshot::Sender<std::result::Result<u64, String>>,
446    },
447    Sync {
448        session_id: String,
449        reply: oneshot::Sender<std::result::Result<(), String>>,
450    },
451    RespondElicitation {
452        session_id: String,
453        elicitation_id: String,
454        response: ElicitationResponse,
455        reply: oneshot::Sender<std::result::Result<(), String>>,
456    },
457    Reviewer {
458        session_id: String,
459        /// Which reviewing role the action drives; `None` is the default one.
460        role: Option<String>,
461        action: ReviewerAction,
462        reply: oneshot::Sender<std::result::Result<ReviewerOutcome, String>>,
463    },
464}
465
466impl RemoteSessionRequest {
467    /// The session this request acts on. Requests for one session have to be
468    /// carried out in the order they were made.
469    pub fn session_id(&self) -> &str {
470        match self {
471            Self::Submit { session_id, .. }
472            | Self::Sync { session_id, .. }
473            | Self::RespondElicitation { session_id, .. }
474            | Self::Reviewer { session_id, .. } => session_id,
475        }
476    }
477}
478
479/// Keeps each session's relay requests in the order they were made, while
480/// letting different sessions overlap.
481///
482/// A bridge that spawns every request concurrently loses the order the caller
483/// submitted them in, and the order is load-bearing: `/effort` followed by a
484/// prompt has to reach the relay that way round, or the prompt runs under the
485/// old setting. Awaiting each request inline would restore the order but would
486/// also make one slow session block every other one, so instead each request
487/// waits on its own session's previous request and nothing else.
488#[derive(Default)]
489pub struct SessionRequestOrder {
490    latest: std::collections::HashMap<String, tokio::task::JoinHandle<()>>,
491}
492
493impl SessionRequestOrder {
494    #[must_use]
495    pub fn new() -> Self {
496        Self::default()
497    }
498
499    /// Runs `forward` for `request` after everything already queued for the
500    /// same session has finished.
501    pub fn dispatch<F, Fut>(&mut self, request: RemoteSessionRequest, forward: F)
502    where
503        F: FnOnce(RemoteSessionRequest) -> Fut + Send + 'static,
504        Fut: std::future::Future<Output = ()> + Send,
505    {
506        // Sessions that have gone quiet leave a finished handle behind; drop
507        // them here so the map tracks live work rather than every session the
508        // bridge has ever served.
509        self.latest.retain(|_, handle| !handle.is_finished());
510        let session_id = request.session_id().to_owned();
511        let previous = self.latest.remove(&session_id);
512        let handle = tokio::spawn(async move {
513            if let Some(previous) = previous {
514                // A panicked predecessor still releases its successor: the
515                // request behind it is the user's, and dropping it silently
516                // would be worse than running it late.
517                let _ = previous.await;
518            }
519            forward(request).await;
520        });
521        self.latest.insert(session_id, handle);
522    }
523}
524
525/// Exclusive owner of the manager task and every relay actor below it.
526///
527/// Long-running control surfaces explicitly await [`Self::shutdown`] before
528/// their Tokio runtime goes away. Drop remains an aborting fallback for tests
529/// and early-return paths that cannot await.
530pub struct SessionManagerShutdown {
531    signal: Option<oneshot::Sender<()>>,
532    task: Option<tokio::task::JoinHandle<()>>,
533}
534
535impl SessionManagerShutdown {
536    pub async fn shutdown(mut self) -> Result<()> {
537        if let Some(signal) = self.signal.take() {
538            let _ = signal.send(());
539        }
540        if let Some(task) = self.task.take() {
541            task.await.context("session manager shutdown task failed")?;
542        }
543        Ok(())
544    }
545}
546
547impl Drop for SessionManagerShutdown {
548    fn drop(&mut self) {
549        if let Some(signal) = self.signal.take() {
550            let _ = signal.send(());
551        }
552        if let Some(task) = self.task.take() {
553            task.abort();
554        }
555    }
556}
557
558#[derive(Clone)]
559struct CoalescedUpdateSender {
560    pending: Arc<Mutex<BTreeMap<String, SessionManagerUpdate>>>,
561    wake: mpsc::Sender<()>,
562}
563
564/// Bounded latest-state feed for the dashboard. At most one snapshot per
565/// session is retained while the consumer is busy.
566pub struct SessionManagerUpdates {
567    pending: Arc<Mutex<BTreeMap<String, SessionManagerUpdate>>>,
568    wake: mpsc::Receiver<()>,
569}
570
571impl CoalescedUpdateSender {
572    fn send(&self, update: SessionManagerUpdate) {
573        if self.wake.is_closed() {
574            return;
575        }
576        self.pending
577            .lock()
578            .expect("session update coalescer poisoned")
579            .insert(update.session_id.clone(), update);
580        let _ = self.wake.try_send(());
581    }
582}
583
584impl SessionManagerUpdates {
585    fn pop_pending(&self) -> Option<SessionManagerUpdate> {
586        self.pending
587            .lock()
588            .expect("session update coalescer poisoned")
589            .pop_first()
590            .map(|(_, update)| update)
591    }
592
593    pub async fn recv(&mut self) -> Option<SessionManagerUpdate> {
594        loop {
595            if let Some(update) = self.pop_pending() {
596                return Some(update);
597            }
598            self.wake.recv().await?;
599        }
600    }
601
602    pub fn try_recv(
603        &mut self,
604    ) -> std::result::Result<SessionManagerUpdate, mpsc::error::TryRecvError> {
605        if let Some(update) = self.pop_pending() {
606            return Ok(update);
607        }
608        self.wake.try_recv()?;
609        self.pop_pending().ok_or(mpsc::error::TryRecvError::Empty)
610    }
611}
612
613fn coalesced_update_channel() -> (CoalescedUpdateSender, SessionManagerUpdates) {
614    let pending = Arc::new(Mutex::new(BTreeMap::new()));
615    let (wake_tx, wake_rx) = mpsc::channel(1);
616    (
617        CoalescedUpdateSender {
618            pending: pending.clone(),
619            wake: wake_tx,
620        },
621        SessionManagerUpdates {
622            pending,
623            wake: wake_rx,
624        },
625    )
626}
627
628#[derive(Clone)]
629pub struct SessionManagerControl {
630    commands: mpsc::Sender<ManagerCommand>,
631}
632
633#[derive(Clone, Debug)]
634pub struct ManagedSessionHandle {
635    session_id: String,
636    commands: mpsc::Sender<ActorCommand>,
637    releases: mpsc::UnboundedSender<ReturnedConnection>,
638    view: watch::Receiver<ManagedSessionView>,
639}
640
641/// Exclusive ownership of a session actor's existing relay connection.
642///
643/// Lifecycle operations use this instead of opening a competing projection
644/// client. Dropping an unreleased lease drops the proxy connection, which in
645/// turn cancels any ordinary relay checkpoint barrier.
646///
647/// Prompt submissions that arrive while the lease is active are not rejected.
648/// The actor queues them and forwards them in arrival order once the lease is
649/// released or dropped.
650pub struct ManagedSessionLease {
651    session_id: String,
652    lease_id: Option<u64>,
653    connection: Option<StandaloneSession>,
654    releases: mpsc::UnboundedSender<ReturnedConnection>,
655}
656
657impl ManagedSessionLease {
658    pub fn connection_mut(&mut self) -> &mut StandaloneSession {
659        self.connection
660            .as_mut()
661            .expect("managed session lease has already been released")
662    }
663
664    /// Swap the leased proxy after the worker process behind it was replaced.
665    /// The actor stays leased, so queued prompts cannot race the new latch.
666    pub fn replace_connection(&mut self, connection: StandaloneSession) {
667        drop(self.connection.take());
668        self.connection = Some(connection);
669    }
670
671    pub fn release(mut self) {
672        let lease_id = self
673            .lease_id
674            .take()
675            .expect("managed session lease has already been released");
676        let connection = self.connection.take();
677        if let Err(error) = self.releases.send(ReturnedConnection {
678            lease_id,
679            connection,
680        }) {
681            tracing::warn!(
682                session_id = %self.session_id,
683                operation = "lease_release",
684                %error,
685                "session actor stopped before receiving released relay connection"
686            );
687        }
688    }
689}
690
691impl Drop for ManagedSessionLease {
692    fn drop(&mut self) {
693        let Some(lease_id) = self.lease_id.take() else {
694            return;
695        };
696        // Drop the proxy before telling the actor to reconnect so the relay
697        // observes EOF and releases any abandoned checkpoint barrier first.
698        drop(self.connection.take());
699        if let Err(error) = self.releases.send(ReturnedConnection {
700            lease_id,
701            connection: None,
702        }) {
703            tracing::warn!(
704                session_id = %self.session_id,
705                operation = "lease_drop",
706                %error,
707                "session actor stopped before receiving dropped relay lease"
708            );
709        }
710    }
711}
712
713impl ManagedSessionHandle {
714    pub fn session_id(&self) -> &str {
715        &self.session_id
716    }
717
718    pub fn view(&self) -> ManagedSessionView {
719        self.view.borrow().clone()
720    }
721
722    /// Whether the per-session actor behind this handle has retired. The
723    /// manager itself may still be alive with a replacement actor, so callers
724    /// holding long-lived handles use this to reacquire the current one.
725    pub fn is_stopped(&self) -> bool {
726        self.commands.is_closed()
727    }
728
729    pub fn has_changed(&self) -> Result<bool> {
730        self.view.has_changed().context("session manager stopped")
731    }
732
733    pub async fn changed(&mut self) -> Result<ManagedSessionView> {
734        self.view
735            .changed()
736            .await
737            .context("session manager stopped")?;
738        Ok(self.view())
739    }
740
741    pub async fn submit(&self, command_id: String, command: RelayCommand) -> Result<u64> {
742        self.enqueue_submit(command_id, command).await?.wait().await
743    }
744
745    pub async fn enqueue_submit(
746        &self,
747        command_id: String,
748        command: RelayCommand,
749    ) -> Result<PendingRelaySubmit> {
750        let (reply, response) = oneshot::channel();
751        self.commands
752            .send(ActorCommand::Submit {
753                command_id,
754                command,
755                reply,
756            })
757            .await
758            .context("session manager stopped")?;
759        Ok(PendingRelaySubmit { response })
760    }
761
762    pub async fn sync_now(&self) -> Result<()> {
763        self.enqueue_sync().await?.wait().await
764    }
765
766    pub async fn respond_elicitation(
767        &self,
768        elicitation_id: String,
769        response: ElicitationResponse,
770    ) -> Result<()> {
771        let (reply, result) = oneshot::channel();
772        self.commands
773            .send(ActorCommand::RespondElicitation {
774                elicitation_id,
775                response,
776                reply,
777            })
778            .await
779            .context("session manager stopped")?;
780        result
781            .await
782            .context("session manager stopped")?
783            .map_err(anyhow::Error::msg)
784    }
785
786    /// Drive the session's second-opinion reviewer.
787    ///
788    /// The reviewer shares this session's relay connection, so its actions
789    /// queue behind the session's own and are refused while a lifecycle
790    /// operation holds the connection.
791    pub async fn reviewer(&self, action: ReviewerAction) -> Result<ReviewerOutcome> {
792        self.reviewer_as(None, action).await
793    }
794
795    /// Drive one reviewing role. `None` is the default role, which is the one
796    /// plan review uses; a turn review in the extended tier names its
797    /// supervisor, its intent analyst, and each specialist lane.
798    pub async fn reviewer_as(
799        &self,
800        role: Option<String>,
801        action: ReviewerAction,
802    ) -> Result<ReviewerOutcome> {
803        let (reply, result) = oneshot::channel();
804        self.commands
805            .send(ActorCommand::Reviewer {
806                role,
807                action,
808                reply,
809            })
810            .await
811            .context("session manager stopped")?;
812        result
813            .await
814            .context("session manager stopped")?
815            .map_err(anyhow::Error::msg)
816    }
817
818    pub async fn enqueue_sync(&self) -> Result<PendingRelaySync> {
819        let (reply, response) = oneshot::channel();
820        self.commands
821            .send(ActorCommand::Sync { reply })
822            .await
823            .context("session manager stopped")?;
824        Ok(PendingRelaySync { response })
825    }
826
827    pub async fn lease_connection(&self) -> Result<ManagedSessionLease> {
828        let (reply, response) = oneshot::channel();
829        self.commands
830            .send(ActorCommand::Lease { reply })
831            .await
832            .context("session manager stopped")?;
833        let (lease_id, connection) = response.await.context("session manager stopped")??;
834        Ok(ManagedSessionLease {
835            session_id: self.session_id.clone(),
836            lease_id: Some(lease_id),
837            connection: Some(connection),
838            releases: self.releases.clone(),
839        })
840    }
841}
842
843pub struct PendingRelaySubmit {
844    response: oneshot::Receiver<std::result::Result<u64, String>>,
845}
846
847impl PendingRelaySubmit {
848    pub async fn wait(self) -> Result<u64> {
849        self.response
850            .await
851            .context("session manager stopped")?
852            .map_err(anyhow::Error::msg)
853    }
854}
855
856pub struct PendingRelaySync {
857    response: oneshot::Receiver<std::result::Result<(), String>>,
858}
859
860impl PendingRelaySync {
861    pub async fn wait(self) -> Result<()> {
862        self.response
863            .await
864            .context("session manager stopped")?
865            .map_err(anyhow::Error::msg)
866    }
867}
868
869impl SessionManagerControl {
870    pub async fn session(&self, session_id: impl Into<String>) -> Result<ManagedSessionHandle> {
871        let session_id = session_id.into();
872        let (reply, response) = oneshot::channel();
873        self.commands
874            .send(ManagerCommand::Session {
875                session_id: session_id.clone(),
876                reply,
877            })
878            .await
879            .context("session manager stopped")?;
880        response
881            .await
882            .context("session manager stopped")?
883            .with_context(|| format!("session {session_id} is not managed"))
884    }
885
886    pub async fn wait_for_session(
887        &self,
888        session_id: &str,
889        timeout: Duration,
890    ) -> Result<ManagedSessionHandle> {
891        let deadline = tokio::time::Instant::now() + timeout;
892        loop {
893            match self.session(session_id.to_owned()).await {
894                Ok(handle) => return Ok(handle),
895                Err(error) if tokio::time::Instant::now() < deadline => {
896                    tracing::trace!(session_id, "waiting for session actor: {error:#}");
897                    tokio::time::sleep(Duration::from_millis(25)).await;
898                }
899                Err(error) => return Err(error),
900            }
901        }
902    }
903}
904
905enum ManagerCommand {
906    Session {
907        session_id: String,
908        reply: oneshot::Sender<Option<ManagedSessionHandle>>,
909    },
910}
911
912enum ActorCommand {
913    Submit {
914        command_id: String,
915        command: RelayCommand,
916        reply: oneshot::Sender<std::result::Result<u64, String>>,
917    },
918    Sync {
919        reply: oneshot::Sender<std::result::Result<(), String>>,
920    },
921    RespondElicitation {
922        elicitation_id: String,
923        response: ElicitationResponse,
924        reply: oneshot::Sender<std::result::Result<(), String>>,
925    },
926    Reviewer {
927        role: Option<String>,
928        action: ReviewerAction,
929        reply: oneshot::Sender<std::result::Result<ReviewerOutcome, String>>,
930    },
931    /// The connection is handed over whole, and so is the failure: a caller
932    /// that must decide whether to restart the worker needs the typed cause,
933    /// which formatting the error to a string would destroy.
934    Lease {
935        reply: oneshot::Sender<Result<(u64, StandaloneSession)>>,
936    },
937}
938
939impl ActorCommand {
940    fn operation_name(&self) -> &'static str {
941        match self {
942            Self::Submit { .. } => "submit",
943            Self::Sync { .. } => "sync",
944            Self::RespondElicitation { .. } => "respond_elicitation",
945            Self::Reviewer { action, .. } => action.operation_name(),
946            Self::Lease { .. } => "lease",
947        }
948    }
949
950    fn reject(self, session_id: &str, message: &str) {
951        match self {
952            Self::Submit { reply, .. } => {
953                if reply.send(Err(message.to_owned())).is_err() {
954                    tracing::debug!(
955                        %session_id,
956                        operation = "submit",
957                        "submit rejection receiver was already closed"
958                    );
959                }
960            }
961            Self::Sync { reply } => {
962                if reply.send(Err(message.to_owned())).is_err() {
963                    tracing::debug!(
964                        %session_id,
965                        operation = "sync",
966                        "sync rejection receiver was already closed"
967                    );
968                }
969            }
970            Self::RespondElicitation { reply, .. } => {
971                if reply.send(Err(message.to_owned())).is_err() {
972                    tracing::debug!(
973                        %session_id,
974                        operation = "respond_elicitation",
975                        "elicitation rejection receiver was already closed"
976                    );
977                }
978            }
979            Self::Reviewer { reply, .. } => {
980                if reply.send(Err(message.to_owned())).is_err() {
981                    tracing::debug!(
982                        %session_id,
983                        operation = "reviewer",
984                        "reviewer rejection receiver was already closed"
985                    );
986                }
987            }
988            Self::Lease { reply } => {
989                if reply
990                    .send(Err(anyhow::anyhow!(message.to_owned())))
991                    .is_err()
992                {
993                    tracing::debug!(
994                        %session_id,
995                        operation = "lease",
996                        "lease rejection receiver was already closed"
997                    );
998                }
999            }
1000        }
1001    }
1002}
1003
1004struct ReturnedConnection {
1005    lease_id: u64,
1006    connection: Option<StandaloneSession>,
1007}
1008
1009/// A submission that arrived while a lifecycle operation held the connection.
1010/// The actor replays these in arrival order once the lease comes back.
1011struct DeferredSubmit {
1012    command_id: String,
1013    command: RelayCommand,
1014    reply: oneshot::Sender<std::result::Result<u64, String>>,
1015}
1016
1017#[derive(Debug, Default)]
1018struct ActorLifecycle {
1019    active_lease: Option<u64>,
1020    retirement_requested: bool,
1021}
1022
1023impl ActorLifecycle {
1024    fn set_retirement_requested(&mut self, requested: bool) {
1025        self.retirement_requested = requested;
1026    }
1027
1028    fn is_leased(&self) -> bool {
1029        self.active_lease.is_some()
1030    }
1031
1032    fn should_stop(&self) -> bool {
1033        self.retirement_requested && !self.is_leased()
1034    }
1035
1036    fn accepts_new_work(&self) -> bool {
1037        !self.retirement_requested
1038    }
1039
1040    fn activate_lease(&mut self, lease_id: u64) {
1041        debug_assert!(self.active_lease.is_none());
1042        self.active_lease = Some(lease_id);
1043    }
1044
1045    fn return_lease(&mut self, lease_id: u64) -> bool {
1046        if self.active_lease != Some(lease_id) {
1047            return false;
1048        }
1049        self.active_lease = None;
1050        true
1051    }
1052}
1053
1054struct ActorRegistration {
1055    target: RelaySessionTarget,
1056    commands: mpsc::Sender<ActorCommand>,
1057    releases: mpsc::UnboundedSender<ReturnedConnection>,
1058    retirement: watch::Sender<bool>,
1059    view: watch::Receiver<ManagedSessionView>,
1060    abort: tokio::task::AbortHandle,
1061}
1062
1063struct RemoteActorRegistration {
1064    commands: mpsc::Sender<ActorCommand>,
1065    releases: mpsc::UnboundedSender<ReturnedConnection>,
1066    view: watch::Receiver<ManagedSessionView>,
1067    view_tx: watch::Sender<ManagedSessionView>,
1068    abort: tokio::task::AbortHandle,
1069}
1070
1071enum RemoteManagerUpdate {
1072    Publish {
1073        session_id: String,
1074        view: ManagedSessionView,
1075    },
1076}
1077
1078#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1079enum ReconcileAction {
1080    Idle,
1081    Spawn,
1082    Keep,
1083    Retire,
1084}
1085
1086fn reconcile_action(
1087    actor: Option<&RelaySessionTarget>,
1088    desired: Option<&RelaySessionTarget>,
1089) -> ReconcileAction {
1090    match (actor, desired) {
1091        (None, None) => ReconcileAction::Idle,
1092        (None, Some(_)) => ReconcileAction::Spawn,
1093        (Some(actor), Some(desired)) if actor == desired => ReconcileAction::Keep,
1094        (Some(_), Some(_) | None) => ReconcileAction::Retire,
1095    }
1096}
1097
1098fn target_map(targets: &[RelaySessionTarget]) -> BTreeMap<String, RelaySessionTarget> {
1099    targets
1100        .iter()
1101        .cloned()
1102        .map(|target| (target.session_id.clone(), target))
1103        .collect()
1104}
1105
1106fn remove_actor_task(
1107    actors: &mut BTreeMap<String, ActorRegistration>,
1108    task_id: tokio::task::Id,
1109) -> Option<String> {
1110    let session_id = actors.iter().find_map(|(session_id, actor)| {
1111        (actor.abort.id() == task_id).then(|| session_id.clone())
1112    })?;
1113    actors.remove(&session_id);
1114    Some(session_id)
1115}
1116
1117fn reconcile_actors(
1118    targets: &BTreeMap<String, RelaySessionTarget>,
1119    actors: &mut BTreeMap<String, ActorRegistration>,
1120    tasks: &mut tokio::task::JoinSet<String>,
1121    updates: &CoalescedUpdateSender,
1122) {
1123    // A completed or cancelled task closes its command receiver before the
1124    // JoinSet completion necessarily wins the manager's select. Do not let
1125    // that dead registration suppress the replacement this reconciliation is
1126    // responsible for starting. Task-ID-aware completion cleanup below keeps
1127    // the old completion from removing the replacement later.
1128    actors.retain(|session_id, actor| {
1129        let live = !actor.commands.is_closed();
1130        if !live {
1131            tracing::warn!(session_id, "replacing stopped session relay actor");
1132        }
1133        live
1134    });
1135
1136    for (session_id, actor) in actors.iter() {
1137        let retiring = matches!(
1138            reconcile_action(Some(&actor.target), targets.get(session_id)),
1139            ReconcileAction::Retire
1140        );
1141        actor.retirement.send_replace(retiring);
1142    }
1143
1144    for (session_id, target) in targets {
1145        if !matches!(
1146            reconcile_action(
1147                actors.get(session_id).map(|actor| &actor.target),
1148                Some(target)
1149            ),
1150            ReconcileAction::Spawn
1151        ) {
1152            continue;
1153        }
1154        let (actor_tx, actor_rx) = mpsc::channel(32);
1155        let (release_tx, release_rx) = mpsc::unbounded_channel();
1156        let (retirement_tx, retirement_rx) = watch::channel(false);
1157        let (view_tx, view_rx) = watch::channel(ManagedSessionView::default());
1158        let actor_updates = updates.clone();
1159        let task_target = target.clone();
1160        let task_id = session_id.clone();
1161        let abort = tasks.spawn(async move {
1162            run_session_actor(
1163                task_target,
1164                actor_rx,
1165                release_rx,
1166                retirement_rx,
1167                view_tx,
1168                actor_updates,
1169            )
1170            .await;
1171            task_id
1172        });
1173        actors.insert(
1174            session_id.clone(),
1175            ActorRegistration {
1176                target: target.clone(),
1177                commands: actor_tx,
1178                releases: release_tx,
1179                retirement: retirement_tx,
1180                view: view_rx,
1181                abort,
1182            },
1183        );
1184    }
1185}
1186
1187async fn run_remote_session_actor(
1188    session_id: String,
1189    mut commands: mpsc::Receiver<ActorCommand>,
1190    requests: mpsc::Sender<RemoteSessionRequest>,
1191) {
1192    while let Some(command) = commands.recv().await {
1193        let request = match command {
1194            ActorCommand::Submit {
1195                command_id,
1196                command,
1197                reply,
1198            } => RemoteSessionRequest::Submit {
1199                session_id: session_id.clone(),
1200                command_id,
1201                command,
1202                reply,
1203            },
1204            ActorCommand::Sync { reply } => RemoteSessionRequest::Sync {
1205                session_id: session_id.clone(),
1206                reply,
1207            },
1208            ActorCommand::RespondElicitation {
1209                elicitation_id,
1210                response,
1211                reply,
1212            } => RemoteSessionRequest::RespondElicitation {
1213                session_id: session_id.clone(),
1214                elicitation_id,
1215                response,
1216                reply,
1217            },
1218            ActorCommand::Reviewer {
1219                role,
1220                action,
1221                reply,
1222            } => RemoteSessionRequest::Reviewer {
1223                session_id: session_id.clone(),
1224                role,
1225                action,
1226                reply,
1227            },
1228            ActorCommand::Lease { reply } => {
1229                let _ = reply.send(Err(anyhow::anyhow!(
1230                    "relay connection leases are available only inside the controller daemon"
1231                )));
1232                continue;
1233            }
1234        };
1235        if let Err(error) = requests.send(request).await {
1236            match error.0 {
1237                RemoteSessionRequest::Submit { reply, .. } => {
1238                    let _ = reply.send(Err("controller daemon request bridge stopped".into()));
1239                }
1240                RemoteSessionRequest::Sync { reply, .. }
1241                | RemoteSessionRequest::RespondElicitation { reply, .. } => {
1242                    let _ = reply.send(Err("controller daemon request bridge stopped".into()));
1243                }
1244                RemoteSessionRequest::Reviewer { reply, .. } => {
1245                    let _ = reply.send(Err("controller daemon request bridge stopped".into()));
1246                }
1247            }
1248            break;
1249        }
1250    }
1251}
1252
1253fn spawn_remote_actor(
1254    session_id: String,
1255    view: ManagedSessionView,
1256    requests: &mpsc::Sender<RemoteSessionRequest>,
1257    actors: &mut BTreeMap<String, RemoteActorRegistration>,
1258    updates: &CoalescedUpdateSender,
1259) {
1260    let (actor_tx, actor_rx) = mpsc::channel(32);
1261    let (release_tx, _release_rx) = mpsc::unbounded_channel();
1262    let (view_tx, view_rx) = watch::channel(view.clone());
1263    let abort = tokio::spawn(run_remote_session_actor(
1264        session_id.clone(),
1265        actor_rx,
1266        requests.clone(),
1267    ))
1268    .abort_handle();
1269    actors.insert(
1270        session_id.clone(),
1271        RemoteActorRegistration {
1272            commands: actor_tx,
1273            releases: release_tx,
1274            view: view_rx,
1275            view_tx,
1276            abort,
1277        },
1278    );
1279    updates.send(SessionManagerUpdate { session_id, view });
1280}
1281
1282/// Build the read/control facade used by a control surface whose relay actors
1283/// live in another process. Target updates still decide which session handles
1284/// exist, while [`RemoteSessionPublisher`] supplies their latest views.
1285pub fn spawn_remote_session_manager() -> Result<RemoteSessionManagerChannels> {
1286    let (targets_tx, mut targets_rx) = watch::channel(Vec::<RelaySessionTarget>::new());
1287    let (commands_tx, mut commands_rx) = mpsc::channel(32);
1288    let (updates_tx, updates_rx) = coalesced_update_channel();
1289    let (published_tx, mut published_rx) = mpsc::unbounded_channel();
1290    let (requests_tx, requests_rx) = mpsc::channel(64);
1291    let (shutdown_tx, mut shutdown_rx) = oneshot::channel();
1292    let task = tokio::spawn(async move {
1293        let mut actors = BTreeMap::<String, RemoteActorRegistration>::new();
1294        let mut latest = BTreeMap::<String, ManagedSessionView>::new();
1295        let mut desired = BTreeMap::<String, RelaySessionTarget>::new();
1296        loop {
1297            tokio::select! {
1298                _ = &mut shutdown_rx => break,
1299                changed = targets_rx.changed() => {
1300                    if changed.is_err() {
1301                        break;
1302                    }
1303                    desired = target_map(&targets_rx.borrow_and_update());
1304                    actors.retain(|session_id, actor| {
1305                        if desired.contains_key(session_id) {
1306                            true
1307                        } else {
1308                            actor.abort.abort();
1309                            false
1310                        }
1311                    });
1312                    for session_id in desired.keys() {
1313                        if !actors.contains_key(session_id)
1314                            && let Some(view) = latest.get(session_id).cloned()
1315                        {
1316                            spawn_remote_actor(
1317                                session_id.clone(),
1318                                view,
1319                                &requests_tx,
1320                                &mut actors,
1321                                &updates_tx,
1322                            );
1323                        }
1324                    }
1325                }
1326                command = commands_rx.recv() => {
1327                    let Some(ManagerCommand::Session { session_id, reply }) = command else {
1328                        break;
1329                    };
1330                    let handle = actors.get(&session_id).map(|actor| ManagedSessionHandle {
1331                        session_id: session_id.clone(),
1332                        commands: actor.commands.clone(),
1333                        releases: actor.releases.clone(),
1334                        view: actor.view.clone(),
1335                    });
1336                    let _ = reply.send(handle);
1337                }
1338                published = published_rx.recv() => {
1339                    let Some(RemoteManagerUpdate::Publish { session_id, view }) = published else {
1340                        break;
1341                    };
1342                    latest.insert(session_id.clone(), view.clone());
1343                    if !desired.contains_key(&session_id) {
1344                        continue;
1345                    }
1346                    if let Some(actor) = actors.get(&session_id) {
1347                        publish_view(&session_id, view, &actor.view_tx, &updates_tx);
1348                        continue;
1349                    }
1350                    spawn_remote_actor(
1351                        session_id,
1352                        view,
1353                        &requests_tx,
1354                        &mut actors,
1355                        &updates_tx,
1356                    );
1357                }
1358            }
1359        }
1360        for actor in actors.into_values() {
1361            actor.abort.abort();
1362        }
1363    });
1364    Ok(RemoteSessionManagerChannels {
1365        targets: targets_tx,
1366        control: SessionManagerControl {
1367            commands: commands_tx,
1368        },
1369        updates: updates_rx,
1370        shutdown: SessionManagerShutdown {
1371            signal: Some(shutdown_tx),
1372            task: Some(task),
1373        },
1374        publisher: RemoteSessionPublisher {
1375            updates: published_tx,
1376        },
1377        requests: RemoteSessionRequests {
1378            requests: requests_rx,
1379        },
1380    })
1381}
1382
1383pub fn spawn_session_manager() -> Result<SessionManagerChannels> {
1384    let (targets_tx, mut targets_rx) = watch::channel(Vec::<RelaySessionTarget>::new());
1385    let (commands_tx, mut commands_rx) = mpsc::channel(32);
1386    let (updates_tx, updates_rx) = coalesced_update_channel();
1387    let (shutdown_tx, mut shutdown_rx) = oneshot::channel();
1388    let task = tokio::spawn(async move {
1389        let mut actors = BTreeMap::<String, ActorRegistration>::new();
1390        let mut tasks = tokio::task::JoinSet::<String>::new();
1391        let mut desired_targets = BTreeMap::<String, RelaySessionTarget>::new();
1392        loop {
1393            tokio::select! {
1394                _ = &mut shutdown_rx => break,
1395                changed = targets_rx.changed() => {
1396                    if changed.is_err() {
1397                        break;
1398                    }
1399                    desired_targets = target_map(&targets_rx.borrow_and_update());
1400                    reconcile_actors(
1401                        &desired_targets,
1402                        &mut actors,
1403                        &mut tasks,
1404                        &updates_tx,
1405                    );
1406                }
1407                command = commands_rx.recv() => {
1408                    let Some(ManagerCommand::Session { session_id, reply }) = command else {
1409                        break;
1410                    };
1411                    let handle = actors
1412                        .get(&session_id)
1413                        .filter(|actor| !actor.commands.is_closed())
1414                        .filter(|actor| desired_targets.get(&session_id) == Some(&actor.target))
1415                        .map(|actor| ManagedSessionHandle {
1416                            session_id: session_id.clone(),
1417                            commands: actor.commands.clone(),
1418                            releases: actor.releases.clone(),
1419                            view: actor.view.clone(),
1420                        });
1421                    if reply.send(handle).is_err() {
1422                        tracing::debug!(
1423                            session_id = %session_id,
1424                            operation = "session_lookup",
1425                            "session lookup receiver was already closed"
1426                        );
1427                    }
1428                }
1429                joined = tasks.join_next_with_id(), if !tasks.is_empty() => {
1430                    match joined {
1431                        Some(Ok((task_id, session_id))) => {
1432                            let removed = remove_actor_task(&mut actors, task_id);
1433                            if removed.as_deref().is_some_and(|removed| removed != session_id) {
1434                                tracing::error!(
1435                                    completed_session_id = session_id,
1436                                    registered_session_id = removed,
1437                                    "session relay actor completed under the wrong registration"
1438                                );
1439                            }
1440                            // A watch sender may have published another target while this
1441                            // completion was already ready. Reconcile against its newest
1442                            // value so an intermediate replacement is never started.
1443                            desired_targets = target_map(&targets_rx.borrow());
1444                            reconcile_actors(
1445                                &desired_targets,
1446                                &mut actors,
1447                                &mut tasks,
1448                                &updates_tx,
1449                            );
1450                        }
1451                        Some(Err(error)) if error.is_cancelled() => {
1452                            let cancelled_task = error.id();
1453                            let session_id = remove_actor_task(&mut actors, cancelled_task);
1454                            desired_targets = target_map(&targets_rx.borrow());
1455                            reconcile_actors(
1456                                &desired_targets,
1457                                &mut actors,
1458                                &mut tasks,
1459                                &updates_tx,
1460                            );
1461                            tracing::warn!(
1462                                session_id = ?session_id,
1463                                "cancelled session relay actor was replaced"
1464                            );
1465                        }
1466                        Some(Err(error)) => {
1467                            let failed_task = error.id();
1468                            remove_actor_task(&mut actors, failed_task);
1469                            desired_targets = target_map(&targets_rx.borrow());
1470                            reconcile_actors(
1471                                &desired_targets,
1472                                &mut actors,
1473                                &mut tasks,
1474                                &updates_tx,
1475                            );
1476                            tracing::error!(%error, "session relay actor failed");
1477                        }
1478                        None => {}
1479                    }
1480                }
1481            }
1482        }
1483        shutdown_session_actors(&mut actors, &mut tasks).await;
1484    });
1485    Ok(SessionManagerChannels {
1486        targets: targets_tx,
1487        control: SessionManagerControl {
1488            commands: commands_tx,
1489        },
1490        updates: updates_rx,
1491        shutdown: SessionManagerShutdown {
1492            signal: Some(shutdown_tx),
1493            task: Some(task),
1494        },
1495    })
1496}
1497
1498async fn shutdown_session_actors(
1499    actors: &mut BTreeMap<String, ActorRegistration>,
1500    tasks: &mut tokio::task::JoinSet<String>,
1501) {
1502    for actor in actors.values() {
1503        actor.retirement.send_replace(true);
1504    }
1505    actors.clear();
1506
1507    let graceful = async {
1508        while let Some(joined) = tasks.join_next().await {
1509            match joined {
1510                Ok(_) => {}
1511                Err(error) if error.is_cancelled() => {}
1512                Err(error) => {
1513                    tracing::error!(%error, "session relay actor failed during shutdown");
1514                }
1515            }
1516        }
1517    };
1518    if tokio::time::timeout(SESSION_MANAGER_SHUTDOWN_GRACE, graceful)
1519        .await
1520        .is_ok()
1521    {
1522        return;
1523    }
1524
1525    tracing::warn!(
1526        timeout_ms = SESSION_MANAGER_SHUTDOWN_GRACE.as_millis(),
1527        "session relay actors did not stop before the shutdown deadline; aborting them"
1528    );
1529    tasks.abort_all();
1530    while let Some(joined) = tasks.join_next().await {
1531        if let Err(error) = joined
1532            && !error.is_cancelled()
1533        {
1534            tracing::error!(%error, "session relay actor failed while being aborted");
1535        }
1536    }
1537}
1538
1539async fn run_session_actor(
1540    target: RelaySessionTarget,
1541    mut commands: mpsc::Receiver<ActorCommand>,
1542    mut releases: mpsc::UnboundedReceiver<ReturnedConnection>,
1543    mut retirement: watch::Receiver<bool>,
1544    view_tx: watch::Sender<ManagedSessionView>,
1545    updates: CoalescedUpdateSender,
1546) {
1547    let mut connection: Option<StandaloneSession> = None;
1548    let mut failures = 0_u32;
1549    let mut last_recovery_probe = None;
1550    let mut lifecycle = ActorLifecycle::default();
1551    let mut deferred_submits: VecDeque<DeferredSubmit> = VecDeque::new();
1552    let mut next_lease_id = 1_u64;
1553    let mut interval = tokio::time::interval(SESSION_SYNC_INTERVAL);
1554    interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
1555    loop {
1556        lifecycle.set_retirement_requested(*retirement.borrow_and_update());
1557        if lifecycle.should_stop() {
1558            break;
1559        }
1560        tokio::select! {
1561            _ = interval.tick() => {
1562                lifecycle.set_retirement_requested(*retirement.borrow());
1563                if lifecycle.should_stop() {
1564                    break;
1565                }
1566                if lifecycle.is_leased() {
1567                    continue;
1568                }
1569                let result = sync_actor_connection(
1570                    &target,
1571                    &mut connection,
1572                ).await;
1573                match result {
1574                    Ok(snapshot) => {
1575                        failures = 0;
1576                        if let Some(snapshot) = snapshot {
1577                            publish_view(&target.session_id, ManagedSessionView {
1578                                snapshot: Some(snapshot),
1579                                connected: true,
1580                                error: None,
1581                            }, &view_tx, &updates);
1582                        }
1583                    }
1584                    Err(error) => {
1585                        connection = None;
1586                        failures = failures.saturating_add(1);
1587                        // A projection integrity failure repeats on every
1588                        // retry, so report it at once rather than waiting for
1589                        // the unreachable threshold.
1590                        let integrity = projection_integrity_failure(&error);
1591                        tracing::warn!(
1592                            session_id = target.session_id,
1593                            consecutive_failures = failures,
1594                            projection_integrity = integrity,
1595                            transport_dead = worker_connect_needs_restart(&error),
1596                            "session relay sync failed: {error:#}"
1597                        );
1598                        let recovery_due = !integrity
1599                            && failures >= UNREACHABLE_FAILURE_THRESHOLD
1600                            && worker_connect_needs_restart(&error)
1601                            && target.worker_recovery.is_some()
1602                            && last_recovery_probe.is_none_or(|last: tokio::time::Instant| {
1603                                last.elapsed() >= WORKER_RESTART_COOLDOWN
1604                            });
1605                        if integrity || failures >= UNREACHABLE_FAILURE_THRESHOLD {
1606                            // Bind the clone first: borrowing inside the call
1607                            // would hold the watch read guard while
1608                            // `publish_view` takes the write lock, deadlocking
1609                            // this actor on its own view.
1610                            let snapshot = view_tx.borrow().snapshot.clone();
1611                            let mut detail = format!("{error:#}");
1612                            if recovery_due {
1613                                detail.push_str("; checking whether the relay worker is dead");
1614                            }
1615                            publish_view(&target.session_id, ManagedSessionView {
1616                                snapshot,
1617                                connected: false,
1618                                error: Some(if integrity {
1619                                    ViewError::ProjectionIntegrity(detail)
1620                                } else {
1621                                    ViewError::Unreachable(detail)
1622                                }),
1623                            }, &view_tx, &updates);
1624                        }
1625                        if recovery_due {
1626                            last_recovery_probe = Some(tokio::time::Instant::now());
1627                            let plan = target
1628                                .worker_recovery
1629                                .clone()
1630                                .expect("recovery eligibility requires a plan");
1631                            let restart_unresponsive =
1632                                worker_connect_allows_live_restart(&error);
1633                            tracing::warn!(
1634                                session_id = target.session_id,
1635                                "relay worker is unreachable; probing it before recovery: {error:#}"
1636                            );
1637                            match recover_worker(plan, restart_unresponsive).await {
1638                                Ok(
1639                                    outcome @ (WorkerRecoveryOutcome::RestartedDead
1640                                    | WorkerRecoveryOutcome::RestartedUnresponsive),
1641                                ) => {
1642                                    failures = 0;
1643                                    let snapshot = view_tx.borrow().snapshot.clone();
1644                                    let recovery = match outcome {
1645                                        WorkerRecoveryOutcome::RestartedDead => {
1646                                            "confirmed the relay worker was dead and restarted it"
1647                                        }
1648                                        WorkerRecoveryOutcome::RestartedUnresponsive => {
1649                                            "the relay worker was alive but not serving handshakes, so it was restarted"
1650                                        }
1651                                        WorkerRecoveryOutcome::Alive
1652                                        | WorkerRecoveryOutcome::Starting
1653                                        | WorkerRecoveryOutcome::TargetMissing => unreachable!(),
1654                                    };
1655                                    publish_view(&target.session_id, ManagedSessionView {
1656                                        snapshot,
1657                                        connected: false,
1658                                        error: Some(ViewError::Unreachable(format!(
1659                                            "{error:#}; {recovery}"
1660                                        ))),
1661                                    }, &view_tx, &updates);
1662                                    interval.reset_after(RECONNECT_INTERVAL);
1663                                }
1664                                Ok(WorkerRecoveryOutcome::Alive) => {
1665                                    tracing::warn!(
1666                                        session_id = target.session_id,
1667                                        "relay transport failed but the worker is alive; leaving it running"
1668                                    );
1669                                    let snapshot = view_tx.borrow().snapshot.clone();
1670                                    publish_view(&target.session_id, ManagedSessionView {
1671                                        snapshot,
1672                                        connected: false,
1673                                        error: Some(ViewError::Unreachable(format!(
1674                                            "{error:#}; relay worker is still alive, so it was not restarted"
1675                                        ))),
1676                                    }, &view_tx, &updates);
1677                                    interval.reset_after(reconnect_delay(failures));
1678                                }
1679                                Ok(WorkerRecoveryOutcome::Starting) => {
1680                                    tracing::warn!(
1681                                        session_id = target.session_id,
1682                                        "relay worker is still starting; leaving it running"
1683                                    );
1684                                    let snapshot = view_tx.borrow().snapshot.clone();
1685                                    publish_view(&target.session_id, ManagedSessionView {
1686                                        snapshot,
1687                                        connected: false,
1688                                        error: Some(ViewError::Unreachable(format!(
1689                                            "{error:#}; relay worker is still recovering its durable state, so it was not restarted"
1690                                        ))),
1691                                    }, &view_tx, &updates);
1692                                    interval.reset_after(reconnect_delay(failures));
1693                                }
1694                                Ok(WorkerRecoveryOutcome::TargetMissing) => {
1695                                    let snapshot = view_tx.borrow().snapshot.clone();
1696                                    publish_view(&target.session_id, ManagedSessionView {
1697                                        snapshot,
1698                                        connected: false,
1699                                        error: Some(ViewError::TargetMissing(
1700                                            "the managed Podman session container no longer exists"
1701                                                .into(),
1702                                        )),
1703                                    }, &view_tx, &updates);
1704                                    interval.reset_after(RECONNECT_BACKOFF_CEILING);
1705                                }
1706                                Err(recovery_error) => {
1707                                    tracing::warn!(
1708                                        session_id = target.session_id,
1709                                        "automatic relay worker recovery failed safely: {recovery_error:#}"
1710                                    );
1711                                    let snapshot = view_tx.borrow().snapshot.clone();
1712                                    publish_view(&target.session_id, ManagedSessionView {
1713                                        snapshot,
1714                                        connected: false,
1715                                        error: Some(ViewError::Unreachable(format!(
1716                                            "{error:#}; could not confirm the relay worker was dead, so it was not restarted: {recovery_error:#}"
1717                                        ))),
1718                                    }, &view_tx, &updates);
1719                                    interval.reset_after(reconnect_delay(failures));
1720                                }
1721                            }
1722                        } else {
1723                            interval.reset_after(reconnect_delay(failures));
1724                        }
1725                    }
1726                }
1727            }
1728            command = commands.recv() => {
1729                let Some(command) = command else { break };
1730                lifecycle.set_retirement_requested(*retirement.borrow());
1731                if !lifecycle.accepts_new_work() {
1732                    tracing::debug!(
1733                        session_id = %target.session_id,
1734                        operation = command.operation_name(),
1735                        "rejecting relay operation while session target changes"
1736                    );
1737                    command.reject(&target.session_id, "session target is changing");
1738                    continue;
1739                }
1740                match command {
1741                    ActorCommand::Submit { command_id, command, reply } => {
1742                        // A turn under review holds its session's prompts, and
1743                        // this is where that is enforced: the process that owns
1744                        // the review owns the refusal, so no surface can bypass
1745                        // it and no stale record can outlive it. The review's
1746                        // own corrective prompt is submitted after the review
1747                        // resolves, so it is never the one refused.
1748                        if matches!(command, RelayCommand::Prompt { .. })
1749                            && let Some(refusal) =
1750                                crate::hel_review_host::prompt_refusal(&target.session_id)
1751                        {
1752                            tracing::debug!(
1753                                session_id = %target.session_id,
1754                                %command_id,
1755                                "refusing a prompt while a turn review is unresolved"
1756                            );
1757                            let _ = reply.send(Err(refusal.to_owned()));
1758                            continue;
1759                        }
1760                        if lifecycle.is_leased() {
1761                            // A checkpoint or other lifecycle operation owns the
1762                            // connection. Hold the prompt instead of rejecting it
1763                            // and deliver it when the lease comes back.
1764                            deferred_submits.push_back(DeferredSubmit {
1765                                command_id,
1766                                command,
1767                                reply,
1768                            });
1769                            continue;
1770                        }
1771                        deliver_submit(
1772                            &target,
1773                            &mut connection,
1774                            command_id,
1775                            command,
1776                            reply,
1777                            &view_tx,
1778                            &updates,
1779                        )
1780                        .await;
1781                    }
1782                    ActorCommand::Sync { reply } => {
1783                        if lifecycle.is_leased() {
1784                            tracing::debug!(
1785                                session_id = %target.session_id,
1786                                operation = "sync",
1787                                "rejecting sync while session is leased"
1788                            );
1789                            if reply
1790                                .send(Err("session is reserved for a lifecycle operation".into()))
1791                                .is_err()
1792                            {
1793                                tracing::debug!(
1794                                    session_id = %target.session_id,
1795                                    operation = "sync",
1796                                    "sync rejection receiver was already closed"
1797                                );
1798                            }
1799                            continue;
1800                        }
1801                        let result = sync_actor_connection(
1802                            &target,
1803                            &mut connection,
1804                        ).await.map(|snapshot| {
1805                            if let Some(snapshot) = snapshot {
1806                                publish_view(&target.session_id, ManagedSessionView {
1807                                    snapshot: Some(snapshot),
1808                                    connected: true,
1809                                    error: None,
1810                                }, &view_tx, &updates);
1811                            }
1812                        });
1813                        if result.is_err() {
1814                            connection = None;
1815                        }
1816                        if let Err(error) = &result {
1817                            tracing::warn!(
1818                                session_id = %target.session_id,
1819                                operation = "sync",
1820                                error = %error,
1821                                "explicit relay synchronization failed"
1822                            );
1823                        }
1824                    if reply.send(result.map_err(|error| format!("{error:#}"))).is_err() {
1825                        tracing::debug!(
1826                            session_id = %target.session_id,
1827                            operation = "sync",
1828                            "sync result receiver was already closed"
1829                        );
1830                    }
1831                    }
1832                    ActorCommand::Reviewer {
1833                        role,
1834                        action,
1835                        reply,
1836                    } => {
1837                        if lifecycle.is_leased() {
1838                            // A lifecycle operation owns the connection, and a
1839                            // reviewer action is not worth deferring: the user
1840                            // is waiting on its answer now.
1841                            tracing::debug!(
1842                                session_id = %target.session_id,
1843                                operation = action.operation_name(),
1844                                "rejecting a reviewer action while the session is leased"
1845                            );
1846                            if reply
1847                                .send(Err("session is reserved for a lifecycle operation".into()))
1848                                .is_err()
1849                            {
1850                                tracing::debug!(
1851                                    session_id = %target.session_id,
1852                                    operation = "reviewer",
1853                                    "reviewer rejection receiver was already closed"
1854                                );
1855                            }
1856                            continue;
1857                        }
1858                        let operation = action.operation_name();
1859                        let result = async {
1860                            sync_actor_connection(&target, &mut connection).await?;
1861                            let connection =
1862                                connection.as_mut().context("relay is disconnected")?;
1863                            drive_reviewer(connection, role, action).await
1864                        }
1865                        .await;
1866                        match &result {
1867                            Ok(_) => {}
1868                            Err(error) if !is_final_rejection(error) => connection = None,
1869                            Err(_) => {}
1870                        }
1871                        if let Err(error) = &result {
1872                            tracing::warn!(
1873                                session_id = %target.session_id,
1874                                %operation,
1875                                error = %error,
1876                                "reviewer action failed"
1877                            );
1878                        }
1879                        if reply
1880                            .send(result.map_err(|error| format!("{error:#}")))
1881                            .is_err()
1882                        {
1883                            tracing::debug!(
1884                                session_id = %target.session_id,
1885                                %operation,
1886                                "reviewer result receiver was already closed"
1887                            );
1888                        }
1889                    }
1890                    ActorCommand::RespondElicitation {
1891                        elicitation_id,
1892                        response,
1893                        reply,
1894                    } => {
1895                        if lifecycle.is_leased() {
1896                            tracing::debug!(
1897                                session_id = %target.session_id,
1898                                operation = "respond_elicitation",
1899                                "rejecting elicitation response while session is leased"
1900                            );
1901                            if reply
1902                                .send(Err("session is reserved for a lifecycle operation".into()))
1903                                .is_err()
1904                            {
1905                                tracing::debug!(
1906                                    session_id = %target.session_id,
1907                                    operation = "respond_elicitation",
1908                                    "elicitation rejection receiver was already closed"
1909                                );
1910                            }
1911                            continue;
1912                        }
1913                        let result = async {
1914                            sync_actor_connection(&target, &mut connection).await?;
1915                            let connection = connection
1916                                .as_mut()
1917                                .context("relay is disconnected")?;
1918                            connection
1919                                .respond_elicitation(elicitation_id, response)
1920                                .await?;
1921                            Ok::<_, anyhow::Error>(connection.snapshot())
1922                        }
1923                        .await;
1924                        match result {
1925                            Ok(ref snapshot) => publish_view(
1926                                &target.session_id,
1927                                ManagedSessionView {
1928                                    snapshot: Some(snapshot.clone()),
1929                                    connected: true,
1930                                    error: None,
1931                                },
1932                                &view_tx,
1933                                &updates,
1934                            ),
1935                            Err(ref error) if !is_final_rejection(error) => connection = None,
1936                            Err(_) => {}
1937                        }
1938                        if let Err(error) = &result {
1939                            tracing::warn!(
1940                                session_id = %target.session_id,
1941                                operation = "respond_elicitation",
1942                                error = %error,
1943                                "relay elicitation response failed"
1944                            );
1945                        }
1946                        if reply
1947                            .send(result.map(|_| ()).map_err(|error| format!("{error:#}")))
1948                            .is_err()
1949                        {
1950                            tracing::debug!(
1951                                session_id = %target.session_id,
1952                                operation = "respond_elicitation",
1953                                "elicitation result receiver was already closed"
1954                            );
1955                        }
1956                    }
1957                    ActorCommand::Lease { reply } => {
1958                        if lifecycle.is_leased() {
1959                            tracing::debug!(
1960                                session_id = %target.session_id,
1961                                operation = "lease",
1962                                "rejecting duplicate session lifecycle lease"
1963                            );
1964                            if reply
1965                                .send(Err(anyhow::anyhow!(
1966                                    "session already has a lifecycle operation"
1967                                )))
1968                                .is_err()
1969                            {
1970                                tracing::debug!(
1971                                    session_id = %target.session_id,
1972                                    operation = "lease",
1973                                    "lease rejection receiver was already closed"
1974                                );
1975                            }
1976                            continue;
1977                        }
1978                        let lease_id = next_lease_id;
1979                        let result = sync_actor_connection(
1980                            &target,
1981                            &mut connection,
1982                        )
1983                        .await
1984                        .map(|_| {
1985                            next_lease_id = next_lease_id.wrapping_add(1).max(1);
1986                            (
1987                                lease_id,
1988                                connection
1989                                    .take()
1990                                    .expect("successful sync retained its connection"),
1991                            )
1992                        });
1993                        if result.is_err() {
1994                            connection = None;
1995                        }
1996                        if let Err(error) = &result {
1997                            tracing::warn!(
1998                                session_id = %target.session_id,
1999                                operation = "lease",
2000                                error = %error,
2001                                "could not acquire relay session lease"
2002                            );
2003                        }
2004                        let acquired = result.is_ok();
2005                        match reply.send(result) {
2006                            Ok(()) if acquired => lifecycle.activate_lease(lease_id),
2007                            Ok(()) => {}
2008                            Err(Ok((_lease_id, returned))) => connection = Some(returned),
2009                            Err(Err(_)) => {}
2010                        }
2011                    }
2012                }
2013            }
2014            returned = releases.recv() => {
2015                let Some(returned) = returned else { continue };
2016                if lifecycle.return_lease(returned.lease_id) {
2017                    // A dropped lease returns no connection; `submit_actor_command`
2018                    // reconnects on demand, so the drain needs no special case.
2019                    connection = returned.connection;
2020                    failures = 0;
2021                    interval.reset();
2022                    // A lease syncs the connection it borrowed, so this actor's
2023                    // next sync can find nothing left to apply. Publish what the
2024                    // returned connection already knows or watchers keep reading
2025                    // pre-lease state.
2026                    if let Some(returned) = connection.as_ref() {
2027                        publish_view(&target.session_id, ManagedSessionView {
2028                            snapshot: Some(returned.snapshot()),
2029                            connected: true,
2030                            error: None,
2031                        }, &view_tx, &updates);
2032                    }
2033                    let retiring = *retirement.borrow();
2034                    while let Some(deferred) = deferred_submits.pop_front() {
2035                        if retiring {
2036                            if deferred
2037                                .reply
2038                                .send(Err("session target is changing".into()))
2039                                .is_err()
2040                            {
2041                                tracing::debug!(
2042                                    session_id = %target.session_id,
2043                                    operation = "submit",
2044                                    "deferred submit rejection receiver was already closed"
2045                                );
2046                            }
2047                            continue;
2048                        }
2049                        deliver_submit(
2050                            &target,
2051                            &mut connection,
2052                            deferred.command_id,
2053                            deferred.command,
2054                            deferred.reply,
2055                            &view_tx,
2056                            &updates,
2057                        )
2058                        .await;
2059                    }
2060                }
2061            }
2062            changed = retirement.changed() => {
2063                if changed.is_err() {
2064                    break;
2065                }
2066            }
2067        }
2068    }
2069    if let Some(connection) = connection.take()
2070        && let Err(error) = connection.detach().await
2071    {
2072        tracing::warn!(
2073            session_id = %target.session_id,
2074            %error,
2075            "could not detach relay connection during session actor shutdown"
2076        );
2077    }
2078    // No caller may wait forever on a submission this actor will never deliver.
2079    for deferred in deferred_submits {
2080        if deferred
2081            .reply
2082            .send(Err("session manager stopped".into()))
2083            .is_err()
2084        {
2085            tracing::debug!(
2086                session_id = %target.session_id,
2087                operation = "submit",
2088                "deferred submit shutdown receiver was already closed"
2089            );
2090        }
2091    }
2092}
2093
2094/// Submit one relay command and publish the resulting snapshot. Live and
2095/// deferred submissions share this path so both report identical results.
2096async fn deliver_submit(
2097    target: &RelaySessionTarget,
2098    connection: &mut Option<StandaloneSession>,
2099    command_id: String,
2100    command: RelayCommand,
2101    reply: oneshot::Sender<std::result::Result<u64, String>>,
2102    view_tx: &watch::Sender<ManagedSessionView>,
2103    updates: &CoalescedUpdateSender,
2104) {
2105    let result = submit_actor_command(target, connection, &command_id, &command).await;
2106    if let Err(error) = result.as_ref() {
2107        tracing::warn!(
2108            session_id = %target.session_id,
2109            operation = "submit",
2110            %command_id,
2111            retryable = !is_final_rejection(error),
2112            error = %error,
2113            "relay command submission failed"
2114        );
2115    }
2116    if let Err(error) = result.as_ref()
2117        && !is_final_rejection(error)
2118    {
2119        *connection = None;
2120    }
2121    let accepted = result.as_ref().ok().copied();
2122    // Answer the caller the moment the relay has the command. Catching the
2123    // local projection up to it is the expensive half and nobody waiting to
2124    // hear "accepted" needs it first: the caller has an ordinal, and the view
2125    // it would read is published below anyway.
2126    if reply
2127        .send(result.map_err(|error| format!("{error:#}")))
2128        .is_err()
2129    {
2130        tracing::debug!(
2131            session_id = %target.session_id,
2132            operation = "submit",
2133            %command_id,
2134            "submit result receiver was already closed"
2135        );
2136    }
2137    let Some(ordinal) = accepted else {
2138        return;
2139    };
2140    tracing::trace!(%ordinal, %command_id, "relay command accepted");
2141    let Some(session) = connection.as_mut() else {
2142        return;
2143    };
2144    // The command landed either way, so a failed catch-up is a connection
2145    // problem to retire rather than a failed submission: the caller has
2146    // already been told the relay took it.
2147    match session.sync().await {
2148        Ok(snapshot) => publish_view(
2149            &target.session_id,
2150            ManagedSessionView {
2151                snapshot: Some(snapshot),
2152                connected: true,
2153                error: None,
2154            },
2155            view_tx,
2156            updates,
2157        ),
2158        Err(error) => {
2159            tracing::warn!(
2160                session_id = %target.session_id,
2161                operation = "submit",
2162                %command_id,
2163                error = %format!("{error:#}"),
2164                "projection could not catch up to an accepted command"
2165            );
2166            *connection = None;
2167        }
2168    }
2169}
2170
2171/// Whether the relay refused this request outright.
2172///
2173/// A refusal is a completed round trip, so the connection is healthy. Dropping
2174/// it would discard whatever that connection owns on the worker, including a
2175/// checkpoint barrier a controller is still holding.
2176fn is_final_rejection(error: &anyhow::Error) -> bool {
2177    error
2178        .downcast_ref::<RelayRejected>()
2179        .is_some_and(|rejected| !rejected.is_retryable())
2180}
2181
2182async fn submit_actor_command(
2183    target: &RelaySessionTarget,
2184    connection: &mut Option<StandaloneSession>,
2185    command_id: &str,
2186    command: &RelayCommand,
2187) -> Result<u64> {
2188    let mut first_error = None;
2189    for attempt in 1..=2 {
2190        if connection.is_none() {
2191            sync_actor_connection(target, connection).await?;
2192        }
2193        let result = connection
2194            .as_mut()
2195            .context("relay is disconnected")?
2196            .submit_accepted(command_id.to_owned(), command.clone())
2197            .await;
2198        match result {
2199            Ok(ordinal) => return Ok(ordinal),
2200            // A final rejection is a completed round trip: the relay read the
2201            // command and refused it, so retrying would only be refused again.
2202            // Reconnecting would also cancel any checkpoint barrier this
2203            // connection owns, which is how a controller probing for a command
2204            // an older worker does not understand would lose it.
2205            Err(error) if is_final_rejection(&error) => return Err(error),
2206            Err(error) => {
2207                tracing::warn!(
2208                    session_id = %target.session_id,
2209                    operation = "submit",
2210                    %command_id,
2211                    attempt,
2212                    retryable = true,
2213                    error = %error,
2214                    "retryable relay command failure; reconnecting"
2215                );
2216                if first_error.is_none() {
2217                    first_error = Some(format!("{error:#}"));
2218                }
2219                *connection = None;
2220            }
2221        }
2222    }
2223    let detail = first_error.unwrap_or_else(|| "relay submission failed".into());
2224    bail!("relay command {command_id} failed after an idempotent reconnect: {detail}")
2225}
2226
2227/// Perform one reviewer action on a synchronized relay connection.
2228///
2229/// The reviewer's own relay answers most of these, so the outcomes mirror the
2230/// primary's: an attach page, an acknowledgement cursor, an accepted command.
2231async fn drive_reviewer(
2232    connection: &mut StandaloneSession,
2233    role: Option<String>,
2234    action: ReviewerAction,
2235) -> Result<ReviewerOutcome> {
2236    let client = &mut connection.client;
2237    let role = role.as_deref();
2238    Ok(match action {
2239        ReviewerAction::Start { config } => {
2240            ReviewerOutcome::Started(Box::new(client.start_reviewer(role, *config).await?))
2241        }
2242        ReviewerAction::Submit {
2243            command_id,
2244            command,
2245        } => ReviewerOutcome::Accepted {
2246            ordinal: client.submit_to_reviewer(role, command_id, command).await?,
2247        },
2248        ReviewerAction::Attach {
2249            after_ordinal,
2250            after_digest,
2251        } => ReviewerOutcome::Attached(Box::new(
2252            client
2253                .attach_reviewer(role, after_ordinal, after_digest)
2254                .await?,
2255        )),
2256        ReviewerAction::Acknowledge {
2257            through_ordinal,
2258            through_digest,
2259        } => ReviewerOutcome::Acknowledged(
2260            client
2261                .acknowledge_reviewer(role, through_ordinal, through_digest)
2262                .await?,
2263        ),
2264        ReviewerAction::Status => {
2265            ReviewerOutcome::Status(Box::new(client.reviewer_status(role).await?))
2266        }
2267        ReviewerAction::RespondElicitation {
2268            elicitation_id,
2269            response,
2270        } => {
2271            client
2272                .respond_to_reviewer(role, elicitation_id, response)
2273                .await?;
2274            ReviewerOutcome::ElicitationResolved
2275        }
2276        ReviewerAction::Pause => {
2277            client.pause_reviewer(role).await?;
2278            ReviewerOutcome::Paused
2279        }
2280        ReviewerAction::CaptureDelta { baselines } => ReviewerOutcome::Delta {
2281            repositories: client.capture_review_delta(role, baselines).await?,
2282        },
2283        ReviewerAction::AdvanceBaseline { trees } => {
2284            client.advance_review_baseline(role, trees).await?;
2285            ReviewerOutcome::BaselineAdvanced
2286        }
2287        ReviewerAction::AnalyzeDelta { repositories } => ReviewerOutcome::ChangedFunctions {
2288            packet: client.analyze_review_delta(role, repositories).await?,
2289        },
2290        ReviewerAction::TakeLaneDispatches => ReviewerOutcome::LaneDispatches {
2291            requests: client.take_lane_dispatches().await?,
2292        },
2293    })
2294}
2295
2296async fn sync_actor_connection(
2297    target: &RelaySessionTarget,
2298    connection: &mut Option<StandaloneSession>,
2299) -> Result<Option<ManagedSessionSnapshot>> {
2300    if connection.is_none() {
2301        *connection = Some(StandaloneSession::connect(target).await?);
2302        return Ok(Some(
2303            connection
2304                .as_ref()
2305                .expect("connection was initialized")
2306                .snapshot(),
2307        ));
2308    }
2309    let connection = connection.as_mut().expect("connection was initialized");
2310    if connection.sync_in_place().await? {
2311        Ok(Some(connection.snapshot()))
2312    } else {
2313        Ok(None)
2314    }
2315}
2316
2317/// Cheap equivalence for published views.
2318///
2319/// The materialized projection is a function of the relay event chain, so its
2320/// transcript can only differ when the applied event frontier differs. Every
2321/// sync tick would otherwise walk the whole conversation to prove nothing
2322/// changed. The remaining scalars are compared directly because they are small
2323/// and bound the projection's non-transcript state.
2324fn view_is_unchanged(current: &ManagedSessionView, next: &ManagedSessionView) -> bool {
2325    if current.connected != next.connected || current.error != next.error {
2326        return false;
2327    }
2328    match (&current.snapshot, &next.snapshot) {
2329        (None, None) => true,
2330        (Some(current), Some(next)) => {
2331            let (current_session, next_session) = (&current.materialized, &next.materialized);
2332            current.latest_credential_sync_signal == next.latest_credential_sync_signal
2333                && current.operational == next.operational
2334                && current_session.session_id == next_session.session_id
2335                && current_session.applied_event_ordinal == next_session.applied_event_ordinal
2336                && current_session.applied_event_digest == next_session.applied_event_digest
2337                && current_session.last_activity_at_ms == next_session.last_activity_at_ms
2338                && current_session.execution == next_session.execution
2339                && current_session.session_title == next_session.session_title
2340                && current_session.queued_prompts == next_session.queued_prompts
2341        }
2342        (None, Some(_)) | (Some(_), None) => false,
2343    }
2344}
2345
2346fn publish_view(
2347    session_id: &str,
2348    view: ManagedSessionView,
2349    watch: &watch::Sender<ManagedSessionView>,
2350    updates: &CoalescedUpdateSender,
2351) {
2352    // Compare and replace under one lock acquisition; a separate
2353    // `watch.borrow()` check would reacquire the lock and invite the
2354    // read-then-write deadlock this function's callers must avoid.
2355    let changed = watch.send_if_modified(|current| {
2356        if view_is_unchanged(current, &view) {
2357            return false;
2358        }
2359        *current = view.clone();
2360        true
2361    });
2362    if changed {
2363        updates.send(SessionManagerUpdate {
2364            session_id: session_id.to_owned(),
2365            view,
2366        });
2367    }
2368}
2369
2370/// Read a stored projection without blocking the runtime. The rusqlite read
2371/// and the transcript deserialization behind it are synchronous and grow with
2372/// the conversation, so a long session must not stall a worker thread that
2373/// other actors share.
2374async fn load_projection(session_id: &str) -> Result<MaterializedSession> {
2375    let session_id = session_id.to_owned();
2376    tokio::task::spawn_blocking(move || -> Result<MaterializedSession> {
2377        let loaded = hel::hel_database::load_materialized_session(&session_id)?;
2378        Ok(loaded.unwrap_or_else(|| MaterializedSession::empty(session_id)))
2379    })
2380    .await
2381    .context("controller projection load task failed")?
2382}
2383
2384pub struct StandaloneSession {
2385    client: RelayClient,
2386    materialized: MaterializedSession,
2387    operational: RelayOperationalState,
2388    latest_credential_sync_signal: Option<CredentialSyncSignal>,
2389    project_memory: Option<ProjectMemorySyncTarget>,
2390}
2391
2392impl StandaloneSession {
2393    pub fn set_project_memory_target(&mut self, target: Option<ProjectMemorySyncTarget>) {
2394        self.project_memory = target;
2395    }
2396
2397    pub async fn connect(target: &RelaySessionTarget) -> Result<Self> {
2398        // Reach the worker before reading the projection. A stored session can
2399        // be tens of megabytes, and the reconnect loop would otherwise pay that
2400        // whole synchronous read on every attempt against a worker that is down.
2401        let mut client = RelayClient::connect(&target.spec, &target.session_id).await?;
2402        let operational = client.status().await?;
2403        let materialized = load_projection(&target.session_id).await?;
2404        let mut connection = Self {
2405            client,
2406            materialized,
2407            operational,
2408            latest_credential_sync_signal: None,
2409            project_memory: target.project_memory.clone(),
2410        };
2411        connection.sync_in_place().await?;
2412        Ok(connection)
2413    }
2414
2415    pub async fn connect_command(spec: &CommandSpec, session_id: &str) -> Result<Self> {
2416        Self::connect(&RelaySessionTarget {
2417            session_id: session_id.to_owned(),
2418            spec: spec.clone(),
2419            worker_recovery: None,
2420            project_memory: None,
2421        })
2422        .await
2423    }
2424
2425    async fn detach(self) -> Result<()> {
2426        self.client.detach().await
2427    }
2428
2429    pub async fn sync(&mut self) -> Result<ManagedSessionSnapshot> {
2430        self.sync_in_place().await?;
2431        Ok(self.snapshot())
2432    }
2433
2434    async fn sync_in_place(&mut self) -> Result<bool> {
2435        let original_ordinal = self.materialized.applied_event_ordinal;
2436        let original_digest = self.materialized.applied_event_digest.clone();
2437        let original_operational = self.operational.clone();
2438        let mut repaired = false;
2439        let mut repaired_frontiers = std::collections::HashSet::new();
2440        loop {
2441            let after_ordinal = self.materialized.applied_event_ordinal;
2442            match self.catch_up_fixed_frontier().await {
2443                Ok(()) => break,
2444                Err(error) if error.downcast_ref::<ProjectionAdvancedError>().is_some() => {
2445                    let durable = load_projection(&self.materialized.session_id).await?;
2446                    if durable.applied_event_ordinal <= after_ordinal {
2447                        return Err(error);
2448                    }
2449                    self.materialized = durable;
2450                    continue;
2451                }
2452                Err(error) if relay_desynchronized(&error) => {
2453                    self.repair_projection()
2454                        .await
2455                        .with_context(|| {
2456                            format!(
2457                                "controller projection for {} cannot catch up from ordinal {after_ordinal}: {error:#}",
2458                                self.materialized.session_id
2459                            )
2460                        })?;
2461                    repaired = true;
2462                    // Repair rebuilds from the same durable checkpoint every
2463                    // time. If catching up from that frontier still desyncs — as
2464                    // it does when relay history is unreadable past the
2465                    // checkpoint — repairing again lands on the same frontier and
2466                    // would loop forever. Fail loudly on the second visit instead
2467                    // of hanging; recovery got everything the checkpoint covers.
2468                    let frontier = self.materialized.applied_event_ordinal;
2469                    if !repaired_frontiers.insert(frontier) {
2470                        bail!(
2471                            "controller projection for {} cannot catch up: relay history is \
2472                             unreadable and rebuilding from checkpoint frontier {frontier} does \
2473                             not get past it",
2474                            self.materialized.session_id
2475                        );
2476                    }
2477                    continue;
2478                }
2479                Err(error) => return Err(error),
2480            }
2481        }
2482        let changed = repaired
2483            || self.materialized.applied_event_ordinal != original_ordinal
2484            || self.materialized.applied_event_digest != original_digest
2485            || self.operational != original_operational;
2486        Ok(changed)
2487    }
2488
2489    /// Apply relay pages through the exact frontier captured by the first
2490    /// response, then acknowledge that frontier once. Every projection page is
2491    /// independently durable; delaying the relay's GC watermark avoids one
2492    /// snapshot fsync per transport-sized page without risking redelivery.
2493    async fn catch_up_fixed_frontier(&mut self) -> Result<()> {
2494        let after = RelayCursor {
2495            ordinal: self.materialized.applied_event_ordinal,
2496            digest: self.materialized.applied_event_digest.clone(),
2497        };
2498        let catch_up = self
2499            .client
2500            .begin_catch_up(after.ordinal, &after.digest)
2501            .await?;
2502        let mut cursor = self.apply_event_page(catch_up.first_page).await?;
2503        let mut pages_remaining = catch_up.frontier.ordinal.saturating_sub(cursor.ordinal);
2504        while cursor.ordinal < catch_up.frontier.ordinal {
2505            ensure!(
2506                pages_remaining > 0,
2507                "relay catch-up exceeded its fixed page bound"
2508            );
2509            pages_remaining -= 1;
2510            let page = self
2511                .client
2512                .next_catch_up_page(&cursor, &catch_up.frontier)
2513                .await?;
2514            cursor = self.apply_event_page(page).await?;
2515        }
2516        ensure!(
2517            cursor == catch_up.frontier,
2518            "controller projection did not reach the captured relay frontier"
2519        );
2520        if cursor.ordinal > 0 {
2521            let acknowledged = self
2522                .client
2523                .acknowledge(cursor.ordinal, &cursor.digest)
2524                .await?;
2525            ensure!(
2526                acknowledged == cursor,
2527                "relay acknowledged cursor {}:{} instead of {}:{}",
2528                acknowledged.ordinal,
2529                acknowledged.digest,
2530                cursor.ordinal,
2531                cursor.digest,
2532            );
2533        }
2534        let mut operational = catch_up.state;
2535        operational.acknowledged_through = cursor.ordinal;
2536        operational.acknowledged_digest = cursor.digest;
2537        self.operational = operational;
2538        Ok(())
2539    }
2540
2541    async fn repair_projection(&mut self) -> Result<()> {
2542        let state = hel::hel_database::load_state()?;
2543        let record = state
2544            .sessions
2545            .get(&self.materialized.session_id)
2546            .context("controller session disappeared while repairing its projection")?;
2547        let Some(checkpoint) = record.checkpoint.as_ref() else {
2548            let replacement = MaterializedSession::empty(&self.materialized.session_id);
2549            self.client
2550                .attach(
2551                    replacement.applied_event_ordinal,
2552                    &replacement.applied_event_digest,
2553                )
2554                .await
2555                .context("relay cannot rebuild the projection from its genesis")?;
2556            save_materialized_session(&replacement)?;
2557            self.materialized = replacement;
2558            return Ok(());
2559        };
2560        let checkpoint_path = checkpoint.archive_path.clone();
2561        let archive = tokio::task::spawn_blocking(move || {
2562            verify_archive_streaming(&checkpoint_path).with_context(|| {
2563                format!(
2564                    "verify projection repair checkpoint {}",
2565                    checkpoint_path.display()
2566                )
2567            })
2568        })
2569        .await
2570        .context("projection repair archive verification task failed")??;
2571        ensure!(
2572            archive.archive_sha256 == checkpoint.sha256,
2573            "projection repair checkpoint checksum does not match controller metadata"
2574        );
2575        ensure!(
2576            archive.manifest.session.id == self.materialized.session_id,
2577            "projection repair checkpoint belongs to session {}, not {}",
2578            archive.manifest.session.id,
2579            self.materialized.session_id
2580        );
2581        let canonical = archive.canonical_session;
2582        ensure!(
2583            canonical.event_frontier == checkpoint.event_frontier,
2584            "projection repair checkpoint metadata frontier {} does not match archive frontier {}",
2585            checkpoint.event_frontier,
2586            canonical.event_frontier
2587        );
2588
2589        // Prove that the relay recognizes this exact event-chain cursor before
2590        // replacing any controller state. A matching ordinal alone is not a
2591        // repair proof.
2592        self.client
2593            .attach(canonical.event_frontier, &canonical.event_frontier_digest)
2594            .await
2595            .context("relay rejected the verified checkpoint repair cursor")?;
2596        let replacement =
2597            materialized_session_from_canonical(&self.materialized.session_id, &canonical)?;
2598        save_materialized_session(&replacement)?;
2599        self.materialized = replacement;
2600        Ok(())
2601    }
2602
2603    pub fn snapshot(&self) -> ManagedSessionSnapshot {
2604        ManagedSessionSnapshot {
2605            window: hel::hel_state::ProjectionWindow::of(&self.materialized),
2606            materialized: self.materialized.clone(),
2607            operational: self.operational.clone(),
2608            latest_credential_sync_signal: self.latest_credential_sync_signal.clone(),
2609            worker_build: self.client.worker_build().map(str::to_owned),
2610        }
2611    }
2612
2613    /// Hands one command to the relay and returns the ordinal it accepted it
2614    /// at, without catching the local projection up to it.
2615    ///
2616    /// Callers that need the projection current call [`Self::sync`] after.
2617    /// Keeping the two apart matters on the prompt path: the catch-up is the
2618    /// expensive half, and a caller waiting to hear that the relay took the
2619    /// command should not wait for it. It also stops a failed catch-up from
2620    /// looking like a failed submission to a caller that would retry.
2621    pub async fn submit_accepted(
2622        &mut self,
2623        command_id: String,
2624        command: RelayCommand,
2625    ) -> Result<u64> {
2626        self.client.submit(command_id, command).await
2627    }
2628
2629    pub async fn submit(&mut self, command_id: String, command: RelayCommand) -> Result<u64> {
2630        let ordinal = self.submit_accepted(command_id, command).await?;
2631        self.sync_in_place().await?;
2632        Ok(ordinal)
2633    }
2634
2635    pub async fn respond_elicitation(
2636        &mut self,
2637        elicitation_id: String,
2638        response: ElicitationResponse,
2639    ) -> Result<()> {
2640        self.client
2641            .respond_elicitation(elicitation_id, response)
2642            .await?;
2643        self.sync_in_place().await?;
2644        Ok(())
2645    }
2646
2647    /// Persist relay-private context for the next real prompt. It never
2648    /// contributes an event to the canonical projection.
2649    pub async fn install_prompt_context(&mut self, text: String) -> Result<()> {
2650        self.client.install_prompt_context(text).await
2651    }
2652
2653    /// Apply one relay transport page in bounded durable chunks. A transport
2654    /// page can contain thousands of events, but SQLite has one global writer;
2655    /// regularly releasing it lets other session actors keep their views
2656    /// current. The relay GC watermark advances only after the complete page.
2657    async fn apply_event_page(&mut self, page: RelayEventPage) -> Result<RelayCursor> {
2658        let RelayEventPage {
2659            events,
2660            through_ordinal,
2661            through_digest,
2662        } = page;
2663        let event_count = events.len();
2664        let transaction_count = event_count.div_ceil(PROJECTION_TRANSACTION_EVENT_BUDGET);
2665        let started = Instant::now();
2666        for events in events.chunks(PROJECTION_TRANSACTION_EVENT_BUDGET) {
2667            let session_id = self.materialized.session_id.clone();
2668            let events = events.to_vec();
2669            let projection = self.materialized.clone();
2670            // Projection is CPU work and its durable page uses synchronous
2671            // SQLite. Keep both off the async actor runtime so independent
2672            // sessions stay responsive during each bounded catch-up chunk.
2673            let (projection, credential_sync_signal) = tokio::task::spawn_blocking(
2674                move || -> Result<(MaterializedSession, Option<CredentialSyncSignal>)> {
2675                    // The in-memory projection advances on a working copy and
2676                    // is published only once its page is durable.
2677                    let mut projection = projection;
2678                    let mut projection_index = ProjectionIndex::new(&projection);
2679                    let mut credential_sync_signal = None;
2680                    let mut prepared = Vec::with_capacity(events.len());
2681                    for event in &events {
2682                        let mutation =
2683                            project_relay_event_indexed(&projection, &projection_index, event)?
2684                                .mutation;
2685                        prepared.push((
2686                            event.ordinal,
2687                            event.previous_digest.clone(),
2688                            event.digest.clone(),
2689                            mutation.clone(),
2690                        ));
2691                        apply_committed_projection_event_indexed(
2692                            &mut projection,
2693                            &mut projection_index,
2694                            event,
2695                            mutation,
2696                        )?;
2697                        if let Some(reason) = relay_event_credential_sync_reason(event) {
2698                            credential_sync_signal = Some(CredentialSyncSignal {
2699                                ordinal: event.ordinal,
2700                                reason,
2701                            });
2702                        }
2703                    }
2704                    drop(projection_index);
2705                    apply_projection_page(&session_id, move |committed| {
2706                        for (ordinal, previous_digest, digest, mutation) in prepared {
2707                            match committed.apply(ordinal, &previous_digest, &digest, &mutation)? {
2708                                ProjectionApplyOutcome::Applied => {}
2709                                ProjectionApplyOutcome::AlreadyApplied => {
2710                                    return Err(ProjectionAdvancedError {
2711                                        event_ordinal: ordinal,
2712                                    }
2713                                    .into());
2714                                }
2715                            }
2716                        }
2717                        Ok((projection, credential_sync_signal))
2718                    })
2719                },
2720            )
2721            .await
2722            .context("relay projection page task failed")??;
2723            self.materialized = projection;
2724            if let Some(signal) = credential_sync_signal {
2725                self.latest_credential_sync_signal = Some(signal);
2726            }
2727        }
2728        if transaction_count > 1 {
2729            tracing::debug!(
2730                session_id = self.materialized.session_id,
2731                event_count,
2732                transaction_count,
2733                elapsed_ms = started.elapsed().as_millis(),
2734                "applied a large relay page in bounded projection transactions"
2735            );
2736        }
2737        let delivered_through = self.materialized.applied_event_ordinal;
2738        ensure!(
2739            delivered_through == through_ordinal,
2740            "relay page claimed frontier {} but delivered through {delivered_through}",
2741            through_ordinal
2742        );
2743        ensure!(
2744            self.materialized.applied_event_digest == through_digest,
2745            "relay page digest does not match its claimed frontier"
2746        );
2747        Ok(RelayCursor {
2748            ordinal: delivered_through,
2749            digest: self.materialized.applied_event_digest.clone(),
2750        })
2751    }
2752
2753    /// Reconcile this worker's project-memory replica at an explicit durable
2754    /// boundary. Normal relay attachment and polling must never perform this
2755    /// filesystem work: a degraded target could otherwise turn reconnects
2756    /// into an unbounded queue of timed-out snapshot writes.
2757    pub async fn sync_project_memory(&mut self) -> Result<()> {
2758        let Some(target) = self.project_memory.clone() else {
2759            return Ok(());
2760        };
2761        if !self.client.supports_project_memory_sync() {
2762            tracing::warn!(
2763                session_id = self.materialized.session_id,
2764                "worker protocol predates project-memory synchronization; preserving memory through checkpoints only"
2765            );
2766            self.project_memory = None;
2767            return Ok(());
2768        }
2769        let (baseline, replica) = match self.client.project_memory_snapshot().await {
2770            Ok(snapshot) => snapshot,
2771            Err(error)
2772                if error
2773                    .downcast_ref::<RelayRejected>()
2774                    .is_some_and(|rejected| {
2775                        rejected.0.code == hel::hel_worker::RelayErrorCode::InvalidState
2776                    }) =>
2777            {
2778                tracing::warn!(
2779                    session_id = self.materialized.session_id,
2780                    "worker has no project-memory endpoint; preserving memory through checkpoints only"
2781                );
2782                self.project_memory = None;
2783                return Ok(());
2784            }
2785            Err(error) => return Err(error),
2786        };
2787        let canonical_root = target.canonical_root;
2788        let session_id = self.materialized.session_id.clone();
2789        let (reconciliation, worker_install_needed) = tokio::task::spawn_blocking(move || {
2790            let reconciliation = hel::hel_project_memory::reconcile_into_canonical(
2791                &canonical_root,
2792                &baseline,
2793                &replica,
2794                &session_id,
2795            )?;
2796            let worker_install_needed =
2797                reconciliation.merged != baseline || reconciliation.merged != replica;
2798            Ok::<_, anyhow::Error>((reconciliation, worker_install_needed))
2799        })
2800        .await
2801        .context("project memory reconciliation task failed")??;
2802        for conflict in &reconciliation.conflicts {
2803            tracing::warn!(session_id = self.materialized.session_id, %conflict, "project memory conflict preserved");
2804        }
2805        if worker_install_needed {
2806            self.client
2807                .install_project_memory_snapshot(reconciliation.merged)
2808                .await?;
2809        }
2810        Ok(())
2811    }
2812}
2813
2814fn relay_desynchronized(error: &anyhow::Error) -> bool {
2815    error.chain().any(|cause| {
2816        cause
2817            .downcast_ref::<RelayRejected>()
2818            .is_some_and(RelayRejected::is_desynchronized)
2819    })
2820}
2821
2822fn projection_integrity_failure(error: &anyhow::Error) -> bool {
2823    error
2824        .chain()
2825        .any(|cause| cause.downcast_ref::<ProjectionIntegrityError>().is_some())
2826}
2827
2828pub fn new_command_id(prefix: &str) -> Result<String> {
2829    ensure!(!prefix.trim().is_empty(), "command ID prefix is required");
2830    let mut random = [0_u8; 16];
2831    getrandom::fill(&mut random)
2832        .map_err(|error| anyhow::anyhow!("generate command ID: {error}"))?;
2833    Ok(format!("{prefix}-{}", hex(&random)))
2834}
2835
2836fn hex(bytes: &[u8]) -> String {
2837    const DIGITS: &[u8; 16] = b"0123456789abcdef";
2838    let mut output = String::with_capacity(bytes.len() * 2);
2839    for byte in bytes {
2840        output.push(char::from(DIGITS[usize::from(byte >> 4)]));
2841        output.push(char::from(DIGITS[usize::from(byte & 0x0f)]));
2842    }
2843    output
2844}
2845
2846/// A stopped actor and the manager that resolves its live replacement.
2847///
2848/// This fixture and its constructor are compiled unconditionally and hidden
2849/// from the documentation because the chat crate's tests need them, and a
2850/// `#[cfg(test)]` item is invisible to another crate.
2851#[doc(hidden)]
2852pub struct ReplacementSessionTestFixture {
2853    pub stopped: ManagedSessionHandle,
2854    pub control: SessionManagerControl,
2855}
2856
2857/// A stopped actor and a manager that resolves its live replacement. Chat
2858/// tests use this hand-written actor instead of mocking the session manager
2859/// protocol.
2860#[doc(hidden)]
2861pub fn replacement_session_test_fixture(
2862    session_id: &str,
2863    accepted_ordinal: u64,
2864) -> ReplacementSessionTestFixture {
2865    let (stopped_commands, stopped_commands_rx) = mpsc::channel(1);
2866    drop(stopped_commands_rx);
2867    let (stopped_releases, stopped_releases_rx) = mpsc::unbounded_channel();
2868    drop(stopped_releases_rx);
2869    let (stopped_view_tx, stopped_view) = watch::channel(ManagedSessionView::default());
2870    drop(stopped_view_tx);
2871    let stopped = ManagedSessionHandle {
2872        session_id: session_id.to_owned(),
2873        commands: stopped_commands,
2874        releases: stopped_releases,
2875        view: stopped_view,
2876    };
2877
2878    let (commands, mut commands_rx) = mpsc::channel(4);
2879    let (releases, _releases_rx) = mpsc::unbounded_channel();
2880    let (view_tx, view) = watch::channel(ManagedSessionView::default());
2881    let replacement = ManagedSessionHandle {
2882        session_id: session_id.to_owned(),
2883        commands,
2884        releases,
2885        view,
2886    };
2887    let actor_session_id = session_id.to_owned();
2888    tokio::spawn(async move {
2889        let _view_tx = view_tx;
2890        while let Some(command) = commands_rx.recv().await {
2891            match command {
2892                ActorCommand::Submit { reply, .. } => {
2893                    let _ = reply.send(Ok(accepted_ordinal));
2894                }
2895                ActorCommand::Sync { reply } => {
2896                    let _ = reply.send(Ok(()));
2897                }
2898                command => command.reject(&actor_session_id, "unsupported test operation"),
2899            }
2900        }
2901    });
2902
2903    let (manager_commands, mut manager_commands_rx) = mpsc::channel(4);
2904    let manager_replacement = replacement.clone();
2905    tokio::spawn(async move {
2906        while let Some(ManagerCommand::Session {
2907            session_id: requested,
2908            reply,
2909        }) = manager_commands_rx.recv().await
2910        {
2911            let resolved =
2912                (requested == manager_replacement.session_id).then(|| manager_replacement.clone());
2913            let _ = reply.send(resolved);
2914        }
2915    });
2916    ReplacementSessionTestFixture {
2917        stopped,
2918        control: SessionManagerControl {
2919            commands: manager_commands,
2920        },
2921    }
2922}
2923
2924#[cfg(test)]
2925mod tests {
2926    use super::*;
2927    #[cfg(unix)]
2928    use agent_client_protocol::schema::v1::{ContentBlock, TextContent};
2929    use sha2::Digest;
2930
2931    fn ordering_request(session_id: &str, command_id: &str) -> RemoteSessionRequest {
2932        let (reply, _response) = oneshot::channel();
2933        RemoteSessionRequest::Submit {
2934            session_id: session_id.into(),
2935            command_id: command_id.into(),
2936            command: RelayCommand::SetConfig {
2937                key: "effort".into(),
2938                value: "high".into(),
2939            },
2940            reply,
2941        }
2942    }
2943
2944    /// `/effort` followed by a prompt has to reach the relay that way round,
2945    /// or the prompt runs under the old setting. A bridge that spawns every
2946    /// request concurrently loses that, so the order is pinned here: the
2947    /// first request is held up, and the second must not overtake it.
2948    #[tokio::test]
2949    async fn one_session_keeps_its_requests_in_the_order_they_were_made() {
2950        let observed = Arc::new(Mutex::new(Vec::new()));
2951        let release = Arc::new(tokio::sync::Notify::new());
2952        let mut order = SessionRequestOrder::new();
2953
2954        for command_id in ["first", "second", "third"] {
2955            let observed = Arc::clone(&observed);
2956            let release = Arc::clone(&release);
2957            order.dispatch(ordering_request("session-a", command_id), move |request| {
2958                let RemoteSessionRequest::Submit { command_id, .. } = request else {
2959                    unreachable!("the fixture only submits")
2960                };
2961                async move {
2962                    // Only the first request waits. If the order were lost,
2963                    // the other two would finish while it is held.
2964                    if command_id == "first" {
2965                        release.notified().await;
2966                    }
2967                    observed.lock().unwrap().push(command_id);
2968                }
2969            });
2970        }
2971
2972        // Nothing may run while the first request is held. Yield generously:
2973        // the point is that the later requests never get to run, not that
2974        // they have not been polled yet.
2975        for _ in 0..64 {
2976            tokio::task::yield_now().await;
2977        }
2978        assert!(
2979            observed.lock().unwrap().is_empty(),
2980            "a later request overtook the one being held: {:?}",
2981            observed.lock().unwrap()
2982        );
2983
2984        release.notify_one();
2985        tokio::time::timeout(std::time::Duration::from_secs(5), async {
2986            while observed.lock().unwrap().len() < 3 {
2987                tokio::task::yield_now().await;
2988            }
2989        })
2990        .await
2991        .expect("every request ran");
2992        assert_eq!(*observed.lock().unwrap(), ["first", "second", "third"]);
2993    }
2994
2995    /// Ordering is per session: one session waiting on a slow relay must not
2996    /// hold up another session's prompt.
2997    #[tokio::test]
2998    async fn different_sessions_still_overlap() {
2999        let finished = Arc::new(Mutex::new(Vec::new()));
3000        let release = Arc::new(tokio::sync::Notify::new());
3001        let mut order = SessionRequestOrder::new();
3002
3003        let held = Arc::clone(&release);
3004        let recorder = Arc::clone(&finished);
3005        order.dispatch(ordering_request("session-a", "slow"), move |_| async move {
3006            held.notified().await;
3007            recorder.lock().unwrap().push("slow");
3008        });
3009        let recorder = Arc::clone(&finished);
3010        order.dispatch(ordering_request("session-b", "fast"), move |_| async move {
3011            recorder.lock().unwrap().push("fast");
3012        });
3013
3014        tokio::time::timeout(std::time::Duration::from_secs(5), async {
3015            while finished.lock().unwrap().is_empty() {
3016                tokio::task::yield_now().await;
3017            }
3018        })
3019        .await
3020        .expect("the other session ran while the first was held");
3021        assert_eq!(*finished.lock().unwrap(), ["fast"]);
3022
3023        release.notify_one();
3024        tokio::time::timeout(std::time::Duration::from_secs(5), async {
3025            while finished.lock().unwrap().len() < 2 {
3026                tokio::task::yield_now().await;
3027            }
3028        })
3029        .await
3030        .expect("the held request ran once released");
3031    }
3032
3033    /// A session that has gone quiet must not leave a handle behind for ever:
3034    /// a long-lived daemon serves many sessions.
3035    #[tokio::test]
3036    async fn finished_sessions_are_forgotten() {
3037        let mut order = SessionRequestOrder::new();
3038        for index in 0..8 {
3039            order.dispatch(
3040                ordering_request(&format!("session-{index}"), "only"),
3041                |_| async {},
3042            );
3043            tokio::time::timeout(std::time::Duration::from_secs(5), async {
3044                while order.latest.values().any(|handle| !handle.is_finished()) {
3045                    tokio::task::yield_now().await;
3046                }
3047            })
3048            .await
3049            .expect("the request finished");
3050        }
3051        // The next dispatch prunes what has finished, so the map tracks live
3052        // work rather than every session ever seen.
3053        order.dispatch(ordering_request("session-last", "only"), |_| async {});
3054        assert_eq!(order.latest.len(), 1);
3055    }
3056
3057    /// A reviewer action reaches a remote controller daemon as JSON, so both
3058    /// halves of the exchange have to survive that round trip intact.
3059    #[test]
3060    fn reviewer_actions_and_outcomes_survive_the_daemon_wire() {
3061        let config = ReviewerLaunchConfig {
3062            profile_id: "claude".into(),
3063            harness: hel::hel_config::HarnessKind::Claude,
3064            bridge_command: "npx".into(),
3065            bridge_args: vec!["claude-code-acp".into()],
3066            environment: BTreeMap::from([("EXTRA".into(), "1".into())]),
3067            execution_policy: hel::hel_config::ExecutionPolicy::Unconstrained,
3068            model: Some("sonnet".into()),
3069            effort: Some("high".into()),
3070            generation: 2,
3071            mcp_servers: Vec::new(),
3072        };
3073        let actions = [
3074            ReviewerAction::Start {
3075                config: Box::new(config),
3076            },
3077            ReviewerAction::Submit {
3078                command_id: "review-1".into(),
3079                command: RelayCommand::Cancel,
3080            },
3081            ReviewerAction::Attach {
3082                after_ordinal: 4,
3083                after_digest: "digest".into(),
3084            },
3085            ReviewerAction::Acknowledge {
3086                through_ordinal: 4,
3087                through_digest: "digest".into(),
3088            },
3089            ReviewerAction::Status,
3090            ReviewerAction::Pause,
3091            ReviewerAction::CaptureDelta {
3092                baselines: BTreeMap::from([(std::path::PathBuf::from("/w/app"), "tree".into())]),
3093            },
3094            ReviewerAction::AdvanceBaseline {
3095                trees: BTreeMap::from([(std::path::PathBuf::from("/w/app"), "tree".into())]),
3096            },
3097            ReviewerAction::AnalyzeDelta {
3098                repositories: vec![hel::hel_worker::AnalyzeDeltaRepository {
3099                    root: std::path::PathBuf::from("/w/app"),
3100                    baseline_tree: Some("base".into()),
3101                    current_tree: "target".into(),
3102                }],
3103            },
3104        ];
3105        for action in actions {
3106            let encoded = serde_json::to_string(&action).unwrap();
3107            let decoded: ReviewerAction = serde_json::from_str(&encoded).unwrap();
3108            assert_eq!(decoded, action);
3109        }
3110
3111        let outcome = ReviewerOutcome::Accepted { ordinal: 9 };
3112        let encoded = serde_json::to_string(&outcome).unwrap();
3113        let decoded: ReviewerOutcome = serde_json::from_str(&encoded).unwrap();
3114        assert!(matches!(decoded, ReviewerOutcome::Accepted { ordinal: 9 }));
3115
3116        let paused = serde_json::to_string(&ReviewerOutcome::Paused).unwrap();
3117        assert!(matches!(
3118            serde_json::from_str::<ReviewerOutcome>(&paused).unwrap(),
3119            ReviewerOutcome::Paused
3120        ));
3121
3122        let delta = ReviewerOutcome::Delta {
3123            repositories: vec![hel::hel_worker::RepoDelta {
3124                root: std::path::PathBuf::from("/w/app"),
3125                baseline_tree: None,
3126                current_tree: "target".into(),
3127                patch: "diff --git a/a b/a\n".into(),
3128                diffstat: "1 file changed".into(),
3129                changed_lines: 1,
3130            }],
3131        };
3132        let encoded = serde_json::to_string(&delta).unwrap();
3133        let ReviewerOutcome::Delta { repositories } =
3134            serde_json::from_str::<ReviewerOutcome>(&encoded).unwrap()
3135        else {
3136            panic!("a captured delta must survive the daemon wire");
3137        };
3138        assert_eq!(repositories.len(), 1);
3139        assert_eq!(repositories[0].current_tree, "target");
3140    }
3141
3142    /// Every reviewer action names itself for the actor's logs and for the
3143    /// rejection path, so a stalled review can be traced to the step it stalled
3144    /// on.
3145    #[test]
3146    fn every_reviewer_action_names_its_operation() {
3147        let names = [
3148            ReviewerAction::Submit {
3149                command_id: String::new(),
3150                command: RelayCommand::Cancel,
3151            }
3152            .operation_name(),
3153            ReviewerAction::Attach {
3154                after_ordinal: 0,
3155                after_digest: String::new(),
3156            }
3157            .operation_name(),
3158            ReviewerAction::Acknowledge {
3159                through_ordinal: 0,
3160                through_digest: String::new(),
3161            }
3162            .operation_name(),
3163            ReviewerAction::Status.operation_name(),
3164            ReviewerAction::Pause.operation_name(),
3165        ];
3166        assert_eq!(
3167            names,
3168            [
3169                "reviewer_submit",
3170                "reviewer_attach",
3171                "reviewer_acknowledge",
3172                "reviewer_status",
3173                "reviewer_pause",
3174            ]
3175        );
3176        assert!(names.iter().all(|name| name.starts_with("reviewer_")));
3177    }
3178
3179    #[test]
3180    fn reconnect_delay_backs_off_and_stops_at_the_ceiling() {
3181        assert_eq!(reconnect_delay(1), RECONNECT_INTERVAL);
3182        assert_eq!(reconnect_delay(2), Duration::from_secs(2));
3183        assert_eq!(reconnect_delay(4), Duration::from_secs(8));
3184        assert_eq!(reconnect_delay(6), RECONNECT_BACKOFF_CEILING);
3185        assert_eq!(reconnect_delay(u32::MAX), RECONNECT_BACKOFF_CEILING);
3186    }
3187
3188    #[test]
3189    fn only_dead_worker_connection_failures_request_a_restart() {
3190        // The wording is deliberately unlike anything a matcher could have
3191        // been written against: the marker, not the message, decides.
3192        let reworded = anyhow::Error::new(RelayTransportDead::new(
3193            "the session proxy vanished mid-conversation",
3194        ))
3195        .context("connect to the session worker for checkpoint");
3196        assert!(worker_connect_needs_restart(&reworded), "{reworded:#}");
3197        assert!(!worker_connect_allows_live_restart(&reworded));
3198
3199        // Text alone proves nothing now, not even the exact text the producing
3200        // sites still use: an unmarked failure must never restart a worker.
3201        for detail in [
3202            "relay proxy disconnected during hello",
3203            "Connection refused (os error 111)",
3204            "relay negotiated unsupported protocol 9",
3205            "controller projection is corrupt",
3206        ] {
3207            assert!(!worker_connect_needs_restart(&anyhow::anyhow!(detail)));
3208        }
3209    }
3210
3211    /// The producing side of the same contract: a proxy that dies without
3212    /// serving the handshake must ask for a worker restart, whatever its
3213    /// failure happens to read like.
3214    #[cfg(unix)]
3215    #[tokio::test]
3216    async fn a_proxy_that_dies_before_hello_requests_a_worker_restart() {
3217        let mut dead = target("sh");
3218        dead.spec = CommandSpec::new("sh", ["-c", "exit 1"]).purpose("dead relay proxy fixture");
3219
3220        let error = StandaloneSession::connect(&dead)
3221            .await
3222            .err()
3223            .expect("a proxy that exits cannot serve a session");
3224
3225        assert!(worker_connect_needs_restart(&error), "{error:#}");
3226        assert!(worker_connect_allows_live_restart(&error));
3227    }
3228
3229    /// A lease answer crosses a channel. Formatting the failure into a string
3230    /// there would strip the cause and silently cost the checkpoint path its
3231    /// restart decision, so prove the typed cause survives the handoff.
3232    #[cfg(unix)]
3233    #[tokio::test]
3234    async fn a_failed_lease_keeps_the_cause_that_decides_a_worker_restart() {
3235        let (commands_tx, commands_rx) = mpsc::channel(4);
3236        let (_releases_tx, releases_rx) = mpsc::unbounded_channel();
3237        let (_retirement_tx, retirement_rx) = watch::channel(false);
3238        let (view_tx, _view_rx) = watch::channel(ManagedSessionView::default());
3239        let (updates_tx, _updates_rx) = coalesced_update_channel();
3240        let mut dead = target("sh");
3241        dead.spec = CommandSpec::new("sh", ["-c", "exit 1"]).purpose("dead relay proxy fixture");
3242        tokio::spawn(run_session_actor(
3243            dead,
3244            commands_rx,
3245            releases_rx,
3246            retirement_rx,
3247            view_tx,
3248            updates_tx,
3249        ));
3250
3251        let (reply, response) = oneshot::channel();
3252        commands_tx
3253            .send(ActorCommand::Lease { reply })
3254            .await
3255            .unwrap();
3256        let error = response
3257            .await
3258            .expect("actor answered the lease request")
3259            .err()
3260            .expect("a dead proxy cannot be leased");
3261
3262        assert!(worker_connect_needs_restart(&error), "{error:#}");
3263    }
3264
3265    #[tokio::test]
3266    async fn recovery_restarts_a_live_worker_only_after_a_failed_handshake() {
3267        let directory = tempfile::tempdir().unwrap();
3268        let restarted = directory.path().join("restarted");
3269        let recovery = |liveness: &str| WorkerRecoveryPlan {
3270            target: None,
3271            liveness_probe: CommandSpec::new("printf", [format!("{liveness}\n")])
3272                .purpose("probe test worker liveness"),
3273            binary_refresh: None,
3274            launch_refresh: None,
3275            restart: CommandPlan {
3276                description: "restart test worker".into(),
3277                commands: vec![
3278                    CommandSpec::new("touch", [restarted.to_string_lossy().into_owned()])
3279                        .purpose("restart test worker"),
3280                ],
3281            },
3282        };
3283
3284        assert_eq!(
3285            recover_worker(recovery("alive"), false).await.unwrap(),
3286            WorkerRecoveryOutcome::Alive
3287        );
3288        assert!(!restarted.exists(), "a live worker must not be restarted");
3289
3290        assert_eq!(
3291            recover_worker(recovery("starting"), true).await.unwrap(),
3292            WorkerRecoveryOutcome::Starting
3293        );
3294        assert!(
3295            !restarted.exists(),
3296            "a worker recovering its journal must not be restarted"
3297        );
3298
3299        assert_eq!(
3300            recover_worker(recovery("alive"), true).await.unwrap(),
3301            WorkerRecoveryOutcome::RestartedUnresponsive
3302        );
3303        assert!(
3304            restarted.exists(),
3305            "a worker that cannot serve a fresh handshake is restarted"
3306        );
3307        std::fs::remove_file(&restarted).unwrap();
3308
3309        assert_eq!(
3310            recover_worker(recovery("dead"), false).await.unwrap(),
3311            WorkerRecoveryOutcome::RestartedDead
3312        );
3313        assert!(restarted.exists(), "a confirmed dead worker is restarted");
3314    }
3315
3316    #[tokio::test]
3317    async fn recovery_replaces_only_a_stale_worker_binary_before_restart() {
3318        let directory = tempfile::tempdir().unwrap();
3319        let source = directory.path().join("current-worker");
3320        let refreshed = directory.path().join("worker-refreshed");
3321        let restarted = directory.path().join("worker-restarted");
3322        std::fs::write(&source, b"current worker binary").unwrap();
3323        let current_digest = format!("{:x}", sha2::Sha256::digest(b"current worker binary"));
3324        let recovery = |installed_digest: &str, require_refresh: bool| {
3325            let mut restart = if require_refresh {
3326                CommandSpec::new(
3327                    "sh",
3328                    [
3329                        "-c",
3330                        "test -f \"$MJ_TEST_REFRESHED\" && touch -- \"$MJ_TEST_RESTARTED\"",
3331                    ],
3332                )
3333            } else {
3334                CommandSpec::new("touch", [restarted.to_string_lossy().into_owned()])
3335            }
3336            .purpose("restart test worker");
3337            restart.env.insert(
3338                "MJ_TEST_REFRESHED".into(),
3339                refreshed.to_string_lossy().into_owned(),
3340            );
3341            restart.env.insert(
3342                "MJ_TEST_RESTARTED".into(),
3343                restarted.to_string_lossy().into_owned(),
3344            );
3345            WorkerRecoveryPlan {
3346                target: None,
3347                liveness_probe: CommandSpec::new("printf", ["dead\n"])
3348                    .purpose("probe test worker liveness"),
3349                binary_refresh: Some(WorkerBinaryRefresh::Prepared(WorkerBinaryRefreshPlan {
3350                    source: source.clone(),
3351                    installed_digest: CommandSpec::new(
3352                        "printf",
3353                        [format!("{installed_digest}  /worker/hel\n")],
3354                    )
3355                    .purpose("identify test worker binary"),
3356                    replace: CommandPlan {
3357                        description: "refresh test worker".into(),
3358                        commands: vec![
3359                            CommandSpec::new("touch", [refreshed.to_string_lossy().into_owned()])
3360                                .purpose("refresh test worker"),
3361                        ],
3362                    },
3363                })),
3364                launch_refresh: None,
3365                restart: CommandPlan {
3366                    description: "restart test worker".into(),
3367                    commands: vec![restart],
3368                },
3369            }
3370        };
3371
3372        assert_eq!(
3373            recover_worker(recovery(&current_digest, false), false)
3374                .await
3375                .unwrap(),
3376            WorkerRecoveryOutcome::RestartedDead
3377        );
3378        assert!(!refreshed.exists(), "a current binary must not be copied");
3379        assert!(restarted.exists());
3380
3381        std::fs::remove_file(&restarted).unwrap();
3382        assert_eq!(
3383            recover_worker(recovery(&"0".repeat(64), true), false)
3384                .await
3385                .unwrap(),
3386            WorkerRecoveryOutcome::RestartedDead
3387        );
3388        assert!(refreshed.exists(), "a stale binary must be refreshed");
3389        assert!(restarted.exists(), "refresh must finish before restart");
3390    }
3391
3392    #[tokio::test]
3393    async fn recovery_refreshes_a_stale_launch_config_before_restart() {
3394        let directory = tempfile::tempdir().unwrap();
3395        let refreshed = directory.path().join("launch-refreshed");
3396        let restarted = directory.path().join("worker-restarted");
3397        let mut restart = CommandSpec::new(
3398            "sh",
3399            [
3400                "-c",
3401                "test -f \"$MJ_TEST_REFRESHED\" && touch -- \"$MJ_TEST_RESTARTED\"",
3402            ],
3403        )
3404        .purpose("restart test worker");
3405        restart.env.insert(
3406            "MJ_TEST_REFRESHED".into(),
3407            refreshed.to_string_lossy().into_owned(),
3408        );
3409        restart.env.insert(
3410            "MJ_TEST_RESTARTED".into(),
3411            restarted.to_string_lossy().into_owned(),
3412        );
3413        let outcome = recover_worker(
3414            WorkerRecoveryPlan {
3415                target: None,
3416                liveness_probe: CommandSpec::new("printf", ["dead\n"])
3417                    .purpose("probe test worker liveness"),
3418                binary_refresh: None,
3419                launch_refresh: Some(WorkerLaunchRefreshPlan {
3420                    expected_sha256: "a".repeat(64),
3421                    installed_digest: CommandSpec::new(
3422                        "printf",
3423                        [format!("{}  /worker/launch.json\n", "b".repeat(64))],
3424                    )
3425                    .purpose("identify test launch config"),
3426                    replace: CommandPlan {
3427                        description: "refresh test launch config".into(),
3428                        commands: vec![
3429                            CommandSpec::new("touch", [refreshed.to_string_lossy().into_owned()])
3430                                .purpose("refresh test launch config"),
3431                        ],
3432                    },
3433                }),
3434                restart: CommandPlan {
3435                    description: "restart test worker".into(),
3436                    commands: vec![restart],
3437                },
3438            },
3439            false,
3440        )
3441        .await
3442        .unwrap();
3443
3444        assert_eq!(outcome, WorkerRecoveryOutcome::RestartedDead);
3445        assert!(refreshed.exists());
3446        assert!(
3447            restarted.exists(),
3448            "config refresh must finish before restart"
3449        );
3450    }
3451
3452    #[tokio::test]
3453    async fn recovery_starts_a_stopped_target_before_probing_its_worker() {
3454        let directory = tempfile::tempdir().unwrap();
3455        let target_started = directory.path().join("target-started");
3456        let worker_restarted = directory.path().join("worker-restarted");
3457        let inspection = |status: &str| {
3458            serde_json::to_string(&serde_json::json!([{
3459                "Config": { "Labels": {
3460                    (hel::hel_targets::MANAGED_LABEL): "true",
3461                    (hel::hel_targets::SESSION_LABEL): "session-1",
3462                }},
3463                "State": { "Status": status },
3464            }]))
3465            .unwrap()
3466        };
3467        let mut inspect = CommandSpec::new(
3468            "sh",
3469            [
3470                "-c",
3471                "if [ -f \"$MJ_TEST_TARGET_STARTED\" ]; then printf '%s\\n' \"$MJ_TEST_RUNNING\"; else printf '%s\\n' \"$MJ_TEST_EXITED\"; fi",
3472            ],
3473        )
3474        .purpose("inspect test target");
3475        inspect.env.insert(
3476            "MJ_TEST_TARGET_STARTED".into(),
3477            target_started.to_string_lossy().into_owned(),
3478        );
3479        inspect
3480            .env
3481            .insert("MJ_TEST_RUNNING".into(), inspection("running"));
3482        inspect
3483            .env
3484            .insert("MJ_TEST_EXITED".into(), inspection("exited"));
3485        let mut start = CommandSpec::new("sh", ["-c", "touch -- \"$MJ_TEST_TARGET_STARTED\""])
3486            .purpose("start test target");
3487        start.env.insert(
3488            "MJ_TEST_TARGET_STARTED".into(),
3489            target_started.to_string_lossy().into_owned(),
3490        );
3491        let mut liveness = CommandSpec::new(
3492            "sh",
3493            [
3494                "-c",
3495                "test -f \"$MJ_TEST_TARGET_STARTED\" && printf 'dead\\n'",
3496            ],
3497        )
3498        .purpose("probe test worker after target start");
3499        liveness.env.insert(
3500            "MJ_TEST_TARGET_STARTED".into(),
3501            target_started.to_string_lossy().into_owned(),
3502        );
3503
3504        let outcome = recover_worker(
3505            WorkerRecoveryPlan {
3506                target: Some(TargetRecoveryPlan {
3507                    exists: CommandSpec::new("true", std::iter::empty::<&str>())
3508                        .purpose("check test target"),
3509                    inspect,
3510                    start,
3511                    session_id: "session-1".into(),
3512                }),
3513                liveness_probe: liveness,
3514                binary_refresh: None,
3515                launch_refresh: None,
3516                restart: CommandPlan {
3517                    description: "restart test worker".into(),
3518                    commands: vec![
3519                        CommandSpec::new(
3520                            "touch",
3521                            [worker_restarted.to_string_lossy().into_owned()],
3522                        )
3523                        .purpose("restart test worker"),
3524                    ],
3525                },
3526            },
3527            false,
3528        )
3529        .await
3530        .unwrap();
3531
3532        assert_eq!(outcome, WorkerRecoveryOutcome::RestartedDead);
3533        assert!(target_started.exists());
3534        assert!(worker_restarted.exists());
3535    }
3536
3537    #[tokio::test]
3538    async fn recovery_reports_a_missing_target_without_running_worker_commands() {
3539        let unreachable = CommandSpec::new("false", std::iter::empty::<&str>());
3540        let outcome = recover_worker(
3541            WorkerRecoveryPlan {
3542                target: Some(TargetRecoveryPlan {
3543                    exists: unreachable,
3544                    inspect: CommandSpec::new("false", std::iter::empty::<&str>()),
3545                    start: CommandSpec::new("false", std::iter::empty::<&str>()),
3546                    session_id: "session-1".into(),
3547                }),
3548                liveness_probe: CommandSpec::new("false", std::iter::empty::<&str>()),
3549                binary_refresh: None,
3550                launch_refresh: None,
3551                restart: CommandPlan {
3552                    description: "must not restart".into(),
3553                    commands: vec![CommandSpec::new("false", std::iter::empty::<&str>())],
3554                },
3555            },
3556            true,
3557        )
3558        .await
3559        .unwrap();
3560
3561        assert_eq!(outcome, WorkerRecoveryOutcome::TargetMissing);
3562    }
3563
3564    fn target(program: &str) -> RelaySessionTarget {
3565        RelaySessionTarget {
3566            session_id: "session-1".to_owned(),
3567            spec: CommandSpec::new(program, std::iter::empty::<&str>()),
3568            worker_recovery: None,
3569            project_memory: None,
3570        }
3571    }
3572
3573    /// A connected view carrying a conversation, so republishing it exercises
3574    /// the case a whole-transcript comparison would have to walk.
3575    fn view_at_ordinal(ordinal: u64) -> ManagedSessionView {
3576        let digest = "a".repeat(64);
3577        let mut materialized = MaterializedSession::empty("session-1");
3578        materialized.applied_event_ordinal = ordinal;
3579        materialized.applied_event_digest = digest.clone();
3580        materialized.transcript = (1..=200)
3581            .map(|position| {
3582                Arc::new(hel::hel_state::TranscriptItem {
3583                    stable_id: format!("system:{position}"),
3584                    position,
3585                    latest_content_event_ordinal: None,
3586                    created_at_ms: 1,
3587                    last_changed_at_ms: 1,
3588                    body: hel::hel_state::TranscriptBody::System {
3589                        text: format!("event {position}"),
3590                    },
3591                })
3592            })
3593            .collect();
3594        ManagedSessionView {
3595            snapshot: Some(ManagedSessionSnapshot {
3596                window: hel::hel_state::ProjectionWindow::of(&materialized),
3597                materialized,
3598                operational: RelayOperationalState {
3599                    session_id: "session-1".into(),
3600                    execution: hel::hel_worker::RelayExecutionState::Idle,
3601                    latest_ordinal: ordinal,
3602                    latest_digest: digest.clone(),
3603                    acknowledged_through: ordinal,
3604                    acknowledged_digest: digest,
3605                    recovery_floor_ordinal: 0,
3606                    recovery_floor_digest: hel::hel_worker::RELAY_EVENT_GENESIS_DIGEST.into(),
3607                    native_session_id: None,
3608                    agent_capabilities: None,
3609                    agent_info: None,
3610                    config_options: Vec::new(),
3611                    modes: None,
3612                    available_commands: Vec::new(),
3613                    config: BTreeMap::new(),
3614                    active_prompt: None,
3615                    queued_prompts: Vec::new(),
3616                    active_user_shells: Vec::new(),
3617                    active_agent_terminals: Vec::new(),
3618                    checkpoint_barrier: None,
3619                    checkpoint_ready: None,
3620                    last_acp_activity_at_ms: None,
3621                    current_step_started_at_ms: None,
3622                    foreground_tool_started_at_ms: None,
3623                    harness_turn: None,
3624                    last_harness_turn_started_ordinal: None,
3625                    background_commands: Vec::new(),
3626                },
3627                latest_credential_sync_signal: None,
3628                worker_build: None,
3629            }),
3630            connected: true,
3631            error: None,
3632        }
3633    }
3634
3635    #[test]
3636    fn republishing_an_unchanged_view_notifies_nobody() {
3637        let (view_tx, mut view_rx) = watch::channel(ManagedSessionView::default());
3638        let (updates_tx, mut updates_rx) = coalesced_update_channel();
3639
3640        publish_view("session-1", view_at_ordinal(7), &view_tx, &updates_tx);
3641        assert!(view_rx.has_changed().expect("watch stays open"));
3642        assert_eq!(
3643            updates_rx.try_recv().expect("the first view is news").view,
3644            view_at_ordinal(7)
3645        );
3646        let _ = view_rx.borrow_and_update();
3647
3648        publish_view("session-1", view_at_ordinal(7), &view_tx, &updates_tx);
3649
3650        assert!(
3651            !view_rx.has_changed().expect("watch stays open"),
3652            "a sync tick that moved nothing must not wake the dashboard"
3653        );
3654        assert!(updates_rx.try_recv().is_err());
3655    }
3656
3657    #[test]
3658    fn publishing_an_advanced_event_frontier_notifies_watchers() {
3659        let (view_tx, mut view_rx) = watch::channel(ManagedSessionView::default());
3660        let (updates_tx, mut updates_rx) = coalesced_update_channel();
3661        publish_view("session-1", view_at_ordinal(7), &view_tx, &updates_tx);
3662        let _ = updates_rx.try_recv();
3663        let _ = view_rx.borrow_and_update();
3664
3665        publish_view("session-1", view_at_ordinal(8), &view_tx, &updates_tx);
3666
3667        assert!(view_rx.has_changed().expect("watch stays open"));
3668        let update = updates_rx.try_recv().expect("the advance is news");
3669        assert_eq!(update.session_id, "session-1");
3670        assert_eq!(
3671            update
3672                .view
3673                .snapshot
3674                .expect("published snapshot")
3675                .materialized
3676                .applied_event_ordinal,
3677            8
3678        );
3679    }
3680
3681    #[test]
3682    fn publishing_relay_state_that_moved_without_the_frontier_notifies_watchers() {
3683        let (view_tx, mut view_rx) = watch::channel(ManagedSessionView::default());
3684        let (updates_tx, mut updates_rx) = coalesced_update_channel();
3685        publish_view("session-1", view_at_ordinal(7), &view_tx, &updates_tx);
3686        let _ = updates_rx.try_recv();
3687        let _ = view_rx.borrow_and_update();
3688
3689        let mut view = view_at_ordinal(7);
3690        view.snapshot
3691            .as_mut()
3692            .expect("published snapshot")
3693            .operational
3694            .execution = hel::hel_worker::RelayExecutionState::Running;
3695        publish_view("session-1", view, &view_tx, &updates_tx);
3696
3697        assert!(view_rx.has_changed().expect("watch stays open"));
3698        assert!(updates_rx.try_recv().is_ok());
3699    }
3700
3701    #[test]
3702    fn losing_the_relay_republishes_the_same_snapshot_as_disconnected() {
3703        let (view_tx, mut view_rx) = watch::channel(ManagedSessionView::default());
3704        let (updates_tx, mut updates_rx) = coalesced_update_channel();
3705        publish_view("session-1", view_at_ordinal(7), &view_tx, &updates_tx);
3706        let _ = updates_rx.try_recv();
3707        let _ = view_rx.borrow_and_update();
3708
3709        let mut view = view_at_ordinal(7);
3710        view.connected = false;
3711        view.error = Some(ViewError::Unreachable("relay is unreachable".into()));
3712        publish_view("session-1", view, &view_tx, &updates_tx);
3713
3714        assert!(view_rx.has_changed().expect("watch stays open"));
3715        assert!(updates_rx.try_recv().is_ok());
3716    }
3717
3718    #[test]
3719    fn command_ids_are_namespaced_and_unique() {
3720        let first = new_command_id("prompt").unwrap();
3721        let second = new_command_id("prompt").unwrap();
3722        assert!(first.starts_with("prompt-"));
3723        assert_ne!(first, second);
3724    }
3725
3726    #[test]
3727    fn leased_actor_defers_replacement_and_uses_latest_queued_target() {
3728        let original = target("relay-v1");
3729        let intermediate = target("relay-v2");
3730        let latest = target("relay-v3");
3731        let mut lifecycle = ActorLifecycle::default();
3732        lifecycle.activate_lease(7);
3733
3734        assert_eq!(
3735            reconcile_action(Some(&original), Some(&intermediate)),
3736            ReconcileAction::Retire
3737        );
3738        lifecycle.set_retirement_requested(true);
3739        assert!(!lifecycle.accepts_new_work());
3740        assert!(!lifecycle.should_stop());
3741
3742        assert_eq!(
3743            reconcile_action(Some(&original), Some(&latest)),
3744            ReconcileAction::Retire
3745        );
3746        assert!(lifecycle.return_lease(7));
3747        assert!(lifecycle.should_stop());
3748
3749        assert_eq!(
3750            reconcile_action(None, Some(&latest)),
3751            ReconcileAction::Spawn
3752        );
3753    }
3754
3755    #[test]
3756    fn leased_actor_defers_removal_until_its_connection_returns() {
3757        let original = target("relay-v1");
3758        let mut lifecycle = ActorLifecycle::default();
3759        lifecycle.activate_lease(11);
3760
3761        assert_eq!(
3762            reconcile_action(Some(&original), None),
3763            ReconcileAction::Retire
3764        );
3765        lifecycle.set_retirement_requested(true);
3766        assert!(!lifecycle.should_stop());
3767        assert!(!lifecycle.return_lease(10));
3768        assert!(!lifecycle.should_stop());
3769        assert!(lifecycle.return_lease(11));
3770        assert!(lifecycle.should_stop());
3771        assert_eq!(reconcile_action(None, None), ReconcileAction::Idle);
3772    }
3773
3774    #[test]
3775    fn queued_change_back_to_current_target_cancels_retirement() {
3776        let original = target("relay-v1");
3777        let replacement = target("relay-v2");
3778        let mut lifecycle = ActorLifecycle::default();
3779        lifecycle.activate_lease(3);
3780
3781        assert_eq!(
3782            reconcile_action(Some(&original), Some(&replacement)),
3783            ReconcileAction::Retire
3784        );
3785        lifecycle.set_retirement_requested(true);
3786        assert_eq!(
3787            reconcile_action(Some(&original), Some(&original)),
3788            ReconcileAction::Keep
3789        );
3790        lifecycle.set_retirement_requested(false);
3791
3792        assert!(lifecycle.return_lease(3));
3793        assert!(!lifecycle.should_stop());
3794        assert!(lifecycle.accepts_new_work());
3795    }
3796
3797    #[tokio::test]
3798    async fn stopped_actor_is_replaced_without_late_completion_removing_replacement() {
3799        let desired = target("sh");
3800        let desired_targets = target_map(std::slice::from_ref(&desired));
3801        let mut actors = BTreeMap::new();
3802        let mut tasks = tokio::task::JoinSet::new();
3803        let (commands, commands_rx) = mpsc::channel(1);
3804        drop(commands_rx);
3805        let (releases, _releases_rx) = mpsc::unbounded_channel();
3806        let (retirement, _retirement_rx) = watch::channel(false);
3807        let (_view_tx, view) = watch::channel(ManagedSessionView::default());
3808        let old_abort = tasks.spawn(async { "session-1".to_owned() });
3809        let old_task_id = old_abort.id();
3810        actors.insert(
3811            "session-1".to_owned(),
3812            ActorRegistration {
3813                target: desired.clone(),
3814                commands,
3815                releases,
3816                retirement,
3817                view,
3818                abort: old_abort,
3819            },
3820        );
3821        let (updates, _updates_rx) = coalesced_update_channel();
3822
3823        reconcile_actors(&desired_targets, &mut actors, &mut tasks, &updates);
3824
3825        let replacement_task_id = actors["session-1"].abort.id();
3826        assert_ne!(replacement_task_id, old_task_id);
3827        assert!(!actors["session-1"].commands.is_closed());
3828        assert_eq!(remove_actor_task(&mut actors, old_task_id), None);
3829        assert_eq!(actors["session-1"].abort.id(), replacement_task_id);
3830        tasks.abort_all();
3831    }
3832
3833    const UNREACHABLE_VIEW_TEST_CHILD: &str = "MJ_TEST_UNREACHABLE_RELAY_CHILD";
3834
3835    #[tokio::test(start_paused = true)]
3836    async fn unreachable_relay_publishes_error_view() {
3837        // MJ_DATA_DIR is process-global, so run the database-backed half in
3838        // an exact child test instead of racing unrelated tests in this
3839        // process.
3840        if std::env::var_os(UNREACHABLE_VIEW_TEST_CHILD).is_none() {
3841            let directory = tempfile::tempdir().unwrap();
3842            let test_name = format!(
3843                "{}::unreachable_relay_publishes_error_view",
3844                module_path!()
3845                    .strip_prefix("mj_controller::")
3846                    .unwrap_or(module_path!())
3847            );
3848            let output = std::process::Command::new(std::env::current_exe().unwrap())
3849                .args(["--exact", &test_name, "--nocapture"])
3850                .env(UNREACHABLE_VIEW_TEST_CHILD, "1")
3851                .env("MJ_DATA_DIR", directory.path())
3852                .output()
3853                .unwrap();
3854            assert!(
3855                output.status.success(),
3856                "isolated unreachable relay test failed\nstdout:\n{}\nstderr:\n{}",
3857                String::from_utf8_lossy(&output.stdout),
3858                String::from_utf8_lossy(&output.stderr)
3859            );
3860            return;
3861        }
3862
3863        // A regression in the publish path deadlocks the actor instead of
3864        // returning an error, so convert a hang into a hard failure.
3865        std::thread::spawn(|| {
3866            std::thread::sleep(Duration::from_secs(60));
3867            eprintln!("unreachable relay error view was never published");
3868            std::process::exit(101);
3869        });
3870
3871        let (_commands_tx, commands_rx) = mpsc::channel(4);
3872        let (_releases_tx, releases_rx) = mpsc::unbounded_channel();
3873        let (_retirement_tx, retirement_rx) = watch::channel(false);
3874        let (view_tx, mut view_rx) = watch::channel(ManagedSessionView::default());
3875        let (updates_tx, mut updates_rx) = coalesced_update_channel();
3876        tokio::spawn(run_session_actor(
3877            target("hel-relay-program-that-does-not-exist"),
3878            commands_rx,
3879            releases_rx,
3880            retirement_rx,
3881            view_tx,
3882            updates_tx,
3883        ));
3884
3885        loop {
3886            view_rx.changed().await.unwrap();
3887            let view = view_rx.borrow_and_update().clone();
3888            if !view.connected {
3889                let error = view
3890                    .error
3891                    .expect("unreachable view carries the connect error");
3892                assert!(
3893                    error.detail().contains("session relay proxy"),
3894                    "unexpected error: {error:?}"
3895                );
3896                break;
3897            }
3898        }
3899        let update = updates_rx
3900            .recv()
3901            .await
3902            .expect("dashboard feed received the error view");
3903        assert_eq!(update.session_id, "session-1");
3904        assert!(!update.view.connected);
3905    }
3906
3907    const UNREADABLE_PROJECTION_TEST_CHILD: &str = "MJ_TEST_UNREADABLE_PROJECTION_CHILD";
3908
3909    #[tokio::test]
3910    async fn connecting_to_an_absent_worker_never_reads_the_projection() {
3911        // MJ_DATA_DIR is process-global, so run the database-backed half in
3912        // an exact child test instead of racing unrelated tests in this
3913        // process.
3914        if std::env::var_os(UNREADABLE_PROJECTION_TEST_CHILD).is_none() {
3915            let directory = tempfile::tempdir().unwrap();
3916            // A directory where the database file belongs makes every
3917            // projection read fail, so a read that happens at all shows up in
3918            // the reported error.
3919            std::fs::create_dir(directory.path().join("mj.sqlite3")).unwrap();
3920            let output = std::process::Command::new(std::env::current_exe().unwrap())
3921                .args([
3922                    "--exact",
3923                    &format!(
3924                        "{}::connecting_to_an_absent_worker_never_reads_the_projection",
3925                        module_path!()
3926                            .strip_prefix("mj_controller::")
3927                            .unwrap_or(module_path!())
3928                    ),
3929                    "--nocapture",
3930                ])
3931                .env(UNREADABLE_PROJECTION_TEST_CHILD, "1")
3932                .env("MJ_DATA_DIR", directory.path())
3933                .output()
3934                .unwrap();
3935            assert!(
3936                output.status.success(),
3937                "isolated projection ordering test failed\nstdout:\n{}\nstderr:\n{}",
3938                String::from_utf8_lossy(&output.stdout),
3939                String::from_utf8_lossy(&output.stderr)
3940            );
3941            return;
3942        }
3943
3944        assert!(
3945            hel::hel_database::load_materialized_session("session-1").is_err(),
3946            "this store must fail every projection read for the test to mean anything"
3947        );
3948        let connected =
3949            StandaloneSession::connect(&target("hel-relay-program-that-does-not-exist")).await;
3950        let error = match connected {
3951            Ok(_) => panic!("a relay program that does not exist cannot connect"),
3952            Err(error) => error,
3953        };
3954        let detail = format!("{error:#}");
3955        assert!(
3956            detail.contains("session relay proxy"),
3957            "unexpected error: {detail}"
3958        );
3959        assert!(
3960            !detail.contains("Mjolnir database"),
3961            "connect read the projection before it reached the relay: {detail}"
3962        );
3963    }
3964
3965    const LEASED_RELAY_ROOT: &str = "MJ_TEST_LEASED_RELAY_ROOT";
3966    #[cfg(unix)]
3967    const AUTO_RESTART_TEST_CHILD: &str = "MJ_TEST_AUTO_RESTART_CHILD";
3968    #[cfg(unix)]
3969    const AUTO_RESTART_MARKER: &str = "MJ_TEST_AUTO_RESTART_MARKER";
3970    #[cfg(unix)]
3971    const DEFERRED_SUBMIT_TEST_CHILD: &str = "MJ_TEST_DEFERRED_SUBMIT_CHILD";
3972    #[cfg(unix)]
3973    const RETIRED_SUBMIT_TEST_CHILD: &str = "MJ_TEST_RETIRED_SUBMIT_CHILD";
3974    #[cfg(unix)]
3975    const RETURNED_LEASE_VIEW_TEST_CHILD: &str = "MJ_TEST_RETURNED_LEASE_VIEW_CHILD";
3976    #[cfg(unix)]
3977    const EXPLICIT_MEMORY_SYNC_TEST_CHILD: &str = "MJ_TEST_EXPLICIT_MEMORY_SYNC_CHILD";
3978    #[cfg(unix)]
3979    const SUBMIT_WITHOUT_SYNC_TEST_CHILD: &str = "MJ_TEST_SUBMIT_WITHOUT_SYNC_CHILD";
3980    #[cfg(unix)]
3981    const MANAGER_SHUTDOWN_TEST_CHILD: &str = "MJ_TEST_MANAGER_SHUTDOWN_CHILD";
3982    const LEASED_RELAY_SESSION: &str = "018f9dd2-a3b4-7c8d-9000-123456789abc";
3983
3984    /// Relay server half of the leased-submission tests. It does nothing unless
3985    /// a parent test points it at a relay journal root.
3986    #[test]
3987    fn leased_relay_child_serves_stdio() {
3988        let Some(root) = std::env::var_os(LEASED_RELAY_ROOT) else {
3989            return;
3990        };
3991        // With `--nocapture` libtest writes `test <name> ... ` without a
3992        // trailing newline before the body runs. End that line first so it
3993        // cannot glue itself onto the first protocol frame.
3994        println!();
3995        let mut relay = hel::hel_worker::DurableRelay::open(
3996            std::path::Path::new(&root),
3997            LEASED_RELAY_SESSION,
3998            "1.0.0",
3999        )
4000        .expect("open the test relay journal");
4001        hel::hel_worker::serve_relay_json_lines(
4002            &mut std::io::stdin().lock(),
4003            &mut std::io::stdout().lock(),
4004            &mut relay,
4005        )
4006        .expect("serve relay frames until the controller disconnects");
4007    }
4008
4009    #[cfg(unix)]
4010    fn exact_test_name(test: &str) -> String {
4011        format!(
4012            "{}::{test}",
4013            module_path!()
4014                .strip_prefix("mj_controller::")
4015                .unwrap_or(module_path!())
4016        )
4017    }
4018
4019    /// MJ_DATA_DIR is process-global, so every test that reaches the
4020    /// controller database runs in an exact child with its own data directory.
4021    #[cfg(unix)]
4022    fn run_in_isolated_child(marker: &str, test: &str) {
4023        let directory = tempfile::tempdir().unwrap();
4024        let output = std::process::Command::new(std::env::current_exe().unwrap())
4025            .args(["--exact", &exact_test_name(test), "--nocapture"])
4026            .env(marker, "1")
4027            .env("MJ_DATA_DIR", directory.path())
4028            .output()
4029            .unwrap();
4030        assert!(
4031            output.status.success(),
4032            "isolated {test} failed\nstdout:\n{}\nstderr:\n{}",
4033            String::from_utf8_lossy(&output.stdout),
4034            String::from_utf8_lossy(&output.stderr)
4035        );
4036    }
4037
4038    #[cfg(unix)]
4039    #[tokio::test]
4040    async fn session_manager_shutdown_joins_a_live_relay_actor() {
4041        if std::env::var_os(MANAGER_SHUTDOWN_TEST_CHILD).is_none() {
4042            run_in_isolated_child(
4043                MANAGER_SHUTDOWN_TEST_CHILD,
4044                "session_manager_shutdown_joins_a_live_relay_actor",
4045            );
4046            return;
4047        }
4048        // Alone in this child process, so it installs the one writer.
4049        let _writer = hel::hel_database::install_isolated_test_writer();
4050        register_leased_relay_session();
4051        let relay_root = tempfile::tempdir().unwrap();
4052        let SessionManagerChannels {
4053            targets,
4054            control,
4055            updates: _updates,
4056            shutdown,
4057        } = spawn_session_manager().expect("spawn the session manager");
4058        targets.send_replace(vec![leased_relay_target(relay_root.path())]);
4059        let session = control
4060            .wait_for_session(LEASED_RELAY_SESSION, Duration::from_secs(2))
4061            .await
4062            .expect("manager registered the relay actor");
4063        session
4064            .sync_now()
4065            .await
4066            .expect("relay actor established a live connection");
4067        assert!(session.view().connected);
4068
4069        tokio::time::timeout(Duration::from_secs(2), shutdown.shutdown())
4070            .await
4071            .expect("manager shutdown stayed within its deadline")
4072            .expect("manager shutdown task completed cleanly");
4073    }
4074
4075    #[cfg(unix)]
4076    #[tokio::test]
4077    async fn relay_attach_does_not_probe_or_install_project_memory() {
4078        if std::env::var_os(EXPLICIT_MEMORY_SYNC_TEST_CHILD).is_none() {
4079            run_in_isolated_child(
4080                EXPLICIT_MEMORY_SYNC_TEST_CHILD,
4081                "relay_attach_does_not_probe_or_install_project_memory",
4082            );
4083            return;
4084        }
4085        // Alone in this child process, so it installs the one writer.
4086        let _writer = hel::hel_database::install_isolated_test_writer();
4087        register_leased_relay_session();
4088        let relay_root = tempfile::tempdir().unwrap();
4089        let canonical = tempfile::tempdir().unwrap();
4090        let mut target = leased_relay_target(relay_root.path());
4091        target.project_memory = Some(ProjectMemorySyncTarget {
4092            canonical_root: canonical.path().to_path_buf(),
4093        });
4094
4095        let mut connection = StandaloneSession::connect(&target)
4096            .await
4097            .expect("relay attach must not depend on its memory endpoint");
4098        assert!(
4099            connection.project_memory.is_some(),
4100            "attach must leave memory pending for an explicit checkpoint sync"
4101        );
4102
4103        connection
4104            .sync_project_memory()
4105            .await
4106            .expect("an explicit sync may detect a legacy memory endpoint");
4107        assert!(
4108            connection.project_memory.is_none(),
4109            "the explicit sync reached the relay and disabled its unavailable endpoint"
4110        );
4111    }
4112
4113    /// Catching the local projection up to an accepted command is the
4114    /// expensive half of a submit, and a caller waiting to hear that the relay
4115    /// took the command should not wait for it. The two are separate calls, so
4116    /// the cheap one can answer first.
4117    #[cfg(unix)]
4118    #[tokio::test]
4119    async fn submitting_does_not_catch_the_projection_up_until_asked() {
4120        if std::env::var_os(SUBMIT_WITHOUT_SYNC_TEST_CHILD).is_none() {
4121            run_in_isolated_child(
4122                SUBMIT_WITHOUT_SYNC_TEST_CHILD,
4123                "submitting_does_not_catch_the_projection_up_until_asked",
4124            );
4125            return;
4126        }
4127        // Alone in this child process, so it installs the one writer.
4128        let _writer = hel::hel_database::install_isolated_test_writer();
4129        register_leased_relay_session();
4130        let relay_root = tempfile::tempdir().unwrap();
4131        let mut connection = StandaloneSession::connect(&leased_relay_target(relay_root.path()))
4132            .await
4133            .expect("connect to the live test relay");
4134        let before = connection.materialized.applied_event_ordinal;
4135
4136        let ordinal = connection
4137            .submit_accepted(
4138                new_command_id("prompt").unwrap(),
4139                RelayCommand::Prompt {
4140                    prompt: vec![ContentBlock::Text(TextContent::new("hello"))],
4141                },
4142            )
4143            .await
4144            .expect("the relay accepted the command");
4145        assert!(ordinal > before, "the relay reported where it accepted it");
4146        assert_eq!(
4147            connection.materialized.applied_event_ordinal, before,
4148            "the caller was answered without paying for the catch-up"
4149        );
4150
4151        connection.sync().await.expect("catch the projection up");
4152        assert!(
4153            connection.materialized.applied_event_ordinal > before,
4154            "the catch-up is what advances the projection"
4155        );
4156    }
4157
4158    #[cfg(unix)]
4159    #[tokio::test]
4160    async fn unresponsive_live_relay_worker_is_restarted_and_reconnected() {
4161        if std::env::var_os(AUTO_RESTART_TEST_CHILD).is_none() {
4162            run_in_isolated_child(
4163                AUTO_RESTART_TEST_CHILD,
4164                "unresponsive_live_relay_worker_is_restarted_and_reconnected",
4165            );
4166            return;
4167        }
4168        // Alone in this child process, so it installs the one writer.
4169        let _writer = hel::hel_database::install_isolated_test_writer();
4170        fail_if_the_actor_stalls("unresponsive live relay worker was never restarted");
4171        register_leased_relay_session();
4172        let relay_root = tempfile::tempdir().unwrap();
4173        let restarted = relay_root.path().join("worker-restarted");
4174        let script = format!(
4175            "if [ ! -f \"${AUTO_RESTART_MARKER}\" ]; then IFS= read -r _; exit 0; fi; \
4176             \"$0\" --exact {} --nocapture | grep --line-buffered '^{{'",
4177            exact_test_name("leased_relay_child_serves_stdio")
4178        );
4179        let mut spec = CommandSpec::new(
4180            "sh",
4181            [
4182                "-c".to_owned(),
4183                script,
4184                std::env::current_exe()
4185                    .unwrap()
4186                    .to_string_lossy()
4187                    .into_owned(),
4188            ],
4189        )
4190        .purpose("test restartable relay");
4191        spec.env.insert(
4192            LEASED_RELAY_ROOT.to_owned(),
4193            relay_root.path().to_string_lossy().into_owned(),
4194        );
4195        spec.env.insert(
4196            AUTO_RESTART_MARKER.to_owned(),
4197            restarted.to_string_lossy().into_owned(),
4198        );
4199        let worker_recovery = WorkerRecoveryPlan {
4200            target: None,
4201            liveness_probe: CommandSpec::new("printf", ["alive\n"])
4202                .purpose("probe test relay worker"),
4203            binary_refresh: None,
4204            launch_refresh: None,
4205            restart: CommandPlan {
4206                description: "restart test relay worker".into(),
4207                commands: vec![
4208                    CommandSpec::new("touch", [restarted.to_string_lossy().into_owned()])
4209                        .purpose("restart test relay worker"),
4210                ],
4211            },
4212        };
4213        let target = RelaySessionTarget {
4214            session_id: LEASED_RELAY_SESSION.to_owned(),
4215            spec,
4216            worker_recovery: Some(worker_recovery),
4217            project_memory: None,
4218        };
4219        let (_commands_tx, commands_rx) = mpsc::channel(4);
4220        let (_releases_tx, releases_rx) = mpsc::unbounded_channel();
4221        let (_retirement_tx, retirement_rx) = watch::channel(false);
4222        let (view_tx, mut view_rx) = watch::channel(ManagedSessionView::default());
4223        let (updates_tx, _updates_rx) = coalesced_update_channel();
4224        tokio::spawn(run_session_actor(
4225            target,
4226            commands_rx,
4227            releases_rx,
4228            retirement_rx,
4229            view_tx,
4230            updates_tx,
4231        ));
4232
4233        tokio::time::timeout(Duration::from_secs(20), async {
4234            loop {
4235                view_rx.changed().await.unwrap();
4236                let view = view_rx.borrow_and_update().clone();
4237                if view.connected {
4238                    assert!(restarted.exists(), "the restart plan did not run");
4239                    assert!(view.error.is_none());
4240                    return;
4241                }
4242            }
4243        })
4244        .await
4245        .unwrap_or_else(|_| panic!("relay stayed disconnected: {:?}", view_rx.borrow().error));
4246    }
4247
4248    /// A deferred submission that is never answered would hang the suite
4249    /// instead of failing it, so turn a stall into a hard error.
4250    #[cfg(unix)]
4251    fn fail_if_the_actor_stalls(reason: &'static str) {
4252        std::thread::spawn(move || {
4253            std::thread::sleep(Duration::from_secs(60));
4254            eprintln!("{reason}");
4255            std::process::exit(101);
4256        });
4257    }
4258
4259    /// A relay target served by this test binary over stdio.
4260    #[cfg(unix)]
4261    fn leased_relay_target(relay_root: &std::path::Path) -> RelaySessionTarget {
4262        // `RelayClient` parses every stdout line as JSON, so libtest's own
4263        // progress lines are dropped before they reach the protocol reader.
4264        let script = format!(
4265            "\"$0\" --exact {} --nocapture | grep --line-buffered '^{{'",
4266            exact_test_name("leased_relay_child_serves_stdio")
4267        );
4268        let mut spec = CommandSpec::new(
4269            "sh",
4270            [
4271                "-c".to_owned(),
4272                script,
4273                std::env::current_exe()
4274                    .unwrap()
4275                    .to_string_lossy()
4276                    .into_owned(),
4277            ],
4278        )
4279        .purpose("test leased relay");
4280        spec.env.insert(
4281            LEASED_RELAY_ROOT.to_owned(),
4282            relay_root.to_string_lossy().into_owned(),
4283        );
4284        RelaySessionTarget {
4285            session_id: LEASED_RELAY_SESSION.to_owned(),
4286            spec,
4287            worker_recovery: None,
4288            project_memory: None,
4289        }
4290    }
4291
4292    /// Register the session the projection writes to. `apply_projection_event`
4293    /// rejects events for sessions the controller database does not know.
4294    #[cfg(unix)]
4295    fn register_leased_relay_session() {
4296        hel::hel_database::save_session(&hel::hel_state::SessionRecord {
4297            workspace_id: hel::hel_workspace::DEFAULT_WORKSPACE_ID.to_owned(),
4298            archived: false,
4299            container_cpus: None,
4300            container_memory: None,
4301            id: LEASED_RELAY_SESSION.into(),
4302            title: "leased relay".into(),
4303            harness_kind: hel::hel_config::HarnessKind::Codex,
4304            last_profile: "codex".into(),
4305            bundle_id: "project".into(),
4306            project_directory: None,
4307            managed_worktree: None,
4308            target_template_id: "podman".into(),
4309            resource_allocation: None,
4310            additional_mounts: Vec::new(),
4311            state: hel::hel_state::SessionState::Running,
4312            target: None,
4313            native_session_id: None,
4314            acp_session_title: None,
4315            session_title_override: None,
4316            created_at: "2026-08-12T00:00:00Z".into(),
4317            updated_at: "2026-08-12T00:00:00Z".into(),
4318            viewed_through_event_ordinal: 0,
4319            draft_input: String::new(),
4320            last_error: None,
4321            last_checkpoint_error: None,
4322            checkpoint: None,
4323        })
4324        .expect("register the test session");
4325    }
4326
4327    #[cfg(unix)]
4328    struct LeasedActor {
4329        commands: mpsc::Sender<ActorCommand>,
4330        releases: mpsc::UnboundedSender<ReturnedConnection>,
4331        retirement: watch::Sender<bool>,
4332        _views: watch::Receiver<ManagedSessionView>,
4333        _updates: SessionManagerUpdates,
4334        _relay_root: tempfile::TempDir,
4335    }
4336
4337    /// Start an actor against a live relay and take its connection under lease.
4338    #[cfg(unix)]
4339    async fn lease_a_live_actor() -> (LeasedActor, u64, StandaloneSession) {
4340        register_leased_relay_session();
4341        let relay_root = tempfile::tempdir().unwrap();
4342        let (commands_tx, commands_rx) = mpsc::channel(4);
4343        let (releases_tx, releases_rx) = mpsc::unbounded_channel();
4344        let (retirement_tx, retirement_rx) = watch::channel(false);
4345        let (view_tx, view_rx) = watch::channel(ManagedSessionView::default());
4346        let (updates_tx, updates_rx) = coalesced_update_channel();
4347        tokio::spawn(run_session_actor(
4348            leased_relay_target(relay_root.path()),
4349            commands_rx,
4350            releases_rx,
4351            retirement_rx,
4352            view_tx,
4353            updates_tx,
4354        ));
4355
4356        let (reply, response) = oneshot::channel();
4357        commands_tx
4358            .send(ActorCommand::Lease { reply })
4359            .await
4360            .unwrap();
4361        let (lease_id, connection) = response
4362            .await
4363            .expect("actor answered the lease request")
4364            .expect("actor leased its relay connection");
4365        (
4366            LeasedActor {
4367                commands: commands_tx,
4368                releases: releases_tx,
4369                retirement: retirement_tx,
4370                _views: view_rx,
4371                _updates: updates_rx,
4372                _relay_root: relay_root,
4373            },
4374            lease_id,
4375            connection,
4376        )
4377    }
4378
4379    #[cfg(unix)]
4380    async fn submit_a_deferred_prompt(
4381        actor: &LeasedActor,
4382    ) -> oneshot::Receiver<std::result::Result<u64, String>> {
4383        let (reply, mut response) = oneshot::channel();
4384        actor
4385            .commands
4386            .send(ActorCommand::Submit {
4387                command_id: new_command_id("prompt").unwrap(),
4388                command: RelayCommand::Prompt {
4389                    prompt: vec![ContentBlock::Text(TextContent::new("hello"))],
4390                },
4391                reply,
4392            })
4393            .await
4394            .unwrap();
4395        assert!(
4396            tokio::time::timeout(Duration::from_millis(300), &mut response)
4397                .await
4398                .is_err(),
4399            "a leased actor must hold the prompt instead of answering it"
4400        );
4401        response
4402    }
4403
4404    #[cfg(unix)]
4405    #[tokio::test]
4406    async fn prompt_submitted_during_lease_is_delivered_after_release() {
4407        if std::env::var_os(DEFERRED_SUBMIT_TEST_CHILD).is_none() {
4408            run_in_isolated_child(
4409                DEFERRED_SUBMIT_TEST_CHILD,
4410                "prompt_submitted_during_lease_is_delivered_after_release",
4411            );
4412            return;
4413        }
4414        // Alone in this child process, so it installs the one writer.
4415        let _writer = hel::hel_database::install_isolated_test_writer();
4416        fail_if_the_actor_stalls("prompt deferred during a lease was never delivered");
4417
4418        let (actor, lease_id, connection) = lease_a_live_actor().await;
4419        let response = submit_a_deferred_prompt(&actor).await;
4420
4421        actor
4422            .releases
4423            .send(ReturnedConnection {
4424                lease_id,
4425                connection: Some(connection),
4426            })
4427            .unwrap();
4428
4429        let ordinal = response
4430            .await
4431            .expect("actor answered the deferred prompt")
4432            .expect("deferred prompt reached the relay");
4433        assert!(
4434            ordinal > 0,
4435            "relay accepted the prompt at ordinal {ordinal}"
4436        );
4437    }
4438
4439    #[cfg(unix)]
4440    #[tokio::test]
4441    async fn returned_lease_publishes_what_it_learned_while_it_held_the_connection() {
4442        if std::env::var_os(RETURNED_LEASE_VIEW_TEST_CHILD).is_none() {
4443            run_in_isolated_child(
4444                RETURNED_LEASE_VIEW_TEST_CHILD,
4445                "returned_lease_publishes_what_it_learned_while_it_held_the_connection",
4446            );
4447            return;
4448        }
4449        // Alone in this child process, so it installs the one writer.
4450        let _writer = hel::hel_database::install_isolated_test_writer();
4451        fail_if_the_actor_stalls("a returned lease never republished its session");
4452
4453        let (actor, lease_id, mut connection) = lease_a_live_actor().await;
4454        let mut views = actor._views.clone();
4455        // The lease applies these events itself, so the actor's own next sync
4456        // has nothing left to catch up on.
4457        let ordinal = connection
4458            .submit(
4459                new_command_id("prompt").unwrap(),
4460                RelayCommand::Prompt {
4461                    prompt: vec![ContentBlock::Text(TextContent::new("hello"))],
4462                },
4463            )
4464            .await
4465            .unwrap();
4466        assert!(views.borrow_and_update().snapshot.is_none());
4467
4468        actor
4469            .releases
4470            .send(ReturnedConnection {
4471                lease_id,
4472                connection: Some(connection),
4473            })
4474            .unwrap();
4475
4476        views.changed().await.unwrap();
4477        let snapshot = views
4478            .borrow_and_update()
4479            .snapshot
4480            .clone()
4481            .expect("the returned connection republished its session");
4482        assert!(
4483            snapshot.materialized.applied_event_ordinal >= ordinal,
4484            "published frontier {} is behind the leased submission at {ordinal}",
4485            snapshot.materialized.applied_event_ordinal
4486        );
4487    }
4488
4489    #[cfg(unix)]
4490    #[tokio::test]
4491    async fn retirement_rejects_prompts_deferred_during_lease() {
4492        if std::env::var_os(RETIRED_SUBMIT_TEST_CHILD).is_none() {
4493            run_in_isolated_child(
4494                RETIRED_SUBMIT_TEST_CHILD,
4495                "retirement_rejects_prompts_deferred_during_lease",
4496            );
4497            return;
4498        }
4499        // Alone in this child process, so it installs the one writer.
4500        let _writer = hel::hel_database::install_isolated_test_writer();
4501        fail_if_the_actor_stalls("prompt deferred during a lease was never answered");
4502
4503        let (actor, lease_id, connection) = lease_a_live_actor().await;
4504        let response = submit_a_deferred_prompt(&actor).await;
4505
4506        actor.retirement.send(true).unwrap();
4507        actor
4508            .releases
4509            .send(ReturnedConnection {
4510                lease_id,
4511                connection: Some(connection),
4512            })
4513            .unwrap();
4514
4515        let error = response
4516            .await
4517            .expect("actor answered the deferred prompt")
4518            .expect_err("a retiring actor must not deliver the prompt");
4519        assert!(
4520            error.contains("session target is changing"),
4521            "unexpected rejection: {error}"
4522        );
4523    }
4524
4525    #[test]
4526    fn projection_integrity_failure_is_detected_only_for_integrity_errors() {
4527        let integrity = anyhow::Error::from(ProjectionIntegrityError(
4528            "transcript item \"tool:call-1\" changed immutable identity fields".into(),
4529        ))
4530        .context("apply projection event");
4531        assert!(projection_integrity_failure(&integrity));
4532
4533        let concurrent = anyhow::Error::from(ProjectionAdvancedError { event_ordinal: 7 });
4534        assert!(!projection_integrity_failure(&concurrent));
4535
4536        let unreachable = anyhow::anyhow!("connection refused").context("connect relay proxy");
4537        assert!(!projection_integrity_failure(&unreachable));
4538    }
4539
4540    #[test]
4541    fn dashboard_updates_keep_only_the_latest_view_per_session() {
4542        let (sender, mut receiver) = coalesced_update_channel();
4543        for revision in 0..1_000 {
4544            sender.send(SessionManagerUpdate {
4545                session_id: "session-1".into(),
4546                view: ManagedSessionView {
4547                    error: Some(ViewError::Unreachable(format!("revision-{revision}"))),
4548                    ..ManagedSessionView::default()
4549                },
4550            });
4551        }
4552        sender.send(SessionManagerUpdate {
4553            session_id: "session-2".into(),
4554            view: ManagedSessionView {
4555                error: Some(ViewError::Unreachable("other".into())),
4556                ..ManagedSessionView::default()
4557            },
4558        });
4559
4560        assert_eq!(
4561            sender
4562                .pending
4563                .lock()
4564                .expect("session update coalescer poisoned")
4565                .len(),
4566            2
4567        );
4568        let updates = [receiver.try_recv().unwrap(), receiver.try_recv().unwrap()]
4569            .into_iter()
4570            .map(|update| (update.session_id, update.view.error.unwrap()))
4571            .collect::<BTreeMap<_, _>>();
4572        assert_eq!(updates["session-1"].detail(), "revision-999");
4573        assert_eq!(updates["session-2"].detail(), "other");
4574        assert!(receiver.try_recv().is_err());
4575    }
4576
4577    #[tokio::test]
4578    async fn remote_session_manager_fans_out_views_and_forwards_commands() {
4579        let mut remote = spawn_remote_session_manager().unwrap();
4580        remote.targets.send_replace(vec![target("unused")]);
4581        remote
4582            .publisher
4583            .publish("session-1".into(), view_at_ordinal(7))
4584            .await
4585            .unwrap();
4586
4587        let session = remote
4588            .control
4589            .wait_for_session("session-1", Duration::from_secs(1))
4590            .await
4591            .unwrap();
4592        assert_eq!(
4593            session
4594                .view()
4595                .snapshot
4596                .as_ref()
4597                .unwrap()
4598                .materialized
4599                .applied_event_ordinal,
4600            7
4601        );
4602
4603        let submitted = session
4604            .enqueue_submit("prompt-1".into(), RelayCommand::Cancel)
4605            .await
4606            .unwrap();
4607        let request = remote.requests.recv().await.unwrap();
4608        match request {
4609            RemoteSessionRequest::Submit {
4610                session_id,
4611                command_id,
4612                command: RelayCommand::Cancel,
4613                reply,
4614            } => {
4615                assert_eq!(session_id, "session-1");
4616                assert_eq!(command_id, "prompt-1");
4617                reply.send(Ok(8)).unwrap();
4618            }
4619            _ => panic!("unexpected remote session request"),
4620        }
4621        assert_eq!(submitted.wait().await.unwrap(), 8);
4622        remote.shutdown.shutdown().await.unwrap();
4623    }
4624}