1use std::collections::{BTreeMap, VecDeque};
4use std::path::PathBuf;
5use std::sync::{Arc, Mutex};
6use std::time::{Duration, Instant};
7
8use anyhow::{Context, Result, bail, ensure};
9use tokio::sync::{mpsc, oneshot, watch};
10
11use crate::hel_worker_client::{
12 RelayAttachment, RelayClient, RelayEventPage, RelayRejected, RelayTransportDead,
13 StartedReviewer,
14};
15use hel::hel_archive::verify_archive_streaming;
16use hel::hel_credentials::{CredentialSyncSignal, relay_event_credential_sync_reason};
17use hel::hel_database::{
18 ProjectionApplyOutcome, ProjectionIntegrityError, apply_projection_page,
19 save_materialized_session,
20};
21use hel::hel_elicitation::ElicitationResponse;
22use hel::hel_projection::{
23 ProjectionIndex, apply_committed_projection_event_indexed, materialized_session_from_canonical,
24 project_relay_event_indexed,
25};
26use hel::hel_state::{ManagedSessionSnapshot, MaterializedSession};
27use hel::hel_targets::{
28 CancellableProcessExecutor, CommandExecutor, CommandPlan, CommandSpec, TargetLocator,
29 TargetRecoveryOutcome, TargetRecoveryPlan, ensure_recovery_target_running,
30};
31use hel::hel_worker::{RelayCommand, RelayCursor, RelayOperationalState};
32use hel::hel_worker_launch::ReviewerLaunchConfig;
33
34const SESSION_SYNC_INTERVAL: Duration = Duration::from_millis(150);
35const PROJECTION_TRANSACTION_EVENT_BUDGET: usize = 128;
39const RECONNECT_INTERVAL: Duration = Duration::from_secs(1);
40const RECONNECT_BACKOFF_CEILING: Duration = Duration::from_secs(30);
43const UNREACHABLE_FAILURE_THRESHOLD: u32 = 2;
44const WORKER_RESTART_TIMEOUT: Duration = Duration::from_secs(30);
45const WORKER_RESTART_COOLDOWN: Duration = Duration::from_secs(60);
46const SESSION_MANAGER_SHUTDOWN_GRACE: Duration = Duration::from_millis(750);
47
48#[derive(Debug)]
49struct ProjectionAdvancedError {
50 event_ordinal: u64,
51}
52
53impl std::fmt::Display for ProjectionAdvancedError {
54 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
55 write!(
56 formatter,
57 "another projector committed relay event {} first",
58 self.event_ordinal
59 )
60 }
61}
62
63impl std::error::Error for ProjectionAdvancedError {}
64
65fn reconnect_delay(failures: u32) -> Duration {
68 let doubling = failures.saturating_sub(1).min(u32::BITS - 1);
69 RECONNECT_INTERVAL
70 .saturating_mul(1_u32 << doubling)
71 .min(RECONNECT_BACKOFF_CEILING)
72}
73
74#[derive(Debug, Clone, PartialEq, Eq)]
75pub struct RelaySessionTarget {
76 pub session_id: String,
77 pub spec: CommandSpec,
78 pub worker_recovery: Option<WorkerRecoveryPlan>,
82 pub project_memory: Option<ProjectMemorySyncTarget>,
83}
84
85#[derive(Debug, Clone, PartialEq, Eq)]
86pub struct ProjectMemorySyncTarget {
87 pub canonical_root: std::path::PathBuf,
88}
89
90#[derive(Debug, Clone, PartialEq, Eq)]
91pub struct WorkerRecoveryPlan {
92 pub target: Option<TargetRecoveryPlan>,
93 pub liveness_probe: CommandSpec,
94 pub binary_refresh: Option<WorkerBinaryRefresh>,
98 pub launch_refresh: Option<WorkerLaunchRefreshPlan>,
101 pub restart: CommandPlan,
102}
103
104#[derive(Debug, Clone, PartialEq, Eq)]
113pub enum WorkerBinaryRefresh {
114 Prepared(WorkerBinaryRefreshPlan),
115 Remote(RemoteWorkerBinaryRefresh),
116}
117
118#[derive(Debug, Clone, PartialEq, Eq)]
119pub struct WorkerBinaryRefreshPlan {
120 pub source: PathBuf,
121 pub installed_digest: CommandSpec,
122 pub replace: CommandPlan,
123}
124
125#[derive(Debug, Clone, PartialEq, Eq)]
129pub struct RemoteWorkerBinaryRefresh {
130 pub locator: TargetLocator,
131 pub session_id: String,
132 pub installed_digest: CommandSpec,
133}
134
135#[derive(Debug, Clone, PartialEq, Eq)]
136pub struct WorkerLaunchRefreshPlan {
137 pub expected_sha256: String,
138 pub installed_digest: CommandSpec,
139 pub replace: CommandPlan,
140}
141
142pub(crate) fn worker_connect_needs_restart(error: &anyhow::Error) -> bool {
149 RelayTransportDead::marks(error)
150}
151
152fn worker_connect_allows_live_restart(error: &anyhow::Error) -> bool {
153 RelayTransportDead::marks_failed_handshake(error)
154}
155
156#[derive(Debug, Clone, Copy, PartialEq, Eq)]
157enum WorkerRecoveryOutcome {
158 Alive,
159 Starting,
160 TargetMissing,
161 RestartedDead,
162 RestartedUnresponsive,
163}
164
165fn refresh_worker_binary_if_stale(
166 executor: &impl CommandExecutor,
167 refresh: Option<&WorkerBinaryRefresh>,
168) -> Result<()> {
169 match refresh {
170 None => Ok(()),
171 Some(WorkerBinaryRefresh::Prepared(plan)) => {
172 let expected = hel::hel_worker_launch::worker_executable_digest(&plan.source)?;
173 if installed_digest_matches(executor, &plan.installed_digest, &expected) {
174 return Ok(());
175 }
176 plan.replace
177 .execute(executor)
178 .context("replace stale relay worker binary")?;
179 Ok(())
180 }
181 Some(WorkerBinaryRefresh::Remote(refresh)) => {
184 crate::hel_controller::refresh_remote_worker_binary_if_stale(executor, refresh)
185 }
186 }
187}
188
189fn installed_digest_matches(
190 executor: &impl CommandExecutor,
191 command: &CommandSpec,
192 expected: &str,
193) -> bool {
194 executor.execute(command).as_ref().is_ok_and(|output| {
195 output.status == 0
196 && String::from_utf8_lossy(&output.stdout)
197 .split_whitespace()
198 .next()
199 .is_some_and(|digest| digest.eq_ignore_ascii_case(expected))
200 })
201}
202
203fn refresh_worker_launch_if_stale(
204 executor: &impl CommandExecutor,
205 plan: Option<&WorkerLaunchRefreshPlan>,
206) -> Result<()> {
207 let Some(plan) = plan else {
208 return Ok(());
209 };
210 if installed_digest_matches(executor, &plan.installed_digest, &plan.expected_sha256) {
211 return Ok(());
212 }
213 plan.replace
214 .execute(executor)
215 .context("replace stale relay worker launch config")?;
216 Ok(())
217}
218
219async fn recover_worker(
220 plan: WorkerRecoveryPlan,
221 restart_unresponsive: bool,
222) -> Result<WorkerRecoveryOutcome> {
223 tokio::task::spawn_blocking(move || {
224 let executor = CancellableProcessExecutor::with_timeout(WORKER_RESTART_TIMEOUT);
225 if ensure_recovery_target_running(&executor, plan.target.as_ref())
226 .context("restore relay worker target")?
227 == TargetRecoveryOutcome::Missing
228 {
229 return Ok(WorkerRecoveryOutcome::TargetMissing);
230 }
231 let output = executor
232 .execute(&plan.liveness_probe)
233 .context("probe relay worker liveness")?;
234 if output.status != 0 {
235 bail!(
236 "{} failed with status {}: {}",
237 plan.liveness_probe.purpose,
238 output.status,
239 String::from_utf8_lossy(&output.stderr).trim()
240 );
241 }
242 match String::from_utf8_lossy(&output.stdout).trim() {
243 "starting" => Ok(WorkerRecoveryOutcome::Starting),
244 "alive" if !restart_unresponsive => Ok(WorkerRecoveryOutcome::Alive),
245 "alive" => {
246 refresh_worker_binary_if_stale(&executor, plan.binary_refresh.as_ref())?;
247 refresh_worker_launch_if_stale(&executor, plan.launch_refresh.as_ref())?;
248 plan.restart.execute(&executor)?;
249 Ok(WorkerRecoveryOutcome::RestartedUnresponsive)
250 }
251 "dead" => {
252 refresh_worker_binary_if_stale(&executor, plan.binary_refresh.as_ref())?;
253 refresh_worker_launch_if_stale(&executor, plan.launch_refresh.as_ref())?;
254 plan.restart.execute(&executor)?;
255 Ok(WorkerRecoveryOutcome::RestartedDead)
256 }
257 output => bail!("worker liveness probe returned unexpected output {output:?}"),
258 }
259 })
260 .await
261 .context("worker recovery task failed")?
262}
263
264#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
268#[serde(tag = "kind", content = "detail", rename_all = "snake_case")]
269pub enum ViewError {
270 Unreachable(String),
271 TargetMissing(String),
272 ProjectionIntegrity(String),
273}
274
275impl ViewError {
276 pub fn detail(&self) -> &str {
277 match self {
278 Self::Unreachable(detail)
279 | Self::TargetMissing(detail)
280 | Self::ProjectionIntegrity(detail) => detail,
281 }
282 }
283}
284
285#[derive(Debug, Clone, PartialEq, Default)]
286pub struct ManagedSessionView {
287 pub snapshot: Option<ManagedSessionSnapshot>,
288 pub connected: bool,
289 pub error: Option<ViewError>,
290}
291
292#[derive(Debug, Clone)]
293pub struct SessionManagerUpdate {
294 pub session_id: String,
295 pub view: ManagedSessionView,
296}
297
298pub struct SessionManagerChannels {
299 pub targets: watch::Sender<Vec<RelaySessionTarget>>,
300 pub control: SessionManagerControl,
301 pub updates: SessionManagerUpdates,
302 pub shutdown: SessionManagerShutdown,
303}
304
305pub struct RemoteSessionManagerChannels {
311 pub targets: watch::Sender<Vec<RelaySessionTarget>>,
312 pub control: SessionManagerControl,
313 pub updates: SessionManagerUpdates,
314 pub shutdown: SessionManagerShutdown,
315 pub publisher: RemoteSessionPublisher,
316 pub requests: RemoteSessionRequests,
317}
318
319#[derive(Clone)]
320pub struct RemoteSessionPublisher {
321 updates: mpsc::UnboundedSender<RemoteManagerUpdate>,
322}
323
324impl RemoteSessionPublisher {
325 pub async fn publish(&self, session_id: String, view: ManagedSessionView) -> Result<()> {
326 self.updates
327 .send(RemoteManagerUpdate::Publish { session_id, view })
328 .context("remote session manager stopped")
329 }
330
331 pub fn try_publish(&self, session_id: String, view: ManagedSessionView) -> Result<()> {
332 self.updates
333 .send(RemoteManagerUpdate::Publish { session_id, view })
334 .context("remote session manager update queue is unavailable")
335 }
336}
337
338pub struct RemoteSessionRequests {
339 requests: mpsc::Receiver<RemoteSessionRequest>,
340}
341
342impl RemoteSessionRequests {
343 pub async fn recv(&mut self) -> Option<RemoteSessionRequest> {
344 self.requests.recv().await
345 }
346}
347
348#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
353#[serde(rename_all = "snake_case")]
354pub enum ReviewerAction {
355 Start {
356 config: Box<ReviewerLaunchConfig>,
357 },
358 Submit {
359 command_id: String,
360 command: RelayCommand,
361 },
362 Attach {
363 after_ordinal: u64,
364 after_digest: String,
365 },
366 Acknowledge {
367 through_ordinal: u64,
368 through_digest: String,
369 },
370 Status,
371 RespondElicitation {
374 elicitation_id: String,
375 response: ElicitationResponse,
376 },
377 Pause,
378 CaptureDelta {
380 baselines: std::collections::BTreeMap<std::path::PathBuf, String>,
381 },
382 AdvanceBaseline {
384 trees: std::collections::BTreeMap<std::path::PathBuf, String>,
385 },
386 AnalyzeDelta {
388 repositories: Vec<hel::hel_worker::AnalyzeDeltaRepository>,
389 },
390 TakeLaneDispatches,
392}
393
394impl ReviewerAction {
395 pub const fn operation_name(&self) -> &'static str {
396 match self {
397 Self::Start { .. } => "reviewer_start",
398 Self::Submit { .. } => "reviewer_submit",
399 Self::Attach { .. } => "reviewer_attach",
400 Self::Acknowledge { .. } => "reviewer_acknowledge",
401 Self::Status => "reviewer_status",
402 Self::RespondElicitation { .. } => "reviewer_respond_elicitation",
403 Self::Pause => "reviewer_pause",
404 Self::CaptureDelta { .. } => "reviewer_capture_delta",
405 Self::AdvanceBaseline { .. } => "reviewer_advance_baseline",
406 Self::AnalyzeDelta { .. } => "reviewer_analyze_delta",
407 Self::TakeLaneDispatches => "reviewer_take_lane_dispatches",
408 }
409 }
410}
411
412#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
414#[serde(rename_all = "snake_case")]
415pub enum ReviewerOutcome {
416 Started(Box<StartedReviewer>),
417 Accepted {
418 ordinal: u64,
419 },
420 Attached(Box<RelayAttachment>),
421 Acknowledged(RelayCursor),
422 Status(Box<RelayOperationalState>),
423 ElicitationResolved,
424 Paused,
425 Delta {
427 repositories: Vec<hel::hel_worker::RepoDelta>,
428 },
429 BaselineAdvanced,
430 ChangedFunctions {
432 packet: String,
433 },
434 LaneDispatches {
436 requests: Vec<hel::hel_review::lanes::ReviewSubagentRequest>,
437 },
438}
439
440pub enum RemoteSessionRequest {
441 Submit {
442 session_id: String,
443 command_id: String,
444 command: RelayCommand,
445 reply: oneshot::Sender<std::result::Result<u64, String>>,
446 },
447 Sync {
448 session_id: String,
449 reply: oneshot::Sender<std::result::Result<(), String>>,
450 },
451 RespondElicitation {
452 session_id: String,
453 elicitation_id: String,
454 response: ElicitationResponse,
455 reply: oneshot::Sender<std::result::Result<(), String>>,
456 },
457 Reviewer {
458 session_id: String,
459 role: Option<String>,
461 action: ReviewerAction,
462 reply: oneshot::Sender<std::result::Result<ReviewerOutcome, String>>,
463 },
464}
465
466impl RemoteSessionRequest {
467 pub fn session_id(&self) -> &str {
470 match self {
471 Self::Submit { session_id, .. }
472 | Self::Sync { session_id, .. }
473 | Self::RespondElicitation { session_id, .. }
474 | Self::Reviewer { session_id, .. } => session_id,
475 }
476 }
477}
478
479#[derive(Default)]
489pub struct SessionRequestOrder {
490 latest: std::collections::HashMap<String, tokio::task::JoinHandle<()>>,
491}
492
493impl SessionRequestOrder {
494 #[must_use]
495 pub fn new() -> Self {
496 Self::default()
497 }
498
499 pub fn dispatch<F, Fut>(&mut self, request: RemoteSessionRequest, forward: F)
502 where
503 F: FnOnce(RemoteSessionRequest) -> Fut + Send + 'static,
504 Fut: std::future::Future<Output = ()> + Send,
505 {
506 self.latest.retain(|_, handle| !handle.is_finished());
510 let session_id = request.session_id().to_owned();
511 let previous = self.latest.remove(&session_id);
512 let handle = tokio::spawn(async move {
513 if let Some(previous) = previous {
514 let _ = previous.await;
518 }
519 forward(request).await;
520 });
521 self.latest.insert(session_id, handle);
522 }
523}
524
525pub struct SessionManagerShutdown {
531 signal: Option<oneshot::Sender<()>>,
532 task: Option<tokio::task::JoinHandle<()>>,
533}
534
535impl SessionManagerShutdown {
536 pub async fn shutdown(mut self) -> Result<()> {
537 if let Some(signal) = self.signal.take() {
538 let _ = signal.send(());
539 }
540 if let Some(task) = self.task.take() {
541 task.await.context("session manager shutdown task failed")?;
542 }
543 Ok(())
544 }
545}
546
547impl Drop for SessionManagerShutdown {
548 fn drop(&mut self) {
549 if let Some(signal) = self.signal.take() {
550 let _ = signal.send(());
551 }
552 if let Some(task) = self.task.take() {
553 task.abort();
554 }
555 }
556}
557
558#[derive(Clone)]
559struct CoalescedUpdateSender {
560 pending: Arc<Mutex<BTreeMap<String, SessionManagerUpdate>>>,
561 wake: mpsc::Sender<()>,
562}
563
564pub struct SessionManagerUpdates {
567 pending: Arc<Mutex<BTreeMap<String, SessionManagerUpdate>>>,
568 wake: mpsc::Receiver<()>,
569}
570
571impl CoalescedUpdateSender {
572 fn send(&self, update: SessionManagerUpdate) {
573 if self.wake.is_closed() {
574 return;
575 }
576 self.pending
577 .lock()
578 .expect("session update coalescer poisoned")
579 .insert(update.session_id.clone(), update);
580 let _ = self.wake.try_send(());
581 }
582}
583
584impl SessionManagerUpdates {
585 fn pop_pending(&self) -> Option<SessionManagerUpdate> {
586 self.pending
587 .lock()
588 .expect("session update coalescer poisoned")
589 .pop_first()
590 .map(|(_, update)| update)
591 }
592
593 pub async fn recv(&mut self) -> Option<SessionManagerUpdate> {
594 loop {
595 if let Some(update) = self.pop_pending() {
596 return Some(update);
597 }
598 self.wake.recv().await?;
599 }
600 }
601
602 pub fn try_recv(
603 &mut self,
604 ) -> std::result::Result<SessionManagerUpdate, mpsc::error::TryRecvError> {
605 if let Some(update) = self.pop_pending() {
606 return Ok(update);
607 }
608 self.wake.try_recv()?;
609 self.pop_pending().ok_or(mpsc::error::TryRecvError::Empty)
610 }
611}
612
613fn coalesced_update_channel() -> (CoalescedUpdateSender, SessionManagerUpdates) {
614 let pending = Arc::new(Mutex::new(BTreeMap::new()));
615 let (wake_tx, wake_rx) = mpsc::channel(1);
616 (
617 CoalescedUpdateSender {
618 pending: pending.clone(),
619 wake: wake_tx,
620 },
621 SessionManagerUpdates {
622 pending,
623 wake: wake_rx,
624 },
625 )
626}
627
628#[derive(Clone)]
629pub struct SessionManagerControl {
630 commands: mpsc::Sender<ManagerCommand>,
631}
632
633#[derive(Clone, Debug)]
634pub struct ManagedSessionHandle {
635 session_id: String,
636 commands: mpsc::Sender<ActorCommand>,
637 releases: mpsc::UnboundedSender<ReturnedConnection>,
638 view: watch::Receiver<ManagedSessionView>,
639}
640
641pub struct ManagedSessionLease {
651 session_id: String,
652 lease_id: Option<u64>,
653 connection: Option<StandaloneSession>,
654 releases: mpsc::UnboundedSender<ReturnedConnection>,
655}
656
657impl ManagedSessionLease {
658 pub fn connection_mut(&mut self) -> &mut StandaloneSession {
659 self.connection
660 .as_mut()
661 .expect("managed session lease has already been released")
662 }
663
664 pub fn replace_connection(&mut self, connection: StandaloneSession) {
667 drop(self.connection.take());
668 self.connection = Some(connection);
669 }
670
671 pub fn release(mut self) {
672 let lease_id = self
673 .lease_id
674 .take()
675 .expect("managed session lease has already been released");
676 let connection = self.connection.take();
677 if let Err(error) = self.releases.send(ReturnedConnection {
678 lease_id,
679 connection,
680 }) {
681 tracing::warn!(
682 session_id = %self.session_id,
683 operation = "lease_release",
684 %error,
685 "session actor stopped before receiving released relay connection"
686 );
687 }
688 }
689}
690
691impl Drop for ManagedSessionLease {
692 fn drop(&mut self) {
693 let Some(lease_id) = self.lease_id.take() else {
694 return;
695 };
696 drop(self.connection.take());
699 if let Err(error) = self.releases.send(ReturnedConnection {
700 lease_id,
701 connection: None,
702 }) {
703 tracing::warn!(
704 session_id = %self.session_id,
705 operation = "lease_drop",
706 %error,
707 "session actor stopped before receiving dropped relay lease"
708 );
709 }
710 }
711}
712
713impl ManagedSessionHandle {
714 pub fn session_id(&self) -> &str {
715 &self.session_id
716 }
717
718 pub fn view(&self) -> ManagedSessionView {
719 self.view.borrow().clone()
720 }
721
722 pub fn is_stopped(&self) -> bool {
726 self.commands.is_closed()
727 }
728
729 pub fn has_changed(&self) -> Result<bool> {
730 self.view.has_changed().context("session manager stopped")
731 }
732
733 pub async fn changed(&mut self) -> Result<ManagedSessionView> {
734 self.view
735 .changed()
736 .await
737 .context("session manager stopped")?;
738 Ok(self.view())
739 }
740
741 pub async fn submit(&self, command_id: String, command: RelayCommand) -> Result<u64> {
742 self.enqueue_submit(command_id, command).await?.wait().await
743 }
744
745 pub async fn enqueue_submit(
746 &self,
747 command_id: String,
748 command: RelayCommand,
749 ) -> Result<PendingRelaySubmit> {
750 let (reply, response) = oneshot::channel();
751 self.commands
752 .send(ActorCommand::Submit {
753 command_id,
754 command,
755 reply,
756 })
757 .await
758 .context("session manager stopped")?;
759 Ok(PendingRelaySubmit { response })
760 }
761
762 pub async fn sync_now(&self) -> Result<()> {
763 self.enqueue_sync().await?.wait().await
764 }
765
766 pub async fn respond_elicitation(
767 &self,
768 elicitation_id: String,
769 response: ElicitationResponse,
770 ) -> Result<()> {
771 let (reply, result) = oneshot::channel();
772 self.commands
773 .send(ActorCommand::RespondElicitation {
774 elicitation_id,
775 response,
776 reply,
777 })
778 .await
779 .context("session manager stopped")?;
780 result
781 .await
782 .context("session manager stopped")?
783 .map_err(anyhow::Error::msg)
784 }
785
786 pub async fn reviewer(&self, action: ReviewerAction) -> Result<ReviewerOutcome> {
792 self.reviewer_as(None, action).await
793 }
794
795 pub async fn reviewer_as(
799 &self,
800 role: Option<String>,
801 action: ReviewerAction,
802 ) -> Result<ReviewerOutcome> {
803 let (reply, result) = oneshot::channel();
804 self.commands
805 .send(ActorCommand::Reviewer {
806 role,
807 action,
808 reply,
809 })
810 .await
811 .context("session manager stopped")?;
812 result
813 .await
814 .context("session manager stopped")?
815 .map_err(anyhow::Error::msg)
816 }
817
818 pub async fn enqueue_sync(&self) -> Result<PendingRelaySync> {
819 let (reply, response) = oneshot::channel();
820 self.commands
821 .send(ActorCommand::Sync { reply })
822 .await
823 .context("session manager stopped")?;
824 Ok(PendingRelaySync { response })
825 }
826
827 pub async fn lease_connection(&self) -> Result<ManagedSessionLease> {
828 let (reply, response) = oneshot::channel();
829 self.commands
830 .send(ActorCommand::Lease { reply })
831 .await
832 .context("session manager stopped")?;
833 let (lease_id, connection) = response.await.context("session manager stopped")??;
834 Ok(ManagedSessionLease {
835 session_id: self.session_id.clone(),
836 lease_id: Some(lease_id),
837 connection: Some(connection),
838 releases: self.releases.clone(),
839 })
840 }
841}
842
843pub struct PendingRelaySubmit {
844 response: oneshot::Receiver<std::result::Result<u64, String>>,
845}
846
847impl PendingRelaySubmit {
848 pub async fn wait(self) -> Result<u64> {
849 self.response
850 .await
851 .context("session manager stopped")?
852 .map_err(anyhow::Error::msg)
853 }
854}
855
856pub struct PendingRelaySync {
857 response: oneshot::Receiver<std::result::Result<(), String>>,
858}
859
860impl PendingRelaySync {
861 pub async fn wait(self) -> Result<()> {
862 self.response
863 .await
864 .context("session manager stopped")?
865 .map_err(anyhow::Error::msg)
866 }
867}
868
869impl SessionManagerControl {
870 pub async fn session(&self, session_id: impl Into<String>) -> Result<ManagedSessionHandle> {
871 let session_id = session_id.into();
872 let (reply, response) = oneshot::channel();
873 self.commands
874 .send(ManagerCommand::Session {
875 session_id: session_id.clone(),
876 reply,
877 })
878 .await
879 .context("session manager stopped")?;
880 response
881 .await
882 .context("session manager stopped")?
883 .with_context(|| format!("session {session_id} is not managed"))
884 }
885
886 pub async fn wait_for_session(
887 &self,
888 session_id: &str,
889 timeout: Duration,
890 ) -> Result<ManagedSessionHandle> {
891 let deadline = tokio::time::Instant::now() + timeout;
892 loop {
893 match self.session(session_id.to_owned()).await {
894 Ok(handle) => return Ok(handle),
895 Err(error) if tokio::time::Instant::now() < deadline => {
896 tracing::trace!(session_id, "waiting for session actor: {error:#}");
897 tokio::time::sleep(Duration::from_millis(25)).await;
898 }
899 Err(error) => return Err(error),
900 }
901 }
902 }
903}
904
905enum ManagerCommand {
906 Session {
907 session_id: String,
908 reply: oneshot::Sender<Option<ManagedSessionHandle>>,
909 },
910}
911
912enum ActorCommand {
913 Submit {
914 command_id: String,
915 command: RelayCommand,
916 reply: oneshot::Sender<std::result::Result<u64, String>>,
917 },
918 Sync {
919 reply: oneshot::Sender<std::result::Result<(), String>>,
920 },
921 RespondElicitation {
922 elicitation_id: String,
923 response: ElicitationResponse,
924 reply: oneshot::Sender<std::result::Result<(), String>>,
925 },
926 Reviewer {
927 role: Option<String>,
928 action: ReviewerAction,
929 reply: oneshot::Sender<std::result::Result<ReviewerOutcome, String>>,
930 },
931 Lease {
935 reply: oneshot::Sender<Result<(u64, StandaloneSession)>>,
936 },
937}
938
939impl ActorCommand {
940 fn operation_name(&self) -> &'static str {
941 match self {
942 Self::Submit { .. } => "submit",
943 Self::Sync { .. } => "sync",
944 Self::RespondElicitation { .. } => "respond_elicitation",
945 Self::Reviewer { action, .. } => action.operation_name(),
946 Self::Lease { .. } => "lease",
947 }
948 }
949
950 fn reject(self, session_id: &str, message: &str) {
951 match self {
952 Self::Submit { reply, .. } => {
953 if reply.send(Err(message.to_owned())).is_err() {
954 tracing::debug!(
955 %session_id,
956 operation = "submit",
957 "submit rejection receiver was already closed"
958 );
959 }
960 }
961 Self::Sync { reply } => {
962 if reply.send(Err(message.to_owned())).is_err() {
963 tracing::debug!(
964 %session_id,
965 operation = "sync",
966 "sync rejection receiver was already closed"
967 );
968 }
969 }
970 Self::RespondElicitation { reply, .. } => {
971 if reply.send(Err(message.to_owned())).is_err() {
972 tracing::debug!(
973 %session_id,
974 operation = "respond_elicitation",
975 "elicitation rejection receiver was already closed"
976 );
977 }
978 }
979 Self::Reviewer { reply, .. } => {
980 if reply.send(Err(message.to_owned())).is_err() {
981 tracing::debug!(
982 %session_id,
983 operation = "reviewer",
984 "reviewer rejection receiver was already closed"
985 );
986 }
987 }
988 Self::Lease { reply } => {
989 if reply
990 .send(Err(anyhow::anyhow!(message.to_owned())))
991 .is_err()
992 {
993 tracing::debug!(
994 %session_id,
995 operation = "lease",
996 "lease rejection receiver was already closed"
997 );
998 }
999 }
1000 }
1001 }
1002}
1003
1004struct ReturnedConnection {
1005 lease_id: u64,
1006 connection: Option<StandaloneSession>,
1007}
1008
1009struct DeferredSubmit {
1012 command_id: String,
1013 command: RelayCommand,
1014 reply: oneshot::Sender<std::result::Result<u64, String>>,
1015}
1016
1017#[derive(Debug, Default)]
1018struct ActorLifecycle {
1019 active_lease: Option<u64>,
1020 retirement_requested: bool,
1021}
1022
1023impl ActorLifecycle {
1024 fn set_retirement_requested(&mut self, requested: bool) {
1025 self.retirement_requested = requested;
1026 }
1027
1028 fn is_leased(&self) -> bool {
1029 self.active_lease.is_some()
1030 }
1031
1032 fn should_stop(&self) -> bool {
1033 self.retirement_requested && !self.is_leased()
1034 }
1035
1036 fn accepts_new_work(&self) -> bool {
1037 !self.retirement_requested
1038 }
1039
1040 fn activate_lease(&mut self, lease_id: u64) {
1041 debug_assert!(self.active_lease.is_none());
1042 self.active_lease = Some(lease_id);
1043 }
1044
1045 fn return_lease(&mut self, lease_id: u64) -> bool {
1046 if self.active_lease != Some(lease_id) {
1047 return false;
1048 }
1049 self.active_lease = None;
1050 true
1051 }
1052}
1053
1054struct ActorRegistration {
1055 target: RelaySessionTarget,
1056 commands: mpsc::Sender<ActorCommand>,
1057 releases: mpsc::UnboundedSender<ReturnedConnection>,
1058 retirement: watch::Sender<bool>,
1059 view: watch::Receiver<ManagedSessionView>,
1060 abort: tokio::task::AbortHandle,
1061}
1062
1063struct RemoteActorRegistration {
1064 commands: mpsc::Sender<ActorCommand>,
1065 releases: mpsc::UnboundedSender<ReturnedConnection>,
1066 view: watch::Receiver<ManagedSessionView>,
1067 view_tx: watch::Sender<ManagedSessionView>,
1068 abort: tokio::task::AbortHandle,
1069}
1070
1071enum RemoteManagerUpdate {
1072 Publish {
1073 session_id: String,
1074 view: ManagedSessionView,
1075 },
1076}
1077
1078#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1079enum ReconcileAction {
1080 Idle,
1081 Spawn,
1082 Keep,
1083 Retire,
1084}
1085
1086fn reconcile_action(
1087 actor: Option<&RelaySessionTarget>,
1088 desired: Option<&RelaySessionTarget>,
1089) -> ReconcileAction {
1090 match (actor, desired) {
1091 (None, None) => ReconcileAction::Idle,
1092 (None, Some(_)) => ReconcileAction::Spawn,
1093 (Some(actor), Some(desired)) if actor == desired => ReconcileAction::Keep,
1094 (Some(_), Some(_) | None) => ReconcileAction::Retire,
1095 }
1096}
1097
1098fn target_map(targets: &[RelaySessionTarget]) -> BTreeMap<String, RelaySessionTarget> {
1099 targets
1100 .iter()
1101 .cloned()
1102 .map(|target| (target.session_id.clone(), target))
1103 .collect()
1104}
1105
1106fn remove_actor_task(
1107 actors: &mut BTreeMap<String, ActorRegistration>,
1108 task_id: tokio::task::Id,
1109) -> Option<String> {
1110 let session_id = actors.iter().find_map(|(session_id, actor)| {
1111 (actor.abort.id() == task_id).then(|| session_id.clone())
1112 })?;
1113 actors.remove(&session_id);
1114 Some(session_id)
1115}
1116
1117fn reconcile_actors(
1118 targets: &BTreeMap<String, RelaySessionTarget>,
1119 actors: &mut BTreeMap<String, ActorRegistration>,
1120 tasks: &mut tokio::task::JoinSet<String>,
1121 updates: &CoalescedUpdateSender,
1122) {
1123 actors.retain(|session_id, actor| {
1129 let live = !actor.commands.is_closed();
1130 if !live {
1131 tracing::warn!(session_id, "replacing stopped session relay actor");
1132 }
1133 live
1134 });
1135
1136 for (session_id, actor) in actors.iter() {
1137 let retiring = matches!(
1138 reconcile_action(Some(&actor.target), targets.get(session_id)),
1139 ReconcileAction::Retire
1140 );
1141 actor.retirement.send_replace(retiring);
1142 }
1143
1144 for (session_id, target) in targets {
1145 if !matches!(
1146 reconcile_action(
1147 actors.get(session_id).map(|actor| &actor.target),
1148 Some(target)
1149 ),
1150 ReconcileAction::Spawn
1151 ) {
1152 continue;
1153 }
1154 let (actor_tx, actor_rx) = mpsc::channel(32);
1155 let (release_tx, release_rx) = mpsc::unbounded_channel();
1156 let (retirement_tx, retirement_rx) = watch::channel(false);
1157 let (view_tx, view_rx) = watch::channel(ManagedSessionView::default());
1158 let actor_updates = updates.clone();
1159 let task_target = target.clone();
1160 let task_id = session_id.clone();
1161 let abort = tasks.spawn(async move {
1162 run_session_actor(
1163 task_target,
1164 actor_rx,
1165 release_rx,
1166 retirement_rx,
1167 view_tx,
1168 actor_updates,
1169 )
1170 .await;
1171 task_id
1172 });
1173 actors.insert(
1174 session_id.clone(),
1175 ActorRegistration {
1176 target: target.clone(),
1177 commands: actor_tx,
1178 releases: release_tx,
1179 retirement: retirement_tx,
1180 view: view_rx,
1181 abort,
1182 },
1183 );
1184 }
1185}
1186
1187async fn run_remote_session_actor(
1188 session_id: String,
1189 mut commands: mpsc::Receiver<ActorCommand>,
1190 requests: mpsc::Sender<RemoteSessionRequest>,
1191) {
1192 while let Some(command) = commands.recv().await {
1193 let request = match command {
1194 ActorCommand::Submit {
1195 command_id,
1196 command,
1197 reply,
1198 } => RemoteSessionRequest::Submit {
1199 session_id: session_id.clone(),
1200 command_id,
1201 command,
1202 reply,
1203 },
1204 ActorCommand::Sync { reply } => RemoteSessionRequest::Sync {
1205 session_id: session_id.clone(),
1206 reply,
1207 },
1208 ActorCommand::RespondElicitation {
1209 elicitation_id,
1210 response,
1211 reply,
1212 } => RemoteSessionRequest::RespondElicitation {
1213 session_id: session_id.clone(),
1214 elicitation_id,
1215 response,
1216 reply,
1217 },
1218 ActorCommand::Reviewer {
1219 role,
1220 action,
1221 reply,
1222 } => RemoteSessionRequest::Reviewer {
1223 session_id: session_id.clone(),
1224 role,
1225 action,
1226 reply,
1227 },
1228 ActorCommand::Lease { reply } => {
1229 let _ = reply.send(Err(anyhow::anyhow!(
1230 "relay connection leases are available only inside the controller daemon"
1231 )));
1232 continue;
1233 }
1234 };
1235 if let Err(error) = requests.send(request).await {
1236 match error.0 {
1237 RemoteSessionRequest::Submit { reply, .. } => {
1238 let _ = reply.send(Err("controller daemon request bridge stopped".into()));
1239 }
1240 RemoteSessionRequest::Sync { reply, .. }
1241 | RemoteSessionRequest::RespondElicitation { reply, .. } => {
1242 let _ = reply.send(Err("controller daemon request bridge stopped".into()));
1243 }
1244 RemoteSessionRequest::Reviewer { reply, .. } => {
1245 let _ = reply.send(Err("controller daemon request bridge stopped".into()));
1246 }
1247 }
1248 break;
1249 }
1250 }
1251}
1252
1253fn spawn_remote_actor(
1254 session_id: String,
1255 view: ManagedSessionView,
1256 requests: &mpsc::Sender<RemoteSessionRequest>,
1257 actors: &mut BTreeMap<String, RemoteActorRegistration>,
1258 updates: &CoalescedUpdateSender,
1259) {
1260 let (actor_tx, actor_rx) = mpsc::channel(32);
1261 let (release_tx, _release_rx) = mpsc::unbounded_channel();
1262 let (view_tx, view_rx) = watch::channel(view.clone());
1263 let abort = tokio::spawn(run_remote_session_actor(
1264 session_id.clone(),
1265 actor_rx,
1266 requests.clone(),
1267 ))
1268 .abort_handle();
1269 actors.insert(
1270 session_id.clone(),
1271 RemoteActorRegistration {
1272 commands: actor_tx,
1273 releases: release_tx,
1274 view: view_rx,
1275 view_tx,
1276 abort,
1277 },
1278 );
1279 updates.send(SessionManagerUpdate { session_id, view });
1280}
1281
1282pub fn spawn_remote_session_manager() -> Result<RemoteSessionManagerChannels> {
1286 let (targets_tx, mut targets_rx) = watch::channel(Vec::<RelaySessionTarget>::new());
1287 let (commands_tx, mut commands_rx) = mpsc::channel(32);
1288 let (updates_tx, updates_rx) = coalesced_update_channel();
1289 let (published_tx, mut published_rx) = mpsc::unbounded_channel();
1290 let (requests_tx, requests_rx) = mpsc::channel(64);
1291 let (shutdown_tx, mut shutdown_rx) = oneshot::channel();
1292 let task = tokio::spawn(async move {
1293 let mut actors = BTreeMap::<String, RemoteActorRegistration>::new();
1294 let mut latest = BTreeMap::<String, ManagedSessionView>::new();
1295 let mut desired = BTreeMap::<String, RelaySessionTarget>::new();
1296 loop {
1297 tokio::select! {
1298 _ = &mut shutdown_rx => break,
1299 changed = targets_rx.changed() => {
1300 if changed.is_err() {
1301 break;
1302 }
1303 desired = target_map(&targets_rx.borrow_and_update());
1304 actors.retain(|session_id, actor| {
1305 if desired.contains_key(session_id) {
1306 true
1307 } else {
1308 actor.abort.abort();
1309 false
1310 }
1311 });
1312 for session_id in desired.keys() {
1313 if !actors.contains_key(session_id)
1314 && let Some(view) = latest.get(session_id).cloned()
1315 {
1316 spawn_remote_actor(
1317 session_id.clone(),
1318 view,
1319 &requests_tx,
1320 &mut actors,
1321 &updates_tx,
1322 );
1323 }
1324 }
1325 }
1326 command = commands_rx.recv() => {
1327 let Some(ManagerCommand::Session { session_id, reply }) = command else {
1328 break;
1329 };
1330 let handle = actors.get(&session_id).map(|actor| ManagedSessionHandle {
1331 session_id: session_id.clone(),
1332 commands: actor.commands.clone(),
1333 releases: actor.releases.clone(),
1334 view: actor.view.clone(),
1335 });
1336 let _ = reply.send(handle);
1337 }
1338 published = published_rx.recv() => {
1339 let Some(RemoteManagerUpdate::Publish { session_id, view }) = published else {
1340 break;
1341 };
1342 latest.insert(session_id.clone(), view.clone());
1343 if !desired.contains_key(&session_id) {
1344 continue;
1345 }
1346 if let Some(actor) = actors.get(&session_id) {
1347 publish_view(&session_id, view, &actor.view_tx, &updates_tx);
1348 continue;
1349 }
1350 spawn_remote_actor(
1351 session_id,
1352 view,
1353 &requests_tx,
1354 &mut actors,
1355 &updates_tx,
1356 );
1357 }
1358 }
1359 }
1360 for actor in actors.into_values() {
1361 actor.abort.abort();
1362 }
1363 });
1364 Ok(RemoteSessionManagerChannels {
1365 targets: targets_tx,
1366 control: SessionManagerControl {
1367 commands: commands_tx,
1368 },
1369 updates: updates_rx,
1370 shutdown: SessionManagerShutdown {
1371 signal: Some(shutdown_tx),
1372 task: Some(task),
1373 },
1374 publisher: RemoteSessionPublisher {
1375 updates: published_tx,
1376 },
1377 requests: RemoteSessionRequests {
1378 requests: requests_rx,
1379 },
1380 })
1381}
1382
1383pub fn spawn_session_manager() -> Result<SessionManagerChannels> {
1384 let (targets_tx, mut targets_rx) = watch::channel(Vec::<RelaySessionTarget>::new());
1385 let (commands_tx, mut commands_rx) = mpsc::channel(32);
1386 let (updates_tx, updates_rx) = coalesced_update_channel();
1387 let (shutdown_tx, mut shutdown_rx) = oneshot::channel();
1388 let task = tokio::spawn(async move {
1389 let mut actors = BTreeMap::<String, ActorRegistration>::new();
1390 let mut tasks = tokio::task::JoinSet::<String>::new();
1391 let mut desired_targets = BTreeMap::<String, RelaySessionTarget>::new();
1392 loop {
1393 tokio::select! {
1394 _ = &mut shutdown_rx => break,
1395 changed = targets_rx.changed() => {
1396 if changed.is_err() {
1397 break;
1398 }
1399 desired_targets = target_map(&targets_rx.borrow_and_update());
1400 reconcile_actors(
1401 &desired_targets,
1402 &mut actors,
1403 &mut tasks,
1404 &updates_tx,
1405 );
1406 }
1407 command = commands_rx.recv() => {
1408 let Some(ManagerCommand::Session { session_id, reply }) = command else {
1409 break;
1410 };
1411 let handle = actors
1412 .get(&session_id)
1413 .filter(|actor| !actor.commands.is_closed())
1414 .filter(|actor| desired_targets.get(&session_id) == Some(&actor.target))
1415 .map(|actor| ManagedSessionHandle {
1416 session_id: session_id.clone(),
1417 commands: actor.commands.clone(),
1418 releases: actor.releases.clone(),
1419 view: actor.view.clone(),
1420 });
1421 if reply.send(handle).is_err() {
1422 tracing::debug!(
1423 session_id = %session_id,
1424 operation = "session_lookup",
1425 "session lookup receiver was already closed"
1426 );
1427 }
1428 }
1429 joined = tasks.join_next_with_id(), if !tasks.is_empty() => {
1430 match joined {
1431 Some(Ok((task_id, session_id))) => {
1432 let removed = remove_actor_task(&mut actors, task_id);
1433 if removed.as_deref().is_some_and(|removed| removed != session_id) {
1434 tracing::error!(
1435 completed_session_id = session_id,
1436 registered_session_id = removed,
1437 "session relay actor completed under the wrong registration"
1438 );
1439 }
1440 desired_targets = target_map(&targets_rx.borrow());
1444 reconcile_actors(
1445 &desired_targets,
1446 &mut actors,
1447 &mut tasks,
1448 &updates_tx,
1449 );
1450 }
1451 Some(Err(error)) if error.is_cancelled() => {
1452 let cancelled_task = error.id();
1453 let session_id = remove_actor_task(&mut actors, cancelled_task);
1454 desired_targets = target_map(&targets_rx.borrow());
1455 reconcile_actors(
1456 &desired_targets,
1457 &mut actors,
1458 &mut tasks,
1459 &updates_tx,
1460 );
1461 tracing::warn!(
1462 session_id = ?session_id,
1463 "cancelled session relay actor was replaced"
1464 );
1465 }
1466 Some(Err(error)) => {
1467 let failed_task = error.id();
1468 remove_actor_task(&mut actors, failed_task);
1469 desired_targets = target_map(&targets_rx.borrow());
1470 reconcile_actors(
1471 &desired_targets,
1472 &mut actors,
1473 &mut tasks,
1474 &updates_tx,
1475 );
1476 tracing::error!(%error, "session relay actor failed");
1477 }
1478 None => {}
1479 }
1480 }
1481 }
1482 }
1483 shutdown_session_actors(&mut actors, &mut tasks).await;
1484 });
1485 Ok(SessionManagerChannels {
1486 targets: targets_tx,
1487 control: SessionManagerControl {
1488 commands: commands_tx,
1489 },
1490 updates: updates_rx,
1491 shutdown: SessionManagerShutdown {
1492 signal: Some(shutdown_tx),
1493 task: Some(task),
1494 },
1495 })
1496}
1497
1498async fn shutdown_session_actors(
1499 actors: &mut BTreeMap<String, ActorRegistration>,
1500 tasks: &mut tokio::task::JoinSet<String>,
1501) {
1502 for actor in actors.values() {
1503 actor.retirement.send_replace(true);
1504 }
1505 actors.clear();
1506
1507 let graceful = async {
1508 while let Some(joined) = tasks.join_next().await {
1509 match joined {
1510 Ok(_) => {}
1511 Err(error) if error.is_cancelled() => {}
1512 Err(error) => {
1513 tracing::error!(%error, "session relay actor failed during shutdown");
1514 }
1515 }
1516 }
1517 };
1518 if tokio::time::timeout(SESSION_MANAGER_SHUTDOWN_GRACE, graceful)
1519 .await
1520 .is_ok()
1521 {
1522 return;
1523 }
1524
1525 tracing::warn!(
1526 timeout_ms = SESSION_MANAGER_SHUTDOWN_GRACE.as_millis(),
1527 "session relay actors did not stop before the shutdown deadline; aborting them"
1528 );
1529 tasks.abort_all();
1530 while let Some(joined) = tasks.join_next().await {
1531 if let Err(error) = joined
1532 && !error.is_cancelled()
1533 {
1534 tracing::error!(%error, "session relay actor failed while being aborted");
1535 }
1536 }
1537}
1538
1539async fn run_session_actor(
1540 target: RelaySessionTarget,
1541 mut commands: mpsc::Receiver<ActorCommand>,
1542 mut releases: mpsc::UnboundedReceiver<ReturnedConnection>,
1543 mut retirement: watch::Receiver<bool>,
1544 view_tx: watch::Sender<ManagedSessionView>,
1545 updates: CoalescedUpdateSender,
1546) {
1547 let mut connection: Option<StandaloneSession> = None;
1548 let mut failures = 0_u32;
1549 let mut last_recovery_probe = None;
1550 let mut lifecycle = ActorLifecycle::default();
1551 let mut deferred_submits: VecDeque<DeferredSubmit> = VecDeque::new();
1552 let mut next_lease_id = 1_u64;
1553 let mut interval = tokio::time::interval(SESSION_SYNC_INTERVAL);
1554 interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
1555 loop {
1556 lifecycle.set_retirement_requested(*retirement.borrow_and_update());
1557 if lifecycle.should_stop() {
1558 break;
1559 }
1560 tokio::select! {
1561 _ = interval.tick() => {
1562 lifecycle.set_retirement_requested(*retirement.borrow());
1563 if lifecycle.should_stop() {
1564 break;
1565 }
1566 if lifecycle.is_leased() {
1567 continue;
1568 }
1569 let result = sync_actor_connection(
1570 &target,
1571 &mut connection,
1572 ).await;
1573 match result {
1574 Ok(snapshot) => {
1575 failures = 0;
1576 if let Some(snapshot) = snapshot {
1577 publish_view(&target.session_id, ManagedSessionView {
1578 snapshot: Some(snapshot),
1579 connected: true,
1580 error: None,
1581 }, &view_tx, &updates);
1582 }
1583 }
1584 Err(error) => {
1585 connection = None;
1586 failures = failures.saturating_add(1);
1587 let integrity = projection_integrity_failure(&error);
1591 tracing::warn!(
1592 session_id = target.session_id,
1593 consecutive_failures = failures,
1594 projection_integrity = integrity,
1595 transport_dead = worker_connect_needs_restart(&error),
1596 "session relay sync failed: {error:#}"
1597 );
1598 let recovery_due = !integrity
1599 && failures >= UNREACHABLE_FAILURE_THRESHOLD
1600 && worker_connect_needs_restart(&error)
1601 && target.worker_recovery.is_some()
1602 && last_recovery_probe.is_none_or(|last: tokio::time::Instant| {
1603 last.elapsed() >= WORKER_RESTART_COOLDOWN
1604 });
1605 if integrity || failures >= UNREACHABLE_FAILURE_THRESHOLD {
1606 let snapshot = view_tx.borrow().snapshot.clone();
1611 let mut detail = format!("{error:#}");
1612 if recovery_due {
1613 detail.push_str("; checking whether the relay worker is dead");
1614 }
1615 publish_view(&target.session_id, ManagedSessionView {
1616 snapshot,
1617 connected: false,
1618 error: Some(if integrity {
1619 ViewError::ProjectionIntegrity(detail)
1620 } else {
1621 ViewError::Unreachable(detail)
1622 }),
1623 }, &view_tx, &updates);
1624 }
1625 if recovery_due {
1626 last_recovery_probe = Some(tokio::time::Instant::now());
1627 let plan = target
1628 .worker_recovery
1629 .clone()
1630 .expect("recovery eligibility requires a plan");
1631 let restart_unresponsive =
1632 worker_connect_allows_live_restart(&error);
1633 tracing::warn!(
1634 session_id = target.session_id,
1635 "relay worker is unreachable; probing it before recovery: {error:#}"
1636 );
1637 match recover_worker(plan, restart_unresponsive).await {
1638 Ok(
1639 outcome @ (WorkerRecoveryOutcome::RestartedDead
1640 | WorkerRecoveryOutcome::RestartedUnresponsive),
1641 ) => {
1642 failures = 0;
1643 let snapshot = view_tx.borrow().snapshot.clone();
1644 let recovery = match outcome {
1645 WorkerRecoveryOutcome::RestartedDead => {
1646 "confirmed the relay worker was dead and restarted it"
1647 }
1648 WorkerRecoveryOutcome::RestartedUnresponsive => {
1649 "the relay worker was alive but not serving handshakes, so it was restarted"
1650 }
1651 WorkerRecoveryOutcome::Alive
1652 | WorkerRecoveryOutcome::Starting
1653 | WorkerRecoveryOutcome::TargetMissing => unreachable!(),
1654 };
1655 publish_view(&target.session_id, ManagedSessionView {
1656 snapshot,
1657 connected: false,
1658 error: Some(ViewError::Unreachable(format!(
1659 "{error:#}; {recovery}"
1660 ))),
1661 }, &view_tx, &updates);
1662 interval.reset_after(RECONNECT_INTERVAL);
1663 }
1664 Ok(WorkerRecoveryOutcome::Alive) => {
1665 tracing::warn!(
1666 session_id = target.session_id,
1667 "relay transport failed but the worker is alive; leaving it running"
1668 );
1669 let snapshot = view_tx.borrow().snapshot.clone();
1670 publish_view(&target.session_id, ManagedSessionView {
1671 snapshot,
1672 connected: false,
1673 error: Some(ViewError::Unreachable(format!(
1674 "{error:#}; relay worker is still alive, so it was not restarted"
1675 ))),
1676 }, &view_tx, &updates);
1677 interval.reset_after(reconnect_delay(failures));
1678 }
1679 Ok(WorkerRecoveryOutcome::Starting) => {
1680 tracing::warn!(
1681 session_id = target.session_id,
1682 "relay worker is still starting; leaving it running"
1683 );
1684 let snapshot = view_tx.borrow().snapshot.clone();
1685 publish_view(&target.session_id, ManagedSessionView {
1686 snapshot,
1687 connected: false,
1688 error: Some(ViewError::Unreachable(format!(
1689 "{error:#}; relay worker is still recovering its durable state, so it was not restarted"
1690 ))),
1691 }, &view_tx, &updates);
1692 interval.reset_after(reconnect_delay(failures));
1693 }
1694 Ok(WorkerRecoveryOutcome::TargetMissing) => {
1695 let snapshot = view_tx.borrow().snapshot.clone();
1696 publish_view(&target.session_id, ManagedSessionView {
1697 snapshot,
1698 connected: false,
1699 error: Some(ViewError::TargetMissing(
1700 "the managed Podman session container no longer exists"
1701 .into(),
1702 )),
1703 }, &view_tx, &updates);
1704 interval.reset_after(RECONNECT_BACKOFF_CEILING);
1705 }
1706 Err(recovery_error) => {
1707 tracing::warn!(
1708 session_id = target.session_id,
1709 "automatic relay worker recovery failed safely: {recovery_error:#}"
1710 );
1711 let snapshot = view_tx.borrow().snapshot.clone();
1712 publish_view(&target.session_id, ManagedSessionView {
1713 snapshot,
1714 connected: false,
1715 error: Some(ViewError::Unreachable(format!(
1716 "{error:#}; could not confirm the relay worker was dead, so it was not restarted: {recovery_error:#}"
1717 ))),
1718 }, &view_tx, &updates);
1719 interval.reset_after(reconnect_delay(failures));
1720 }
1721 }
1722 } else {
1723 interval.reset_after(reconnect_delay(failures));
1724 }
1725 }
1726 }
1727 }
1728 command = commands.recv() => {
1729 let Some(command) = command else { break };
1730 lifecycle.set_retirement_requested(*retirement.borrow());
1731 if !lifecycle.accepts_new_work() {
1732 tracing::debug!(
1733 session_id = %target.session_id,
1734 operation = command.operation_name(),
1735 "rejecting relay operation while session target changes"
1736 );
1737 command.reject(&target.session_id, "session target is changing");
1738 continue;
1739 }
1740 match command {
1741 ActorCommand::Submit { command_id, command, reply } => {
1742 if matches!(command, RelayCommand::Prompt { .. })
1749 && let Some(refusal) =
1750 crate::hel_review_host::prompt_refusal(&target.session_id)
1751 {
1752 tracing::debug!(
1753 session_id = %target.session_id,
1754 %command_id,
1755 "refusing a prompt while a turn review is unresolved"
1756 );
1757 let _ = reply.send(Err(refusal.to_owned()));
1758 continue;
1759 }
1760 if lifecycle.is_leased() {
1761 deferred_submits.push_back(DeferredSubmit {
1765 command_id,
1766 command,
1767 reply,
1768 });
1769 continue;
1770 }
1771 deliver_submit(
1772 &target,
1773 &mut connection,
1774 command_id,
1775 command,
1776 reply,
1777 &view_tx,
1778 &updates,
1779 )
1780 .await;
1781 }
1782 ActorCommand::Sync { reply } => {
1783 if lifecycle.is_leased() {
1784 tracing::debug!(
1785 session_id = %target.session_id,
1786 operation = "sync",
1787 "rejecting sync while session is leased"
1788 );
1789 if reply
1790 .send(Err("session is reserved for a lifecycle operation".into()))
1791 .is_err()
1792 {
1793 tracing::debug!(
1794 session_id = %target.session_id,
1795 operation = "sync",
1796 "sync rejection receiver was already closed"
1797 );
1798 }
1799 continue;
1800 }
1801 let result = sync_actor_connection(
1802 &target,
1803 &mut connection,
1804 ).await.map(|snapshot| {
1805 if let Some(snapshot) = snapshot {
1806 publish_view(&target.session_id, ManagedSessionView {
1807 snapshot: Some(snapshot),
1808 connected: true,
1809 error: None,
1810 }, &view_tx, &updates);
1811 }
1812 });
1813 if result.is_err() {
1814 connection = None;
1815 }
1816 if let Err(error) = &result {
1817 tracing::warn!(
1818 session_id = %target.session_id,
1819 operation = "sync",
1820 error = %error,
1821 "explicit relay synchronization failed"
1822 );
1823 }
1824 if reply.send(result.map_err(|error| format!("{error:#}"))).is_err() {
1825 tracing::debug!(
1826 session_id = %target.session_id,
1827 operation = "sync",
1828 "sync result receiver was already closed"
1829 );
1830 }
1831 }
1832 ActorCommand::Reviewer {
1833 role,
1834 action,
1835 reply,
1836 } => {
1837 if lifecycle.is_leased() {
1838 tracing::debug!(
1842 session_id = %target.session_id,
1843 operation = action.operation_name(),
1844 "rejecting a reviewer action while the session is leased"
1845 );
1846 if reply
1847 .send(Err("session is reserved for a lifecycle operation".into()))
1848 .is_err()
1849 {
1850 tracing::debug!(
1851 session_id = %target.session_id,
1852 operation = "reviewer",
1853 "reviewer rejection receiver was already closed"
1854 );
1855 }
1856 continue;
1857 }
1858 let operation = action.operation_name();
1859 let result = async {
1860 sync_actor_connection(&target, &mut connection).await?;
1861 let connection =
1862 connection.as_mut().context("relay is disconnected")?;
1863 drive_reviewer(connection, role, action).await
1864 }
1865 .await;
1866 match &result {
1867 Ok(_) => {}
1868 Err(error) if !is_final_rejection(error) => connection = None,
1869 Err(_) => {}
1870 }
1871 if let Err(error) = &result {
1872 tracing::warn!(
1873 session_id = %target.session_id,
1874 %operation,
1875 error = %error,
1876 "reviewer action failed"
1877 );
1878 }
1879 if reply
1880 .send(result.map_err(|error| format!("{error:#}")))
1881 .is_err()
1882 {
1883 tracing::debug!(
1884 session_id = %target.session_id,
1885 %operation,
1886 "reviewer result receiver was already closed"
1887 );
1888 }
1889 }
1890 ActorCommand::RespondElicitation {
1891 elicitation_id,
1892 response,
1893 reply,
1894 } => {
1895 if lifecycle.is_leased() {
1896 tracing::debug!(
1897 session_id = %target.session_id,
1898 operation = "respond_elicitation",
1899 "rejecting elicitation response while session is leased"
1900 );
1901 if reply
1902 .send(Err("session is reserved for a lifecycle operation".into()))
1903 .is_err()
1904 {
1905 tracing::debug!(
1906 session_id = %target.session_id,
1907 operation = "respond_elicitation",
1908 "elicitation rejection receiver was already closed"
1909 );
1910 }
1911 continue;
1912 }
1913 let result = async {
1914 sync_actor_connection(&target, &mut connection).await?;
1915 let connection = connection
1916 .as_mut()
1917 .context("relay is disconnected")?;
1918 connection
1919 .respond_elicitation(elicitation_id, response)
1920 .await?;
1921 Ok::<_, anyhow::Error>(connection.snapshot())
1922 }
1923 .await;
1924 match result {
1925 Ok(ref snapshot) => publish_view(
1926 &target.session_id,
1927 ManagedSessionView {
1928 snapshot: Some(snapshot.clone()),
1929 connected: true,
1930 error: None,
1931 },
1932 &view_tx,
1933 &updates,
1934 ),
1935 Err(ref error) if !is_final_rejection(error) => connection = None,
1936 Err(_) => {}
1937 }
1938 if let Err(error) = &result {
1939 tracing::warn!(
1940 session_id = %target.session_id,
1941 operation = "respond_elicitation",
1942 error = %error,
1943 "relay elicitation response failed"
1944 );
1945 }
1946 if reply
1947 .send(result.map(|_| ()).map_err(|error| format!("{error:#}")))
1948 .is_err()
1949 {
1950 tracing::debug!(
1951 session_id = %target.session_id,
1952 operation = "respond_elicitation",
1953 "elicitation result receiver was already closed"
1954 );
1955 }
1956 }
1957 ActorCommand::Lease { reply } => {
1958 if lifecycle.is_leased() {
1959 tracing::debug!(
1960 session_id = %target.session_id,
1961 operation = "lease",
1962 "rejecting duplicate session lifecycle lease"
1963 );
1964 if reply
1965 .send(Err(anyhow::anyhow!(
1966 "session already has a lifecycle operation"
1967 )))
1968 .is_err()
1969 {
1970 tracing::debug!(
1971 session_id = %target.session_id,
1972 operation = "lease",
1973 "lease rejection receiver was already closed"
1974 );
1975 }
1976 continue;
1977 }
1978 let lease_id = next_lease_id;
1979 let result = sync_actor_connection(
1980 &target,
1981 &mut connection,
1982 )
1983 .await
1984 .map(|_| {
1985 next_lease_id = next_lease_id.wrapping_add(1).max(1);
1986 (
1987 lease_id,
1988 connection
1989 .take()
1990 .expect("successful sync retained its connection"),
1991 )
1992 });
1993 if result.is_err() {
1994 connection = None;
1995 }
1996 if let Err(error) = &result {
1997 tracing::warn!(
1998 session_id = %target.session_id,
1999 operation = "lease",
2000 error = %error,
2001 "could not acquire relay session lease"
2002 );
2003 }
2004 let acquired = result.is_ok();
2005 match reply.send(result) {
2006 Ok(()) if acquired => lifecycle.activate_lease(lease_id),
2007 Ok(()) => {}
2008 Err(Ok((_lease_id, returned))) => connection = Some(returned),
2009 Err(Err(_)) => {}
2010 }
2011 }
2012 }
2013 }
2014 returned = releases.recv() => {
2015 let Some(returned) = returned else { continue };
2016 if lifecycle.return_lease(returned.lease_id) {
2017 connection = returned.connection;
2020 failures = 0;
2021 interval.reset();
2022 if let Some(returned) = connection.as_ref() {
2027 publish_view(&target.session_id, ManagedSessionView {
2028 snapshot: Some(returned.snapshot()),
2029 connected: true,
2030 error: None,
2031 }, &view_tx, &updates);
2032 }
2033 let retiring = *retirement.borrow();
2034 while let Some(deferred) = deferred_submits.pop_front() {
2035 if retiring {
2036 if deferred
2037 .reply
2038 .send(Err("session target is changing".into()))
2039 .is_err()
2040 {
2041 tracing::debug!(
2042 session_id = %target.session_id,
2043 operation = "submit",
2044 "deferred submit rejection receiver was already closed"
2045 );
2046 }
2047 continue;
2048 }
2049 deliver_submit(
2050 &target,
2051 &mut connection,
2052 deferred.command_id,
2053 deferred.command,
2054 deferred.reply,
2055 &view_tx,
2056 &updates,
2057 )
2058 .await;
2059 }
2060 }
2061 }
2062 changed = retirement.changed() => {
2063 if changed.is_err() {
2064 break;
2065 }
2066 }
2067 }
2068 }
2069 if let Some(connection) = connection.take()
2070 && let Err(error) = connection.detach().await
2071 {
2072 tracing::warn!(
2073 session_id = %target.session_id,
2074 %error,
2075 "could not detach relay connection during session actor shutdown"
2076 );
2077 }
2078 for deferred in deferred_submits {
2080 if deferred
2081 .reply
2082 .send(Err("session manager stopped".into()))
2083 .is_err()
2084 {
2085 tracing::debug!(
2086 session_id = %target.session_id,
2087 operation = "submit",
2088 "deferred submit shutdown receiver was already closed"
2089 );
2090 }
2091 }
2092}
2093
2094async fn deliver_submit(
2097 target: &RelaySessionTarget,
2098 connection: &mut Option<StandaloneSession>,
2099 command_id: String,
2100 command: RelayCommand,
2101 reply: oneshot::Sender<std::result::Result<u64, String>>,
2102 view_tx: &watch::Sender<ManagedSessionView>,
2103 updates: &CoalescedUpdateSender,
2104) {
2105 let result = submit_actor_command(target, connection, &command_id, &command).await;
2106 if let Err(error) = result.as_ref() {
2107 tracing::warn!(
2108 session_id = %target.session_id,
2109 operation = "submit",
2110 %command_id,
2111 retryable = !is_final_rejection(error),
2112 error = %error,
2113 "relay command submission failed"
2114 );
2115 }
2116 if let Err(error) = result.as_ref()
2117 && !is_final_rejection(error)
2118 {
2119 *connection = None;
2120 }
2121 let accepted = result.as_ref().ok().copied();
2122 if reply
2127 .send(result.map_err(|error| format!("{error:#}")))
2128 .is_err()
2129 {
2130 tracing::debug!(
2131 session_id = %target.session_id,
2132 operation = "submit",
2133 %command_id,
2134 "submit result receiver was already closed"
2135 );
2136 }
2137 let Some(ordinal) = accepted else {
2138 return;
2139 };
2140 tracing::trace!(%ordinal, %command_id, "relay command accepted");
2141 let Some(session) = connection.as_mut() else {
2142 return;
2143 };
2144 match session.sync().await {
2148 Ok(snapshot) => publish_view(
2149 &target.session_id,
2150 ManagedSessionView {
2151 snapshot: Some(snapshot),
2152 connected: true,
2153 error: None,
2154 },
2155 view_tx,
2156 updates,
2157 ),
2158 Err(error) => {
2159 tracing::warn!(
2160 session_id = %target.session_id,
2161 operation = "submit",
2162 %command_id,
2163 error = %format!("{error:#}"),
2164 "projection could not catch up to an accepted command"
2165 );
2166 *connection = None;
2167 }
2168 }
2169}
2170
2171fn is_final_rejection(error: &anyhow::Error) -> bool {
2177 error
2178 .downcast_ref::<RelayRejected>()
2179 .is_some_and(|rejected| !rejected.is_retryable())
2180}
2181
2182async fn submit_actor_command(
2183 target: &RelaySessionTarget,
2184 connection: &mut Option<StandaloneSession>,
2185 command_id: &str,
2186 command: &RelayCommand,
2187) -> Result<u64> {
2188 let mut first_error = None;
2189 for attempt in 1..=2 {
2190 if connection.is_none() {
2191 sync_actor_connection(target, connection).await?;
2192 }
2193 let result = connection
2194 .as_mut()
2195 .context("relay is disconnected")?
2196 .submit_accepted(command_id.to_owned(), command.clone())
2197 .await;
2198 match result {
2199 Ok(ordinal) => return Ok(ordinal),
2200 Err(error) if is_final_rejection(&error) => return Err(error),
2206 Err(error) => {
2207 tracing::warn!(
2208 session_id = %target.session_id,
2209 operation = "submit",
2210 %command_id,
2211 attempt,
2212 retryable = true,
2213 error = %error,
2214 "retryable relay command failure; reconnecting"
2215 );
2216 if first_error.is_none() {
2217 first_error = Some(format!("{error:#}"));
2218 }
2219 *connection = None;
2220 }
2221 }
2222 }
2223 let detail = first_error.unwrap_or_else(|| "relay submission failed".into());
2224 bail!("relay command {command_id} failed after an idempotent reconnect: {detail}")
2225}
2226
2227async fn drive_reviewer(
2232 connection: &mut StandaloneSession,
2233 role: Option<String>,
2234 action: ReviewerAction,
2235) -> Result<ReviewerOutcome> {
2236 let client = &mut connection.client;
2237 let role = role.as_deref();
2238 Ok(match action {
2239 ReviewerAction::Start { config } => {
2240 ReviewerOutcome::Started(Box::new(client.start_reviewer(role, *config).await?))
2241 }
2242 ReviewerAction::Submit {
2243 command_id,
2244 command,
2245 } => ReviewerOutcome::Accepted {
2246 ordinal: client.submit_to_reviewer(role, command_id, command).await?,
2247 },
2248 ReviewerAction::Attach {
2249 after_ordinal,
2250 after_digest,
2251 } => ReviewerOutcome::Attached(Box::new(
2252 client
2253 .attach_reviewer(role, after_ordinal, after_digest)
2254 .await?,
2255 )),
2256 ReviewerAction::Acknowledge {
2257 through_ordinal,
2258 through_digest,
2259 } => ReviewerOutcome::Acknowledged(
2260 client
2261 .acknowledge_reviewer(role, through_ordinal, through_digest)
2262 .await?,
2263 ),
2264 ReviewerAction::Status => {
2265 ReviewerOutcome::Status(Box::new(client.reviewer_status(role).await?))
2266 }
2267 ReviewerAction::RespondElicitation {
2268 elicitation_id,
2269 response,
2270 } => {
2271 client
2272 .respond_to_reviewer(role, elicitation_id, response)
2273 .await?;
2274 ReviewerOutcome::ElicitationResolved
2275 }
2276 ReviewerAction::Pause => {
2277 client.pause_reviewer(role).await?;
2278 ReviewerOutcome::Paused
2279 }
2280 ReviewerAction::CaptureDelta { baselines } => ReviewerOutcome::Delta {
2281 repositories: client.capture_review_delta(role, baselines).await?,
2282 },
2283 ReviewerAction::AdvanceBaseline { trees } => {
2284 client.advance_review_baseline(role, trees).await?;
2285 ReviewerOutcome::BaselineAdvanced
2286 }
2287 ReviewerAction::AnalyzeDelta { repositories } => ReviewerOutcome::ChangedFunctions {
2288 packet: client.analyze_review_delta(role, repositories).await?,
2289 },
2290 ReviewerAction::TakeLaneDispatches => ReviewerOutcome::LaneDispatches {
2291 requests: client.take_lane_dispatches().await?,
2292 },
2293 })
2294}
2295
2296async fn sync_actor_connection(
2297 target: &RelaySessionTarget,
2298 connection: &mut Option<StandaloneSession>,
2299) -> Result<Option<ManagedSessionSnapshot>> {
2300 if connection.is_none() {
2301 *connection = Some(StandaloneSession::connect(target).await?);
2302 return Ok(Some(
2303 connection
2304 .as_ref()
2305 .expect("connection was initialized")
2306 .snapshot(),
2307 ));
2308 }
2309 let connection = connection.as_mut().expect("connection was initialized");
2310 if connection.sync_in_place().await? {
2311 Ok(Some(connection.snapshot()))
2312 } else {
2313 Ok(None)
2314 }
2315}
2316
2317fn view_is_unchanged(current: &ManagedSessionView, next: &ManagedSessionView) -> bool {
2325 if current.connected != next.connected || current.error != next.error {
2326 return false;
2327 }
2328 match (¤t.snapshot, &next.snapshot) {
2329 (None, None) => true,
2330 (Some(current), Some(next)) => {
2331 let (current_session, next_session) = (¤t.materialized, &next.materialized);
2332 current.latest_credential_sync_signal == next.latest_credential_sync_signal
2333 && current.operational == next.operational
2334 && current_session.session_id == next_session.session_id
2335 && current_session.applied_event_ordinal == next_session.applied_event_ordinal
2336 && current_session.applied_event_digest == next_session.applied_event_digest
2337 && current_session.last_activity_at_ms == next_session.last_activity_at_ms
2338 && current_session.execution == next_session.execution
2339 && current_session.session_title == next_session.session_title
2340 && current_session.queued_prompts == next_session.queued_prompts
2341 }
2342 (None, Some(_)) | (Some(_), None) => false,
2343 }
2344}
2345
2346fn publish_view(
2347 session_id: &str,
2348 view: ManagedSessionView,
2349 watch: &watch::Sender<ManagedSessionView>,
2350 updates: &CoalescedUpdateSender,
2351) {
2352 let changed = watch.send_if_modified(|current| {
2356 if view_is_unchanged(current, &view) {
2357 return false;
2358 }
2359 *current = view.clone();
2360 true
2361 });
2362 if changed {
2363 updates.send(SessionManagerUpdate {
2364 session_id: session_id.to_owned(),
2365 view,
2366 });
2367 }
2368}
2369
2370async fn load_projection(session_id: &str) -> Result<MaterializedSession> {
2375 let session_id = session_id.to_owned();
2376 tokio::task::spawn_blocking(move || -> Result<MaterializedSession> {
2377 let loaded = hel::hel_database::load_materialized_session(&session_id)?;
2378 Ok(loaded.unwrap_or_else(|| MaterializedSession::empty(session_id)))
2379 })
2380 .await
2381 .context("controller projection load task failed")?
2382}
2383
2384pub struct StandaloneSession {
2385 client: RelayClient,
2386 materialized: MaterializedSession,
2387 operational: RelayOperationalState,
2388 latest_credential_sync_signal: Option<CredentialSyncSignal>,
2389 project_memory: Option<ProjectMemorySyncTarget>,
2390}
2391
2392impl StandaloneSession {
2393 pub fn set_project_memory_target(&mut self, target: Option<ProjectMemorySyncTarget>) {
2394 self.project_memory = target;
2395 }
2396
2397 pub async fn connect(target: &RelaySessionTarget) -> Result<Self> {
2398 let mut client = RelayClient::connect(&target.spec, &target.session_id).await?;
2402 let operational = client.status().await?;
2403 let materialized = load_projection(&target.session_id).await?;
2404 let mut connection = Self {
2405 client,
2406 materialized,
2407 operational,
2408 latest_credential_sync_signal: None,
2409 project_memory: target.project_memory.clone(),
2410 };
2411 connection.sync_in_place().await?;
2412 Ok(connection)
2413 }
2414
2415 pub async fn connect_command(spec: &CommandSpec, session_id: &str) -> Result<Self> {
2416 Self::connect(&RelaySessionTarget {
2417 session_id: session_id.to_owned(),
2418 spec: spec.clone(),
2419 worker_recovery: None,
2420 project_memory: None,
2421 })
2422 .await
2423 }
2424
2425 async fn detach(self) -> Result<()> {
2426 self.client.detach().await
2427 }
2428
2429 pub async fn sync(&mut self) -> Result<ManagedSessionSnapshot> {
2430 self.sync_in_place().await?;
2431 Ok(self.snapshot())
2432 }
2433
2434 async fn sync_in_place(&mut self) -> Result<bool> {
2435 let original_ordinal = self.materialized.applied_event_ordinal;
2436 let original_digest = self.materialized.applied_event_digest.clone();
2437 let original_operational = self.operational.clone();
2438 let mut repaired = false;
2439 let mut repaired_frontiers = std::collections::HashSet::new();
2440 loop {
2441 let after_ordinal = self.materialized.applied_event_ordinal;
2442 match self.catch_up_fixed_frontier().await {
2443 Ok(()) => break,
2444 Err(error) if error.downcast_ref::<ProjectionAdvancedError>().is_some() => {
2445 let durable = load_projection(&self.materialized.session_id).await?;
2446 if durable.applied_event_ordinal <= after_ordinal {
2447 return Err(error);
2448 }
2449 self.materialized = durable;
2450 continue;
2451 }
2452 Err(error) if relay_desynchronized(&error) => {
2453 self.repair_projection()
2454 .await
2455 .with_context(|| {
2456 format!(
2457 "controller projection for {} cannot catch up from ordinal {after_ordinal}: {error:#}",
2458 self.materialized.session_id
2459 )
2460 })?;
2461 repaired = true;
2462 let frontier = self.materialized.applied_event_ordinal;
2469 if !repaired_frontiers.insert(frontier) {
2470 bail!(
2471 "controller projection for {} cannot catch up: relay history is \
2472 unreadable and rebuilding from checkpoint frontier {frontier} does \
2473 not get past it",
2474 self.materialized.session_id
2475 );
2476 }
2477 continue;
2478 }
2479 Err(error) => return Err(error),
2480 }
2481 }
2482 let changed = repaired
2483 || self.materialized.applied_event_ordinal != original_ordinal
2484 || self.materialized.applied_event_digest != original_digest
2485 || self.operational != original_operational;
2486 Ok(changed)
2487 }
2488
2489 async fn catch_up_fixed_frontier(&mut self) -> Result<()> {
2494 let after = RelayCursor {
2495 ordinal: self.materialized.applied_event_ordinal,
2496 digest: self.materialized.applied_event_digest.clone(),
2497 };
2498 let catch_up = self
2499 .client
2500 .begin_catch_up(after.ordinal, &after.digest)
2501 .await?;
2502 let mut cursor = self.apply_event_page(catch_up.first_page).await?;
2503 let mut pages_remaining = catch_up.frontier.ordinal.saturating_sub(cursor.ordinal);
2504 while cursor.ordinal < catch_up.frontier.ordinal {
2505 ensure!(
2506 pages_remaining > 0,
2507 "relay catch-up exceeded its fixed page bound"
2508 );
2509 pages_remaining -= 1;
2510 let page = self
2511 .client
2512 .next_catch_up_page(&cursor, &catch_up.frontier)
2513 .await?;
2514 cursor = self.apply_event_page(page).await?;
2515 }
2516 ensure!(
2517 cursor == catch_up.frontier,
2518 "controller projection did not reach the captured relay frontier"
2519 );
2520 if cursor.ordinal > 0 {
2521 let acknowledged = self
2522 .client
2523 .acknowledge(cursor.ordinal, &cursor.digest)
2524 .await?;
2525 ensure!(
2526 acknowledged == cursor,
2527 "relay acknowledged cursor {}:{} instead of {}:{}",
2528 acknowledged.ordinal,
2529 acknowledged.digest,
2530 cursor.ordinal,
2531 cursor.digest,
2532 );
2533 }
2534 let mut operational = catch_up.state;
2535 operational.acknowledged_through = cursor.ordinal;
2536 operational.acknowledged_digest = cursor.digest;
2537 self.operational = operational;
2538 Ok(())
2539 }
2540
2541 async fn repair_projection(&mut self) -> Result<()> {
2542 let state = hel::hel_database::load_state()?;
2543 let record = state
2544 .sessions
2545 .get(&self.materialized.session_id)
2546 .context("controller session disappeared while repairing its projection")?;
2547 let Some(checkpoint) = record.checkpoint.as_ref() else {
2548 let replacement = MaterializedSession::empty(&self.materialized.session_id);
2549 self.client
2550 .attach(
2551 replacement.applied_event_ordinal,
2552 &replacement.applied_event_digest,
2553 )
2554 .await
2555 .context("relay cannot rebuild the projection from its genesis")?;
2556 save_materialized_session(&replacement)?;
2557 self.materialized = replacement;
2558 return Ok(());
2559 };
2560 let checkpoint_path = checkpoint.archive_path.clone();
2561 let archive = tokio::task::spawn_blocking(move || {
2562 verify_archive_streaming(&checkpoint_path).with_context(|| {
2563 format!(
2564 "verify projection repair checkpoint {}",
2565 checkpoint_path.display()
2566 )
2567 })
2568 })
2569 .await
2570 .context("projection repair archive verification task failed")??;
2571 ensure!(
2572 archive.archive_sha256 == checkpoint.sha256,
2573 "projection repair checkpoint checksum does not match controller metadata"
2574 );
2575 ensure!(
2576 archive.manifest.session.id == self.materialized.session_id,
2577 "projection repair checkpoint belongs to session {}, not {}",
2578 archive.manifest.session.id,
2579 self.materialized.session_id
2580 );
2581 let canonical = archive.canonical_session;
2582 ensure!(
2583 canonical.event_frontier == checkpoint.event_frontier,
2584 "projection repair checkpoint metadata frontier {} does not match archive frontier {}",
2585 checkpoint.event_frontier,
2586 canonical.event_frontier
2587 );
2588
2589 self.client
2593 .attach(canonical.event_frontier, &canonical.event_frontier_digest)
2594 .await
2595 .context("relay rejected the verified checkpoint repair cursor")?;
2596 let replacement =
2597 materialized_session_from_canonical(&self.materialized.session_id, &canonical)?;
2598 save_materialized_session(&replacement)?;
2599 self.materialized = replacement;
2600 Ok(())
2601 }
2602
2603 pub fn snapshot(&self) -> ManagedSessionSnapshot {
2604 ManagedSessionSnapshot {
2605 window: hel::hel_state::ProjectionWindow::of(&self.materialized),
2606 materialized: self.materialized.clone(),
2607 operational: self.operational.clone(),
2608 latest_credential_sync_signal: self.latest_credential_sync_signal.clone(),
2609 worker_build: self.client.worker_build().map(str::to_owned),
2610 }
2611 }
2612
2613 pub async fn submit_accepted(
2622 &mut self,
2623 command_id: String,
2624 command: RelayCommand,
2625 ) -> Result<u64> {
2626 self.client.submit(command_id, command).await
2627 }
2628
2629 pub async fn submit(&mut self, command_id: String, command: RelayCommand) -> Result<u64> {
2630 let ordinal = self.submit_accepted(command_id, command).await?;
2631 self.sync_in_place().await?;
2632 Ok(ordinal)
2633 }
2634
2635 pub async fn respond_elicitation(
2636 &mut self,
2637 elicitation_id: String,
2638 response: ElicitationResponse,
2639 ) -> Result<()> {
2640 self.client
2641 .respond_elicitation(elicitation_id, response)
2642 .await?;
2643 self.sync_in_place().await?;
2644 Ok(())
2645 }
2646
2647 pub async fn install_prompt_context(&mut self, text: String) -> Result<()> {
2650 self.client.install_prompt_context(text).await
2651 }
2652
2653 async fn apply_event_page(&mut self, page: RelayEventPage) -> Result<RelayCursor> {
2658 let RelayEventPage {
2659 events,
2660 through_ordinal,
2661 through_digest,
2662 } = page;
2663 let event_count = events.len();
2664 let transaction_count = event_count.div_ceil(PROJECTION_TRANSACTION_EVENT_BUDGET);
2665 let started = Instant::now();
2666 for events in events.chunks(PROJECTION_TRANSACTION_EVENT_BUDGET) {
2667 let session_id = self.materialized.session_id.clone();
2668 let events = events.to_vec();
2669 let projection = self.materialized.clone();
2670 let (projection, credential_sync_signal) = tokio::task::spawn_blocking(
2674 move || -> Result<(MaterializedSession, Option<CredentialSyncSignal>)> {
2675 let mut projection = projection;
2678 let mut projection_index = ProjectionIndex::new(&projection);
2679 let mut credential_sync_signal = None;
2680 let mut prepared = Vec::with_capacity(events.len());
2681 for event in &events {
2682 let mutation =
2683 project_relay_event_indexed(&projection, &projection_index, event)?
2684 .mutation;
2685 prepared.push((
2686 event.ordinal,
2687 event.previous_digest.clone(),
2688 event.digest.clone(),
2689 mutation.clone(),
2690 ));
2691 apply_committed_projection_event_indexed(
2692 &mut projection,
2693 &mut projection_index,
2694 event,
2695 mutation,
2696 )?;
2697 if let Some(reason) = relay_event_credential_sync_reason(event) {
2698 credential_sync_signal = Some(CredentialSyncSignal {
2699 ordinal: event.ordinal,
2700 reason,
2701 });
2702 }
2703 }
2704 drop(projection_index);
2705 apply_projection_page(&session_id, move |committed| {
2706 for (ordinal, previous_digest, digest, mutation) in prepared {
2707 match committed.apply(ordinal, &previous_digest, &digest, &mutation)? {
2708 ProjectionApplyOutcome::Applied => {}
2709 ProjectionApplyOutcome::AlreadyApplied => {
2710 return Err(ProjectionAdvancedError {
2711 event_ordinal: ordinal,
2712 }
2713 .into());
2714 }
2715 }
2716 }
2717 Ok((projection, credential_sync_signal))
2718 })
2719 },
2720 )
2721 .await
2722 .context("relay projection page task failed")??;
2723 self.materialized = projection;
2724 if let Some(signal) = credential_sync_signal {
2725 self.latest_credential_sync_signal = Some(signal);
2726 }
2727 }
2728 if transaction_count > 1 {
2729 tracing::debug!(
2730 session_id = self.materialized.session_id,
2731 event_count,
2732 transaction_count,
2733 elapsed_ms = started.elapsed().as_millis(),
2734 "applied a large relay page in bounded projection transactions"
2735 );
2736 }
2737 let delivered_through = self.materialized.applied_event_ordinal;
2738 ensure!(
2739 delivered_through == through_ordinal,
2740 "relay page claimed frontier {} but delivered through {delivered_through}",
2741 through_ordinal
2742 );
2743 ensure!(
2744 self.materialized.applied_event_digest == through_digest,
2745 "relay page digest does not match its claimed frontier"
2746 );
2747 Ok(RelayCursor {
2748 ordinal: delivered_through,
2749 digest: self.materialized.applied_event_digest.clone(),
2750 })
2751 }
2752
2753 pub async fn sync_project_memory(&mut self) -> Result<()> {
2758 let Some(target) = self.project_memory.clone() else {
2759 return Ok(());
2760 };
2761 if !self.client.supports_project_memory_sync() {
2762 tracing::warn!(
2763 session_id = self.materialized.session_id,
2764 "worker protocol predates project-memory synchronization; preserving memory through checkpoints only"
2765 );
2766 self.project_memory = None;
2767 return Ok(());
2768 }
2769 let (baseline, replica) = match self.client.project_memory_snapshot().await {
2770 Ok(snapshot) => snapshot,
2771 Err(error)
2772 if error
2773 .downcast_ref::<RelayRejected>()
2774 .is_some_and(|rejected| {
2775 rejected.0.code == hel::hel_worker::RelayErrorCode::InvalidState
2776 }) =>
2777 {
2778 tracing::warn!(
2779 session_id = self.materialized.session_id,
2780 "worker has no project-memory endpoint; preserving memory through checkpoints only"
2781 );
2782 self.project_memory = None;
2783 return Ok(());
2784 }
2785 Err(error) => return Err(error),
2786 };
2787 let canonical_root = target.canonical_root;
2788 let session_id = self.materialized.session_id.clone();
2789 let (reconciliation, worker_install_needed) = tokio::task::spawn_blocking(move || {
2790 let reconciliation = hel::hel_project_memory::reconcile_into_canonical(
2791 &canonical_root,
2792 &baseline,
2793 &replica,
2794 &session_id,
2795 )?;
2796 let worker_install_needed =
2797 reconciliation.merged != baseline || reconciliation.merged != replica;
2798 Ok::<_, anyhow::Error>((reconciliation, worker_install_needed))
2799 })
2800 .await
2801 .context("project memory reconciliation task failed")??;
2802 for conflict in &reconciliation.conflicts {
2803 tracing::warn!(session_id = self.materialized.session_id, %conflict, "project memory conflict preserved");
2804 }
2805 if worker_install_needed {
2806 self.client
2807 .install_project_memory_snapshot(reconciliation.merged)
2808 .await?;
2809 }
2810 Ok(())
2811 }
2812}
2813
2814fn relay_desynchronized(error: &anyhow::Error) -> bool {
2815 error.chain().any(|cause| {
2816 cause
2817 .downcast_ref::<RelayRejected>()
2818 .is_some_and(RelayRejected::is_desynchronized)
2819 })
2820}
2821
2822fn projection_integrity_failure(error: &anyhow::Error) -> bool {
2823 error
2824 .chain()
2825 .any(|cause| cause.downcast_ref::<ProjectionIntegrityError>().is_some())
2826}
2827
2828pub fn new_command_id(prefix: &str) -> Result<String> {
2829 ensure!(!prefix.trim().is_empty(), "command ID prefix is required");
2830 let mut random = [0_u8; 16];
2831 getrandom::fill(&mut random)
2832 .map_err(|error| anyhow::anyhow!("generate command ID: {error}"))?;
2833 Ok(format!("{prefix}-{}", hex(&random)))
2834}
2835
2836fn hex(bytes: &[u8]) -> String {
2837 const DIGITS: &[u8; 16] = b"0123456789abcdef";
2838 let mut output = String::with_capacity(bytes.len() * 2);
2839 for byte in bytes {
2840 output.push(char::from(DIGITS[usize::from(byte >> 4)]));
2841 output.push(char::from(DIGITS[usize::from(byte & 0x0f)]));
2842 }
2843 output
2844}
2845
2846#[doc(hidden)]
2852pub struct ReplacementSessionTestFixture {
2853 pub stopped: ManagedSessionHandle,
2854 pub control: SessionManagerControl,
2855}
2856
2857#[doc(hidden)]
2861pub fn replacement_session_test_fixture(
2862 session_id: &str,
2863 accepted_ordinal: u64,
2864) -> ReplacementSessionTestFixture {
2865 let (stopped_commands, stopped_commands_rx) = mpsc::channel(1);
2866 drop(stopped_commands_rx);
2867 let (stopped_releases, stopped_releases_rx) = mpsc::unbounded_channel();
2868 drop(stopped_releases_rx);
2869 let (stopped_view_tx, stopped_view) = watch::channel(ManagedSessionView::default());
2870 drop(stopped_view_tx);
2871 let stopped = ManagedSessionHandle {
2872 session_id: session_id.to_owned(),
2873 commands: stopped_commands,
2874 releases: stopped_releases,
2875 view: stopped_view,
2876 };
2877
2878 let (commands, mut commands_rx) = mpsc::channel(4);
2879 let (releases, _releases_rx) = mpsc::unbounded_channel();
2880 let (view_tx, view) = watch::channel(ManagedSessionView::default());
2881 let replacement = ManagedSessionHandle {
2882 session_id: session_id.to_owned(),
2883 commands,
2884 releases,
2885 view,
2886 };
2887 let actor_session_id = session_id.to_owned();
2888 tokio::spawn(async move {
2889 let _view_tx = view_tx;
2890 while let Some(command) = commands_rx.recv().await {
2891 match command {
2892 ActorCommand::Submit { reply, .. } => {
2893 let _ = reply.send(Ok(accepted_ordinal));
2894 }
2895 ActorCommand::Sync { reply } => {
2896 let _ = reply.send(Ok(()));
2897 }
2898 command => command.reject(&actor_session_id, "unsupported test operation"),
2899 }
2900 }
2901 });
2902
2903 let (manager_commands, mut manager_commands_rx) = mpsc::channel(4);
2904 let manager_replacement = replacement.clone();
2905 tokio::spawn(async move {
2906 while let Some(ManagerCommand::Session {
2907 session_id: requested,
2908 reply,
2909 }) = manager_commands_rx.recv().await
2910 {
2911 let resolved =
2912 (requested == manager_replacement.session_id).then(|| manager_replacement.clone());
2913 let _ = reply.send(resolved);
2914 }
2915 });
2916 ReplacementSessionTestFixture {
2917 stopped,
2918 control: SessionManagerControl {
2919 commands: manager_commands,
2920 },
2921 }
2922}
2923
2924#[cfg(test)]
2925mod tests {
2926 use super::*;
2927 #[cfg(unix)]
2928 use agent_client_protocol::schema::v1::{ContentBlock, TextContent};
2929 use sha2::Digest;
2930
2931 fn ordering_request(session_id: &str, command_id: &str) -> RemoteSessionRequest {
2932 let (reply, _response) = oneshot::channel();
2933 RemoteSessionRequest::Submit {
2934 session_id: session_id.into(),
2935 command_id: command_id.into(),
2936 command: RelayCommand::SetConfig {
2937 key: "effort".into(),
2938 value: "high".into(),
2939 },
2940 reply,
2941 }
2942 }
2943
2944 #[tokio::test]
2949 async fn one_session_keeps_its_requests_in_the_order_they_were_made() {
2950 let observed = Arc::new(Mutex::new(Vec::new()));
2951 let release = Arc::new(tokio::sync::Notify::new());
2952 let mut order = SessionRequestOrder::new();
2953
2954 for command_id in ["first", "second", "third"] {
2955 let observed = Arc::clone(&observed);
2956 let release = Arc::clone(&release);
2957 order.dispatch(ordering_request("session-a", command_id), move |request| {
2958 let RemoteSessionRequest::Submit { command_id, .. } = request else {
2959 unreachable!("the fixture only submits")
2960 };
2961 async move {
2962 if command_id == "first" {
2965 release.notified().await;
2966 }
2967 observed.lock().unwrap().push(command_id);
2968 }
2969 });
2970 }
2971
2972 for _ in 0..64 {
2976 tokio::task::yield_now().await;
2977 }
2978 assert!(
2979 observed.lock().unwrap().is_empty(),
2980 "a later request overtook the one being held: {:?}",
2981 observed.lock().unwrap()
2982 );
2983
2984 release.notify_one();
2985 tokio::time::timeout(std::time::Duration::from_secs(5), async {
2986 while observed.lock().unwrap().len() < 3 {
2987 tokio::task::yield_now().await;
2988 }
2989 })
2990 .await
2991 .expect("every request ran");
2992 assert_eq!(*observed.lock().unwrap(), ["first", "second", "third"]);
2993 }
2994
2995 #[tokio::test]
2998 async fn different_sessions_still_overlap() {
2999 let finished = Arc::new(Mutex::new(Vec::new()));
3000 let release = Arc::new(tokio::sync::Notify::new());
3001 let mut order = SessionRequestOrder::new();
3002
3003 let held = Arc::clone(&release);
3004 let recorder = Arc::clone(&finished);
3005 order.dispatch(ordering_request("session-a", "slow"), move |_| async move {
3006 held.notified().await;
3007 recorder.lock().unwrap().push("slow");
3008 });
3009 let recorder = Arc::clone(&finished);
3010 order.dispatch(ordering_request("session-b", "fast"), move |_| async move {
3011 recorder.lock().unwrap().push("fast");
3012 });
3013
3014 tokio::time::timeout(std::time::Duration::from_secs(5), async {
3015 while finished.lock().unwrap().is_empty() {
3016 tokio::task::yield_now().await;
3017 }
3018 })
3019 .await
3020 .expect("the other session ran while the first was held");
3021 assert_eq!(*finished.lock().unwrap(), ["fast"]);
3022
3023 release.notify_one();
3024 tokio::time::timeout(std::time::Duration::from_secs(5), async {
3025 while finished.lock().unwrap().len() < 2 {
3026 tokio::task::yield_now().await;
3027 }
3028 })
3029 .await
3030 .expect("the held request ran once released");
3031 }
3032
3033 #[tokio::test]
3036 async fn finished_sessions_are_forgotten() {
3037 let mut order = SessionRequestOrder::new();
3038 for index in 0..8 {
3039 order.dispatch(
3040 ordering_request(&format!("session-{index}"), "only"),
3041 |_| async {},
3042 );
3043 tokio::time::timeout(std::time::Duration::from_secs(5), async {
3044 while order.latest.values().any(|handle| !handle.is_finished()) {
3045 tokio::task::yield_now().await;
3046 }
3047 })
3048 .await
3049 .expect("the request finished");
3050 }
3051 order.dispatch(ordering_request("session-last", "only"), |_| async {});
3054 assert_eq!(order.latest.len(), 1);
3055 }
3056
3057 #[test]
3060 fn reviewer_actions_and_outcomes_survive_the_daemon_wire() {
3061 let config = ReviewerLaunchConfig {
3062 profile_id: "claude".into(),
3063 harness: hel::hel_config::HarnessKind::Claude,
3064 bridge_command: "npx".into(),
3065 bridge_args: vec!["claude-code-acp".into()],
3066 environment: BTreeMap::from([("EXTRA".into(), "1".into())]),
3067 execution_policy: hel::hel_config::ExecutionPolicy::Unconstrained,
3068 model: Some("sonnet".into()),
3069 effort: Some("high".into()),
3070 generation: 2,
3071 mcp_servers: Vec::new(),
3072 };
3073 let actions = [
3074 ReviewerAction::Start {
3075 config: Box::new(config),
3076 },
3077 ReviewerAction::Submit {
3078 command_id: "review-1".into(),
3079 command: RelayCommand::Cancel,
3080 },
3081 ReviewerAction::Attach {
3082 after_ordinal: 4,
3083 after_digest: "digest".into(),
3084 },
3085 ReviewerAction::Acknowledge {
3086 through_ordinal: 4,
3087 through_digest: "digest".into(),
3088 },
3089 ReviewerAction::Status,
3090 ReviewerAction::Pause,
3091 ReviewerAction::CaptureDelta {
3092 baselines: BTreeMap::from([(std::path::PathBuf::from("/w/app"), "tree".into())]),
3093 },
3094 ReviewerAction::AdvanceBaseline {
3095 trees: BTreeMap::from([(std::path::PathBuf::from("/w/app"), "tree".into())]),
3096 },
3097 ReviewerAction::AnalyzeDelta {
3098 repositories: vec![hel::hel_worker::AnalyzeDeltaRepository {
3099 root: std::path::PathBuf::from("/w/app"),
3100 baseline_tree: Some("base".into()),
3101 current_tree: "target".into(),
3102 }],
3103 },
3104 ];
3105 for action in actions {
3106 let encoded = serde_json::to_string(&action).unwrap();
3107 let decoded: ReviewerAction = serde_json::from_str(&encoded).unwrap();
3108 assert_eq!(decoded, action);
3109 }
3110
3111 let outcome = ReviewerOutcome::Accepted { ordinal: 9 };
3112 let encoded = serde_json::to_string(&outcome).unwrap();
3113 let decoded: ReviewerOutcome = serde_json::from_str(&encoded).unwrap();
3114 assert!(matches!(decoded, ReviewerOutcome::Accepted { ordinal: 9 }));
3115
3116 let paused = serde_json::to_string(&ReviewerOutcome::Paused).unwrap();
3117 assert!(matches!(
3118 serde_json::from_str::<ReviewerOutcome>(&paused).unwrap(),
3119 ReviewerOutcome::Paused
3120 ));
3121
3122 let delta = ReviewerOutcome::Delta {
3123 repositories: vec![hel::hel_worker::RepoDelta {
3124 root: std::path::PathBuf::from("/w/app"),
3125 baseline_tree: None,
3126 current_tree: "target".into(),
3127 patch: "diff --git a/a b/a\n".into(),
3128 diffstat: "1 file changed".into(),
3129 changed_lines: 1,
3130 }],
3131 };
3132 let encoded = serde_json::to_string(&delta).unwrap();
3133 let ReviewerOutcome::Delta { repositories } =
3134 serde_json::from_str::<ReviewerOutcome>(&encoded).unwrap()
3135 else {
3136 panic!("a captured delta must survive the daemon wire");
3137 };
3138 assert_eq!(repositories.len(), 1);
3139 assert_eq!(repositories[0].current_tree, "target");
3140 }
3141
3142 #[test]
3146 fn every_reviewer_action_names_its_operation() {
3147 let names = [
3148 ReviewerAction::Submit {
3149 command_id: String::new(),
3150 command: RelayCommand::Cancel,
3151 }
3152 .operation_name(),
3153 ReviewerAction::Attach {
3154 after_ordinal: 0,
3155 after_digest: String::new(),
3156 }
3157 .operation_name(),
3158 ReviewerAction::Acknowledge {
3159 through_ordinal: 0,
3160 through_digest: String::new(),
3161 }
3162 .operation_name(),
3163 ReviewerAction::Status.operation_name(),
3164 ReviewerAction::Pause.operation_name(),
3165 ];
3166 assert_eq!(
3167 names,
3168 [
3169 "reviewer_submit",
3170 "reviewer_attach",
3171 "reviewer_acknowledge",
3172 "reviewer_status",
3173 "reviewer_pause",
3174 ]
3175 );
3176 assert!(names.iter().all(|name| name.starts_with("reviewer_")));
3177 }
3178
3179 #[test]
3180 fn reconnect_delay_backs_off_and_stops_at_the_ceiling() {
3181 assert_eq!(reconnect_delay(1), RECONNECT_INTERVAL);
3182 assert_eq!(reconnect_delay(2), Duration::from_secs(2));
3183 assert_eq!(reconnect_delay(4), Duration::from_secs(8));
3184 assert_eq!(reconnect_delay(6), RECONNECT_BACKOFF_CEILING);
3185 assert_eq!(reconnect_delay(u32::MAX), RECONNECT_BACKOFF_CEILING);
3186 }
3187
3188 #[test]
3189 fn only_dead_worker_connection_failures_request_a_restart() {
3190 let reworded = anyhow::Error::new(RelayTransportDead::new(
3193 "the session proxy vanished mid-conversation",
3194 ))
3195 .context("connect to the session worker for checkpoint");
3196 assert!(worker_connect_needs_restart(&reworded), "{reworded:#}");
3197 assert!(!worker_connect_allows_live_restart(&reworded));
3198
3199 for detail in [
3202 "relay proxy disconnected during hello",
3203 "Connection refused (os error 111)",
3204 "relay negotiated unsupported protocol 9",
3205 "controller projection is corrupt",
3206 ] {
3207 assert!(!worker_connect_needs_restart(&anyhow::anyhow!(detail)));
3208 }
3209 }
3210
3211 #[cfg(unix)]
3215 #[tokio::test]
3216 async fn a_proxy_that_dies_before_hello_requests_a_worker_restart() {
3217 let mut dead = target("sh");
3218 dead.spec = CommandSpec::new("sh", ["-c", "exit 1"]).purpose("dead relay proxy fixture");
3219
3220 let error = StandaloneSession::connect(&dead)
3221 .await
3222 .err()
3223 .expect("a proxy that exits cannot serve a session");
3224
3225 assert!(worker_connect_needs_restart(&error), "{error:#}");
3226 assert!(worker_connect_allows_live_restart(&error));
3227 }
3228
3229 #[cfg(unix)]
3233 #[tokio::test]
3234 async fn a_failed_lease_keeps_the_cause_that_decides_a_worker_restart() {
3235 let (commands_tx, commands_rx) = mpsc::channel(4);
3236 let (_releases_tx, releases_rx) = mpsc::unbounded_channel();
3237 let (_retirement_tx, retirement_rx) = watch::channel(false);
3238 let (view_tx, _view_rx) = watch::channel(ManagedSessionView::default());
3239 let (updates_tx, _updates_rx) = coalesced_update_channel();
3240 let mut dead = target("sh");
3241 dead.spec = CommandSpec::new("sh", ["-c", "exit 1"]).purpose("dead relay proxy fixture");
3242 tokio::spawn(run_session_actor(
3243 dead,
3244 commands_rx,
3245 releases_rx,
3246 retirement_rx,
3247 view_tx,
3248 updates_tx,
3249 ));
3250
3251 let (reply, response) = oneshot::channel();
3252 commands_tx
3253 .send(ActorCommand::Lease { reply })
3254 .await
3255 .unwrap();
3256 let error = response
3257 .await
3258 .expect("actor answered the lease request")
3259 .err()
3260 .expect("a dead proxy cannot be leased");
3261
3262 assert!(worker_connect_needs_restart(&error), "{error:#}");
3263 }
3264
3265 #[tokio::test]
3266 async fn recovery_restarts_a_live_worker_only_after_a_failed_handshake() {
3267 let directory = tempfile::tempdir().unwrap();
3268 let restarted = directory.path().join("restarted");
3269 let recovery = |liveness: &str| WorkerRecoveryPlan {
3270 target: None,
3271 liveness_probe: CommandSpec::new("printf", [format!("{liveness}\n")])
3272 .purpose("probe test worker liveness"),
3273 binary_refresh: None,
3274 launch_refresh: None,
3275 restart: CommandPlan {
3276 description: "restart test worker".into(),
3277 commands: vec![
3278 CommandSpec::new("touch", [restarted.to_string_lossy().into_owned()])
3279 .purpose("restart test worker"),
3280 ],
3281 },
3282 };
3283
3284 assert_eq!(
3285 recover_worker(recovery("alive"), false).await.unwrap(),
3286 WorkerRecoveryOutcome::Alive
3287 );
3288 assert!(!restarted.exists(), "a live worker must not be restarted");
3289
3290 assert_eq!(
3291 recover_worker(recovery("starting"), true).await.unwrap(),
3292 WorkerRecoveryOutcome::Starting
3293 );
3294 assert!(
3295 !restarted.exists(),
3296 "a worker recovering its journal must not be restarted"
3297 );
3298
3299 assert_eq!(
3300 recover_worker(recovery("alive"), true).await.unwrap(),
3301 WorkerRecoveryOutcome::RestartedUnresponsive
3302 );
3303 assert!(
3304 restarted.exists(),
3305 "a worker that cannot serve a fresh handshake is restarted"
3306 );
3307 std::fs::remove_file(&restarted).unwrap();
3308
3309 assert_eq!(
3310 recover_worker(recovery("dead"), false).await.unwrap(),
3311 WorkerRecoveryOutcome::RestartedDead
3312 );
3313 assert!(restarted.exists(), "a confirmed dead worker is restarted");
3314 }
3315
3316 #[tokio::test]
3317 async fn recovery_replaces_only_a_stale_worker_binary_before_restart() {
3318 let directory = tempfile::tempdir().unwrap();
3319 let source = directory.path().join("current-worker");
3320 let refreshed = directory.path().join("worker-refreshed");
3321 let restarted = directory.path().join("worker-restarted");
3322 std::fs::write(&source, b"current worker binary").unwrap();
3323 let current_digest = format!("{:x}", sha2::Sha256::digest(b"current worker binary"));
3324 let recovery = |installed_digest: &str, require_refresh: bool| {
3325 let mut restart = if require_refresh {
3326 CommandSpec::new(
3327 "sh",
3328 [
3329 "-c",
3330 "test -f \"$MJ_TEST_REFRESHED\" && touch -- \"$MJ_TEST_RESTARTED\"",
3331 ],
3332 )
3333 } else {
3334 CommandSpec::new("touch", [restarted.to_string_lossy().into_owned()])
3335 }
3336 .purpose("restart test worker");
3337 restart.env.insert(
3338 "MJ_TEST_REFRESHED".into(),
3339 refreshed.to_string_lossy().into_owned(),
3340 );
3341 restart.env.insert(
3342 "MJ_TEST_RESTARTED".into(),
3343 restarted.to_string_lossy().into_owned(),
3344 );
3345 WorkerRecoveryPlan {
3346 target: None,
3347 liveness_probe: CommandSpec::new("printf", ["dead\n"])
3348 .purpose("probe test worker liveness"),
3349 binary_refresh: Some(WorkerBinaryRefresh::Prepared(WorkerBinaryRefreshPlan {
3350 source: source.clone(),
3351 installed_digest: CommandSpec::new(
3352 "printf",
3353 [format!("{installed_digest} /worker/hel\n")],
3354 )
3355 .purpose("identify test worker binary"),
3356 replace: CommandPlan {
3357 description: "refresh test worker".into(),
3358 commands: vec![
3359 CommandSpec::new("touch", [refreshed.to_string_lossy().into_owned()])
3360 .purpose("refresh test worker"),
3361 ],
3362 },
3363 })),
3364 launch_refresh: None,
3365 restart: CommandPlan {
3366 description: "restart test worker".into(),
3367 commands: vec![restart],
3368 },
3369 }
3370 };
3371
3372 assert_eq!(
3373 recover_worker(recovery(¤t_digest, false), false)
3374 .await
3375 .unwrap(),
3376 WorkerRecoveryOutcome::RestartedDead
3377 );
3378 assert!(!refreshed.exists(), "a current binary must not be copied");
3379 assert!(restarted.exists());
3380
3381 std::fs::remove_file(&restarted).unwrap();
3382 assert_eq!(
3383 recover_worker(recovery(&"0".repeat(64), true), false)
3384 .await
3385 .unwrap(),
3386 WorkerRecoveryOutcome::RestartedDead
3387 );
3388 assert!(refreshed.exists(), "a stale binary must be refreshed");
3389 assert!(restarted.exists(), "refresh must finish before restart");
3390 }
3391
3392 #[tokio::test]
3393 async fn recovery_refreshes_a_stale_launch_config_before_restart() {
3394 let directory = tempfile::tempdir().unwrap();
3395 let refreshed = directory.path().join("launch-refreshed");
3396 let restarted = directory.path().join("worker-restarted");
3397 let mut restart = CommandSpec::new(
3398 "sh",
3399 [
3400 "-c",
3401 "test -f \"$MJ_TEST_REFRESHED\" && touch -- \"$MJ_TEST_RESTARTED\"",
3402 ],
3403 )
3404 .purpose("restart test worker");
3405 restart.env.insert(
3406 "MJ_TEST_REFRESHED".into(),
3407 refreshed.to_string_lossy().into_owned(),
3408 );
3409 restart.env.insert(
3410 "MJ_TEST_RESTARTED".into(),
3411 restarted.to_string_lossy().into_owned(),
3412 );
3413 let outcome = recover_worker(
3414 WorkerRecoveryPlan {
3415 target: None,
3416 liveness_probe: CommandSpec::new("printf", ["dead\n"])
3417 .purpose("probe test worker liveness"),
3418 binary_refresh: None,
3419 launch_refresh: Some(WorkerLaunchRefreshPlan {
3420 expected_sha256: "a".repeat(64),
3421 installed_digest: CommandSpec::new(
3422 "printf",
3423 [format!("{} /worker/launch.json\n", "b".repeat(64))],
3424 )
3425 .purpose("identify test launch config"),
3426 replace: CommandPlan {
3427 description: "refresh test launch config".into(),
3428 commands: vec![
3429 CommandSpec::new("touch", [refreshed.to_string_lossy().into_owned()])
3430 .purpose("refresh test launch config"),
3431 ],
3432 },
3433 }),
3434 restart: CommandPlan {
3435 description: "restart test worker".into(),
3436 commands: vec![restart],
3437 },
3438 },
3439 false,
3440 )
3441 .await
3442 .unwrap();
3443
3444 assert_eq!(outcome, WorkerRecoveryOutcome::RestartedDead);
3445 assert!(refreshed.exists());
3446 assert!(
3447 restarted.exists(),
3448 "config refresh must finish before restart"
3449 );
3450 }
3451
3452 #[tokio::test]
3453 async fn recovery_starts_a_stopped_target_before_probing_its_worker() {
3454 let directory = tempfile::tempdir().unwrap();
3455 let target_started = directory.path().join("target-started");
3456 let worker_restarted = directory.path().join("worker-restarted");
3457 let inspection = |status: &str| {
3458 serde_json::to_string(&serde_json::json!([{
3459 "Config": { "Labels": {
3460 (hel::hel_targets::MANAGED_LABEL): "true",
3461 (hel::hel_targets::SESSION_LABEL): "session-1",
3462 }},
3463 "State": { "Status": status },
3464 }]))
3465 .unwrap()
3466 };
3467 let mut inspect = CommandSpec::new(
3468 "sh",
3469 [
3470 "-c",
3471 "if [ -f \"$MJ_TEST_TARGET_STARTED\" ]; then printf '%s\\n' \"$MJ_TEST_RUNNING\"; else printf '%s\\n' \"$MJ_TEST_EXITED\"; fi",
3472 ],
3473 )
3474 .purpose("inspect test target");
3475 inspect.env.insert(
3476 "MJ_TEST_TARGET_STARTED".into(),
3477 target_started.to_string_lossy().into_owned(),
3478 );
3479 inspect
3480 .env
3481 .insert("MJ_TEST_RUNNING".into(), inspection("running"));
3482 inspect
3483 .env
3484 .insert("MJ_TEST_EXITED".into(), inspection("exited"));
3485 let mut start = CommandSpec::new("sh", ["-c", "touch -- \"$MJ_TEST_TARGET_STARTED\""])
3486 .purpose("start test target");
3487 start.env.insert(
3488 "MJ_TEST_TARGET_STARTED".into(),
3489 target_started.to_string_lossy().into_owned(),
3490 );
3491 let mut liveness = CommandSpec::new(
3492 "sh",
3493 [
3494 "-c",
3495 "test -f \"$MJ_TEST_TARGET_STARTED\" && printf 'dead\\n'",
3496 ],
3497 )
3498 .purpose("probe test worker after target start");
3499 liveness.env.insert(
3500 "MJ_TEST_TARGET_STARTED".into(),
3501 target_started.to_string_lossy().into_owned(),
3502 );
3503
3504 let outcome = recover_worker(
3505 WorkerRecoveryPlan {
3506 target: Some(TargetRecoveryPlan {
3507 exists: CommandSpec::new("true", std::iter::empty::<&str>())
3508 .purpose("check test target"),
3509 inspect,
3510 start,
3511 session_id: "session-1".into(),
3512 }),
3513 liveness_probe: liveness,
3514 binary_refresh: None,
3515 launch_refresh: None,
3516 restart: CommandPlan {
3517 description: "restart test worker".into(),
3518 commands: vec![
3519 CommandSpec::new(
3520 "touch",
3521 [worker_restarted.to_string_lossy().into_owned()],
3522 )
3523 .purpose("restart test worker"),
3524 ],
3525 },
3526 },
3527 false,
3528 )
3529 .await
3530 .unwrap();
3531
3532 assert_eq!(outcome, WorkerRecoveryOutcome::RestartedDead);
3533 assert!(target_started.exists());
3534 assert!(worker_restarted.exists());
3535 }
3536
3537 #[tokio::test]
3538 async fn recovery_reports_a_missing_target_without_running_worker_commands() {
3539 let unreachable = CommandSpec::new("false", std::iter::empty::<&str>());
3540 let outcome = recover_worker(
3541 WorkerRecoveryPlan {
3542 target: Some(TargetRecoveryPlan {
3543 exists: unreachable,
3544 inspect: CommandSpec::new("false", std::iter::empty::<&str>()),
3545 start: CommandSpec::new("false", std::iter::empty::<&str>()),
3546 session_id: "session-1".into(),
3547 }),
3548 liveness_probe: CommandSpec::new("false", std::iter::empty::<&str>()),
3549 binary_refresh: None,
3550 launch_refresh: None,
3551 restart: CommandPlan {
3552 description: "must not restart".into(),
3553 commands: vec![CommandSpec::new("false", std::iter::empty::<&str>())],
3554 },
3555 },
3556 true,
3557 )
3558 .await
3559 .unwrap();
3560
3561 assert_eq!(outcome, WorkerRecoveryOutcome::TargetMissing);
3562 }
3563
3564 fn target(program: &str) -> RelaySessionTarget {
3565 RelaySessionTarget {
3566 session_id: "session-1".to_owned(),
3567 spec: CommandSpec::new(program, std::iter::empty::<&str>()),
3568 worker_recovery: None,
3569 project_memory: None,
3570 }
3571 }
3572
3573 fn view_at_ordinal(ordinal: u64) -> ManagedSessionView {
3576 let digest = "a".repeat(64);
3577 let mut materialized = MaterializedSession::empty("session-1");
3578 materialized.applied_event_ordinal = ordinal;
3579 materialized.applied_event_digest = digest.clone();
3580 materialized.transcript = (1..=200)
3581 .map(|position| {
3582 Arc::new(hel::hel_state::TranscriptItem {
3583 stable_id: format!("system:{position}"),
3584 position,
3585 latest_content_event_ordinal: None,
3586 created_at_ms: 1,
3587 last_changed_at_ms: 1,
3588 body: hel::hel_state::TranscriptBody::System {
3589 text: format!("event {position}"),
3590 },
3591 })
3592 })
3593 .collect();
3594 ManagedSessionView {
3595 snapshot: Some(ManagedSessionSnapshot {
3596 window: hel::hel_state::ProjectionWindow::of(&materialized),
3597 materialized,
3598 operational: RelayOperationalState {
3599 session_id: "session-1".into(),
3600 execution: hel::hel_worker::RelayExecutionState::Idle,
3601 latest_ordinal: ordinal,
3602 latest_digest: digest.clone(),
3603 acknowledged_through: ordinal,
3604 acknowledged_digest: digest,
3605 recovery_floor_ordinal: 0,
3606 recovery_floor_digest: hel::hel_worker::RELAY_EVENT_GENESIS_DIGEST.into(),
3607 native_session_id: None,
3608 agent_capabilities: None,
3609 agent_info: None,
3610 config_options: Vec::new(),
3611 modes: None,
3612 available_commands: Vec::new(),
3613 config: BTreeMap::new(),
3614 active_prompt: None,
3615 queued_prompts: Vec::new(),
3616 active_user_shells: Vec::new(),
3617 active_agent_terminals: Vec::new(),
3618 checkpoint_barrier: None,
3619 checkpoint_ready: None,
3620 last_acp_activity_at_ms: None,
3621 current_step_started_at_ms: None,
3622 foreground_tool_started_at_ms: None,
3623 harness_turn: None,
3624 last_harness_turn_started_ordinal: None,
3625 background_commands: Vec::new(),
3626 },
3627 latest_credential_sync_signal: None,
3628 worker_build: None,
3629 }),
3630 connected: true,
3631 error: None,
3632 }
3633 }
3634
3635 #[test]
3636 fn republishing_an_unchanged_view_notifies_nobody() {
3637 let (view_tx, mut view_rx) = watch::channel(ManagedSessionView::default());
3638 let (updates_tx, mut updates_rx) = coalesced_update_channel();
3639
3640 publish_view("session-1", view_at_ordinal(7), &view_tx, &updates_tx);
3641 assert!(view_rx.has_changed().expect("watch stays open"));
3642 assert_eq!(
3643 updates_rx.try_recv().expect("the first view is news").view,
3644 view_at_ordinal(7)
3645 );
3646 let _ = view_rx.borrow_and_update();
3647
3648 publish_view("session-1", view_at_ordinal(7), &view_tx, &updates_tx);
3649
3650 assert!(
3651 !view_rx.has_changed().expect("watch stays open"),
3652 "a sync tick that moved nothing must not wake the dashboard"
3653 );
3654 assert!(updates_rx.try_recv().is_err());
3655 }
3656
3657 #[test]
3658 fn publishing_an_advanced_event_frontier_notifies_watchers() {
3659 let (view_tx, mut view_rx) = watch::channel(ManagedSessionView::default());
3660 let (updates_tx, mut updates_rx) = coalesced_update_channel();
3661 publish_view("session-1", view_at_ordinal(7), &view_tx, &updates_tx);
3662 let _ = updates_rx.try_recv();
3663 let _ = view_rx.borrow_and_update();
3664
3665 publish_view("session-1", view_at_ordinal(8), &view_tx, &updates_tx);
3666
3667 assert!(view_rx.has_changed().expect("watch stays open"));
3668 let update = updates_rx.try_recv().expect("the advance is news");
3669 assert_eq!(update.session_id, "session-1");
3670 assert_eq!(
3671 update
3672 .view
3673 .snapshot
3674 .expect("published snapshot")
3675 .materialized
3676 .applied_event_ordinal,
3677 8
3678 );
3679 }
3680
3681 #[test]
3682 fn publishing_relay_state_that_moved_without_the_frontier_notifies_watchers() {
3683 let (view_tx, mut view_rx) = watch::channel(ManagedSessionView::default());
3684 let (updates_tx, mut updates_rx) = coalesced_update_channel();
3685 publish_view("session-1", view_at_ordinal(7), &view_tx, &updates_tx);
3686 let _ = updates_rx.try_recv();
3687 let _ = view_rx.borrow_and_update();
3688
3689 let mut view = view_at_ordinal(7);
3690 view.snapshot
3691 .as_mut()
3692 .expect("published snapshot")
3693 .operational
3694 .execution = hel::hel_worker::RelayExecutionState::Running;
3695 publish_view("session-1", view, &view_tx, &updates_tx);
3696
3697 assert!(view_rx.has_changed().expect("watch stays open"));
3698 assert!(updates_rx.try_recv().is_ok());
3699 }
3700
3701 #[test]
3702 fn losing_the_relay_republishes_the_same_snapshot_as_disconnected() {
3703 let (view_tx, mut view_rx) = watch::channel(ManagedSessionView::default());
3704 let (updates_tx, mut updates_rx) = coalesced_update_channel();
3705 publish_view("session-1", view_at_ordinal(7), &view_tx, &updates_tx);
3706 let _ = updates_rx.try_recv();
3707 let _ = view_rx.borrow_and_update();
3708
3709 let mut view = view_at_ordinal(7);
3710 view.connected = false;
3711 view.error = Some(ViewError::Unreachable("relay is unreachable".into()));
3712 publish_view("session-1", view, &view_tx, &updates_tx);
3713
3714 assert!(view_rx.has_changed().expect("watch stays open"));
3715 assert!(updates_rx.try_recv().is_ok());
3716 }
3717
3718 #[test]
3719 fn command_ids_are_namespaced_and_unique() {
3720 let first = new_command_id("prompt").unwrap();
3721 let second = new_command_id("prompt").unwrap();
3722 assert!(first.starts_with("prompt-"));
3723 assert_ne!(first, second);
3724 }
3725
3726 #[test]
3727 fn leased_actor_defers_replacement_and_uses_latest_queued_target() {
3728 let original = target("relay-v1");
3729 let intermediate = target("relay-v2");
3730 let latest = target("relay-v3");
3731 let mut lifecycle = ActorLifecycle::default();
3732 lifecycle.activate_lease(7);
3733
3734 assert_eq!(
3735 reconcile_action(Some(&original), Some(&intermediate)),
3736 ReconcileAction::Retire
3737 );
3738 lifecycle.set_retirement_requested(true);
3739 assert!(!lifecycle.accepts_new_work());
3740 assert!(!lifecycle.should_stop());
3741
3742 assert_eq!(
3743 reconcile_action(Some(&original), Some(&latest)),
3744 ReconcileAction::Retire
3745 );
3746 assert!(lifecycle.return_lease(7));
3747 assert!(lifecycle.should_stop());
3748
3749 assert_eq!(
3750 reconcile_action(None, Some(&latest)),
3751 ReconcileAction::Spawn
3752 );
3753 }
3754
3755 #[test]
3756 fn leased_actor_defers_removal_until_its_connection_returns() {
3757 let original = target("relay-v1");
3758 let mut lifecycle = ActorLifecycle::default();
3759 lifecycle.activate_lease(11);
3760
3761 assert_eq!(
3762 reconcile_action(Some(&original), None),
3763 ReconcileAction::Retire
3764 );
3765 lifecycle.set_retirement_requested(true);
3766 assert!(!lifecycle.should_stop());
3767 assert!(!lifecycle.return_lease(10));
3768 assert!(!lifecycle.should_stop());
3769 assert!(lifecycle.return_lease(11));
3770 assert!(lifecycle.should_stop());
3771 assert_eq!(reconcile_action(None, None), ReconcileAction::Idle);
3772 }
3773
3774 #[test]
3775 fn queued_change_back_to_current_target_cancels_retirement() {
3776 let original = target("relay-v1");
3777 let replacement = target("relay-v2");
3778 let mut lifecycle = ActorLifecycle::default();
3779 lifecycle.activate_lease(3);
3780
3781 assert_eq!(
3782 reconcile_action(Some(&original), Some(&replacement)),
3783 ReconcileAction::Retire
3784 );
3785 lifecycle.set_retirement_requested(true);
3786 assert_eq!(
3787 reconcile_action(Some(&original), Some(&original)),
3788 ReconcileAction::Keep
3789 );
3790 lifecycle.set_retirement_requested(false);
3791
3792 assert!(lifecycle.return_lease(3));
3793 assert!(!lifecycle.should_stop());
3794 assert!(lifecycle.accepts_new_work());
3795 }
3796
3797 #[tokio::test]
3798 async fn stopped_actor_is_replaced_without_late_completion_removing_replacement() {
3799 let desired = target("sh");
3800 let desired_targets = target_map(std::slice::from_ref(&desired));
3801 let mut actors = BTreeMap::new();
3802 let mut tasks = tokio::task::JoinSet::new();
3803 let (commands, commands_rx) = mpsc::channel(1);
3804 drop(commands_rx);
3805 let (releases, _releases_rx) = mpsc::unbounded_channel();
3806 let (retirement, _retirement_rx) = watch::channel(false);
3807 let (_view_tx, view) = watch::channel(ManagedSessionView::default());
3808 let old_abort = tasks.spawn(async { "session-1".to_owned() });
3809 let old_task_id = old_abort.id();
3810 actors.insert(
3811 "session-1".to_owned(),
3812 ActorRegistration {
3813 target: desired.clone(),
3814 commands,
3815 releases,
3816 retirement,
3817 view,
3818 abort: old_abort,
3819 },
3820 );
3821 let (updates, _updates_rx) = coalesced_update_channel();
3822
3823 reconcile_actors(&desired_targets, &mut actors, &mut tasks, &updates);
3824
3825 let replacement_task_id = actors["session-1"].abort.id();
3826 assert_ne!(replacement_task_id, old_task_id);
3827 assert!(!actors["session-1"].commands.is_closed());
3828 assert_eq!(remove_actor_task(&mut actors, old_task_id), None);
3829 assert_eq!(actors["session-1"].abort.id(), replacement_task_id);
3830 tasks.abort_all();
3831 }
3832
3833 const UNREACHABLE_VIEW_TEST_CHILD: &str = "MJ_TEST_UNREACHABLE_RELAY_CHILD";
3834
3835 #[tokio::test(start_paused = true)]
3836 async fn unreachable_relay_publishes_error_view() {
3837 if std::env::var_os(UNREACHABLE_VIEW_TEST_CHILD).is_none() {
3841 let directory = tempfile::tempdir().unwrap();
3842 let test_name = format!(
3843 "{}::unreachable_relay_publishes_error_view",
3844 module_path!()
3845 .strip_prefix("mj_controller::")
3846 .unwrap_or(module_path!())
3847 );
3848 let output = std::process::Command::new(std::env::current_exe().unwrap())
3849 .args(["--exact", &test_name, "--nocapture"])
3850 .env(UNREACHABLE_VIEW_TEST_CHILD, "1")
3851 .env("MJ_DATA_DIR", directory.path())
3852 .output()
3853 .unwrap();
3854 assert!(
3855 output.status.success(),
3856 "isolated unreachable relay test failed\nstdout:\n{}\nstderr:\n{}",
3857 String::from_utf8_lossy(&output.stdout),
3858 String::from_utf8_lossy(&output.stderr)
3859 );
3860 return;
3861 }
3862
3863 std::thread::spawn(|| {
3866 std::thread::sleep(Duration::from_secs(60));
3867 eprintln!("unreachable relay error view was never published");
3868 std::process::exit(101);
3869 });
3870
3871 let (_commands_tx, commands_rx) = mpsc::channel(4);
3872 let (_releases_tx, releases_rx) = mpsc::unbounded_channel();
3873 let (_retirement_tx, retirement_rx) = watch::channel(false);
3874 let (view_tx, mut view_rx) = watch::channel(ManagedSessionView::default());
3875 let (updates_tx, mut updates_rx) = coalesced_update_channel();
3876 tokio::spawn(run_session_actor(
3877 target("hel-relay-program-that-does-not-exist"),
3878 commands_rx,
3879 releases_rx,
3880 retirement_rx,
3881 view_tx,
3882 updates_tx,
3883 ));
3884
3885 loop {
3886 view_rx.changed().await.unwrap();
3887 let view = view_rx.borrow_and_update().clone();
3888 if !view.connected {
3889 let error = view
3890 .error
3891 .expect("unreachable view carries the connect error");
3892 assert!(
3893 error.detail().contains("session relay proxy"),
3894 "unexpected error: {error:?}"
3895 );
3896 break;
3897 }
3898 }
3899 let update = updates_rx
3900 .recv()
3901 .await
3902 .expect("dashboard feed received the error view");
3903 assert_eq!(update.session_id, "session-1");
3904 assert!(!update.view.connected);
3905 }
3906
3907 const UNREADABLE_PROJECTION_TEST_CHILD: &str = "MJ_TEST_UNREADABLE_PROJECTION_CHILD";
3908
3909 #[tokio::test]
3910 async fn connecting_to_an_absent_worker_never_reads_the_projection() {
3911 if std::env::var_os(UNREADABLE_PROJECTION_TEST_CHILD).is_none() {
3915 let directory = tempfile::tempdir().unwrap();
3916 std::fs::create_dir(directory.path().join("mj.sqlite3")).unwrap();
3920 let output = std::process::Command::new(std::env::current_exe().unwrap())
3921 .args([
3922 "--exact",
3923 &format!(
3924 "{}::connecting_to_an_absent_worker_never_reads_the_projection",
3925 module_path!()
3926 .strip_prefix("mj_controller::")
3927 .unwrap_or(module_path!())
3928 ),
3929 "--nocapture",
3930 ])
3931 .env(UNREADABLE_PROJECTION_TEST_CHILD, "1")
3932 .env("MJ_DATA_DIR", directory.path())
3933 .output()
3934 .unwrap();
3935 assert!(
3936 output.status.success(),
3937 "isolated projection ordering test failed\nstdout:\n{}\nstderr:\n{}",
3938 String::from_utf8_lossy(&output.stdout),
3939 String::from_utf8_lossy(&output.stderr)
3940 );
3941 return;
3942 }
3943
3944 assert!(
3945 hel::hel_database::load_materialized_session("session-1").is_err(),
3946 "this store must fail every projection read for the test to mean anything"
3947 );
3948 let connected =
3949 StandaloneSession::connect(&target("hel-relay-program-that-does-not-exist")).await;
3950 let error = match connected {
3951 Ok(_) => panic!("a relay program that does not exist cannot connect"),
3952 Err(error) => error,
3953 };
3954 let detail = format!("{error:#}");
3955 assert!(
3956 detail.contains("session relay proxy"),
3957 "unexpected error: {detail}"
3958 );
3959 assert!(
3960 !detail.contains("Mjolnir database"),
3961 "connect read the projection before it reached the relay: {detail}"
3962 );
3963 }
3964
3965 const LEASED_RELAY_ROOT: &str = "MJ_TEST_LEASED_RELAY_ROOT";
3966 #[cfg(unix)]
3967 const AUTO_RESTART_TEST_CHILD: &str = "MJ_TEST_AUTO_RESTART_CHILD";
3968 #[cfg(unix)]
3969 const AUTO_RESTART_MARKER: &str = "MJ_TEST_AUTO_RESTART_MARKER";
3970 #[cfg(unix)]
3971 const DEFERRED_SUBMIT_TEST_CHILD: &str = "MJ_TEST_DEFERRED_SUBMIT_CHILD";
3972 #[cfg(unix)]
3973 const RETIRED_SUBMIT_TEST_CHILD: &str = "MJ_TEST_RETIRED_SUBMIT_CHILD";
3974 #[cfg(unix)]
3975 const RETURNED_LEASE_VIEW_TEST_CHILD: &str = "MJ_TEST_RETURNED_LEASE_VIEW_CHILD";
3976 #[cfg(unix)]
3977 const EXPLICIT_MEMORY_SYNC_TEST_CHILD: &str = "MJ_TEST_EXPLICIT_MEMORY_SYNC_CHILD";
3978 #[cfg(unix)]
3979 const SUBMIT_WITHOUT_SYNC_TEST_CHILD: &str = "MJ_TEST_SUBMIT_WITHOUT_SYNC_CHILD";
3980 #[cfg(unix)]
3981 const MANAGER_SHUTDOWN_TEST_CHILD: &str = "MJ_TEST_MANAGER_SHUTDOWN_CHILD";
3982 const LEASED_RELAY_SESSION: &str = "018f9dd2-a3b4-7c8d-9000-123456789abc";
3983
3984 #[test]
3987 fn leased_relay_child_serves_stdio() {
3988 let Some(root) = std::env::var_os(LEASED_RELAY_ROOT) else {
3989 return;
3990 };
3991 println!();
3995 let mut relay = hel::hel_worker::DurableRelay::open(
3996 std::path::Path::new(&root),
3997 LEASED_RELAY_SESSION,
3998 "1.0.0",
3999 )
4000 .expect("open the test relay journal");
4001 hel::hel_worker::serve_relay_json_lines(
4002 &mut std::io::stdin().lock(),
4003 &mut std::io::stdout().lock(),
4004 &mut relay,
4005 )
4006 .expect("serve relay frames until the controller disconnects");
4007 }
4008
4009 #[cfg(unix)]
4010 fn exact_test_name(test: &str) -> String {
4011 format!(
4012 "{}::{test}",
4013 module_path!()
4014 .strip_prefix("mj_controller::")
4015 .unwrap_or(module_path!())
4016 )
4017 }
4018
4019 #[cfg(unix)]
4022 fn run_in_isolated_child(marker: &str, test: &str) {
4023 let directory = tempfile::tempdir().unwrap();
4024 let output = std::process::Command::new(std::env::current_exe().unwrap())
4025 .args(["--exact", &exact_test_name(test), "--nocapture"])
4026 .env(marker, "1")
4027 .env("MJ_DATA_DIR", directory.path())
4028 .output()
4029 .unwrap();
4030 assert!(
4031 output.status.success(),
4032 "isolated {test} failed\nstdout:\n{}\nstderr:\n{}",
4033 String::from_utf8_lossy(&output.stdout),
4034 String::from_utf8_lossy(&output.stderr)
4035 );
4036 }
4037
4038 #[cfg(unix)]
4039 #[tokio::test]
4040 async fn session_manager_shutdown_joins_a_live_relay_actor() {
4041 if std::env::var_os(MANAGER_SHUTDOWN_TEST_CHILD).is_none() {
4042 run_in_isolated_child(
4043 MANAGER_SHUTDOWN_TEST_CHILD,
4044 "session_manager_shutdown_joins_a_live_relay_actor",
4045 );
4046 return;
4047 }
4048 let _writer = hel::hel_database::install_isolated_test_writer();
4050 register_leased_relay_session();
4051 let relay_root = tempfile::tempdir().unwrap();
4052 let SessionManagerChannels {
4053 targets,
4054 control,
4055 updates: _updates,
4056 shutdown,
4057 } = spawn_session_manager().expect("spawn the session manager");
4058 targets.send_replace(vec![leased_relay_target(relay_root.path())]);
4059 let session = control
4060 .wait_for_session(LEASED_RELAY_SESSION, Duration::from_secs(2))
4061 .await
4062 .expect("manager registered the relay actor");
4063 session
4064 .sync_now()
4065 .await
4066 .expect("relay actor established a live connection");
4067 assert!(session.view().connected);
4068
4069 tokio::time::timeout(Duration::from_secs(2), shutdown.shutdown())
4070 .await
4071 .expect("manager shutdown stayed within its deadline")
4072 .expect("manager shutdown task completed cleanly");
4073 }
4074
4075 #[cfg(unix)]
4076 #[tokio::test]
4077 async fn relay_attach_does_not_probe_or_install_project_memory() {
4078 if std::env::var_os(EXPLICIT_MEMORY_SYNC_TEST_CHILD).is_none() {
4079 run_in_isolated_child(
4080 EXPLICIT_MEMORY_SYNC_TEST_CHILD,
4081 "relay_attach_does_not_probe_or_install_project_memory",
4082 );
4083 return;
4084 }
4085 let _writer = hel::hel_database::install_isolated_test_writer();
4087 register_leased_relay_session();
4088 let relay_root = tempfile::tempdir().unwrap();
4089 let canonical = tempfile::tempdir().unwrap();
4090 let mut target = leased_relay_target(relay_root.path());
4091 target.project_memory = Some(ProjectMemorySyncTarget {
4092 canonical_root: canonical.path().to_path_buf(),
4093 });
4094
4095 let mut connection = StandaloneSession::connect(&target)
4096 .await
4097 .expect("relay attach must not depend on its memory endpoint");
4098 assert!(
4099 connection.project_memory.is_some(),
4100 "attach must leave memory pending for an explicit checkpoint sync"
4101 );
4102
4103 connection
4104 .sync_project_memory()
4105 .await
4106 .expect("an explicit sync may detect a legacy memory endpoint");
4107 assert!(
4108 connection.project_memory.is_none(),
4109 "the explicit sync reached the relay and disabled its unavailable endpoint"
4110 );
4111 }
4112
4113 #[cfg(unix)]
4118 #[tokio::test]
4119 async fn submitting_does_not_catch_the_projection_up_until_asked() {
4120 if std::env::var_os(SUBMIT_WITHOUT_SYNC_TEST_CHILD).is_none() {
4121 run_in_isolated_child(
4122 SUBMIT_WITHOUT_SYNC_TEST_CHILD,
4123 "submitting_does_not_catch_the_projection_up_until_asked",
4124 );
4125 return;
4126 }
4127 let _writer = hel::hel_database::install_isolated_test_writer();
4129 register_leased_relay_session();
4130 let relay_root = tempfile::tempdir().unwrap();
4131 let mut connection = StandaloneSession::connect(&leased_relay_target(relay_root.path()))
4132 .await
4133 .expect("connect to the live test relay");
4134 let before = connection.materialized.applied_event_ordinal;
4135
4136 let ordinal = connection
4137 .submit_accepted(
4138 new_command_id("prompt").unwrap(),
4139 RelayCommand::Prompt {
4140 prompt: vec![ContentBlock::Text(TextContent::new("hello"))],
4141 },
4142 )
4143 .await
4144 .expect("the relay accepted the command");
4145 assert!(ordinal > before, "the relay reported where it accepted it");
4146 assert_eq!(
4147 connection.materialized.applied_event_ordinal, before,
4148 "the caller was answered without paying for the catch-up"
4149 );
4150
4151 connection.sync().await.expect("catch the projection up");
4152 assert!(
4153 connection.materialized.applied_event_ordinal > before,
4154 "the catch-up is what advances the projection"
4155 );
4156 }
4157
4158 #[cfg(unix)]
4159 #[tokio::test]
4160 async fn unresponsive_live_relay_worker_is_restarted_and_reconnected() {
4161 if std::env::var_os(AUTO_RESTART_TEST_CHILD).is_none() {
4162 run_in_isolated_child(
4163 AUTO_RESTART_TEST_CHILD,
4164 "unresponsive_live_relay_worker_is_restarted_and_reconnected",
4165 );
4166 return;
4167 }
4168 let _writer = hel::hel_database::install_isolated_test_writer();
4170 fail_if_the_actor_stalls("unresponsive live relay worker was never restarted");
4171 register_leased_relay_session();
4172 let relay_root = tempfile::tempdir().unwrap();
4173 let restarted = relay_root.path().join("worker-restarted");
4174 let script = format!(
4175 "if [ ! -f \"${AUTO_RESTART_MARKER}\" ]; then IFS= read -r _; exit 0; fi; \
4176 \"$0\" --exact {} --nocapture | grep --line-buffered '^{{'",
4177 exact_test_name("leased_relay_child_serves_stdio")
4178 );
4179 let mut spec = CommandSpec::new(
4180 "sh",
4181 [
4182 "-c".to_owned(),
4183 script,
4184 std::env::current_exe()
4185 .unwrap()
4186 .to_string_lossy()
4187 .into_owned(),
4188 ],
4189 )
4190 .purpose("test restartable relay");
4191 spec.env.insert(
4192 LEASED_RELAY_ROOT.to_owned(),
4193 relay_root.path().to_string_lossy().into_owned(),
4194 );
4195 spec.env.insert(
4196 AUTO_RESTART_MARKER.to_owned(),
4197 restarted.to_string_lossy().into_owned(),
4198 );
4199 let worker_recovery = WorkerRecoveryPlan {
4200 target: None,
4201 liveness_probe: CommandSpec::new("printf", ["alive\n"])
4202 .purpose("probe test relay worker"),
4203 binary_refresh: None,
4204 launch_refresh: None,
4205 restart: CommandPlan {
4206 description: "restart test relay worker".into(),
4207 commands: vec![
4208 CommandSpec::new("touch", [restarted.to_string_lossy().into_owned()])
4209 .purpose("restart test relay worker"),
4210 ],
4211 },
4212 };
4213 let target = RelaySessionTarget {
4214 session_id: LEASED_RELAY_SESSION.to_owned(),
4215 spec,
4216 worker_recovery: Some(worker_recovery),
4217 project_memory: None,
4218 };
4219 let (_commands_tx, commands_rx) = mpsc::channel(4);
4220 let (_releases_tx, releases_rx) = mpsc::unbounded_channel();
4221 let (_retirement_tx, retirement_rx) = watch::channel(false);
4222 let (view_tx, mut view_rx) = watch::channel(ManagedSessionView::default());
4223 let (updates_tx, _updates_rx) = coalesced_update_channel();
4224 tokio::spawn(run_session_actor(
4225 target,
4226 commands_rx,
4227 releases_rx,
4228 retirement_rx,
4229 view_tx,
4230 updates_tx,
4231 ));
4232
4233 tokio::time::timeout(Duration::from_secs(20), async {
4234 loop {
4235 view_rx.changed().await.unwrap();
4236 let view = view_rx.borrow_and_update().clone();
4237 if view.connected {
4238 assert!(restarted.exists(), "the restart plan did not run");
4239 assert!(view.error.is_none());
4240 return;
4241 }
4242 }
4243 })
4244 .await
4245 .unwrap_or_else(|_| panic!("relay stayed disconnected: {:?}", view_rx.borrow().error));
4246 }
4247
4248 #[cfg(unix)]
4251 fn fail_if_the_actor_stalls(reason: &'static str) {
4252 std::thread::spawn(move || {
4253 std::thread::sleep(Duration::from_secs(60));
4254 eprintln!("{reason}");
4255 std::process::exit(101);
4256 });
4257 }
4258
4259 #[cfg(unix)]
4261 fn leased_relay_target(relay_root: &std::path::Path) -> RelaySessionTarget {
4262 let script = format!(
4265 "\"$0\" --exact {} --nocapture | grep --line-buffered '^{{'",
4266 exact_test_name("leased_relay_child_serves_stdio")
4267 );
4268 let mut spec = CommandSpec::new(
4269 "sh",
4270 [
4271 "-c".to_owned(),
4272 script,
4273 std::env::current_exe()
4274 .unwrap()
4275 .to_string_lossy()
4276 .into_owned(),
4277 ],
4278 )
4279 .purpose("test leased relay");
4280 spec.env.insert(
4281 LEASED_RELAY_ROOT.to_owned(),
4282 relay_root.to_string_lossy().into_owned(),
4283 );
4284 RelaySessionTarget {
4285 session_id: LEASED_RELAY_SESSION.to_owned(),
4286 spec,
4287 worker_recovery: None,
4288 project_memory: None,
4289 }
4290 }
4291
4292 #[cfg(unix)]
4295 fn register_leased_relay_session() {
4296 hel::hel_database::save_session(&hel::hel_state::SessionRecord {
4297 workspace_id: hel::hel_workspace::DEFAULT_WORKSPACE_ID.to_owned(),
4298 archived: false,
4299 container_cpus: None,
4300 container_memory: None,
4301 id: LEASED_RELAY_SESSION.into(),
4302 title: "leased relay".into(),
4303 harness_kind: hel::hel_config::HarnessKind::Codex,
4304 last_profile: "codex".into(),
4305 bundle_id: "project".into(),
4306 project_directory: None,
4307 managed_worktree: None,
4308 target_template_id: "podman".into(),
4309 resource_allocation: None,
4310 additional_mounts: Vec::new(),
4311 state: hel::hel_state::SessionState::Running,
4312 target: None,
4313 native_session_id: None,
4314 acp_session_title: None,
4315 session_title_override: None,
4316 created_at: "2026-08-12T00:00:00Z".into(),
4317 updated_at: "2026-08-12T00:00:00Z".into(),
4318 viewed_through_event_ordinal: 0,
4319 draft_input: String::new(),
4320 last_error: None,
4321 last_checkpoint_error: None,
4322 checkpoint: None,
4323 })
4324 .expect("register the test session");
4325 }
4326
4327 #[cfg(unix)]
4328 struct LeasedActor {
4329 commands: mpsc::Sender<ActorCommand>,
4330 releases: mpsc::UnboundedSender<ReturnedConnection>,
4331 retirement: watch::Sender<bool>,
4332 _views: watch::Receiver<ManagedSessionView>,
4333 _updates: SessionManagerUpdates,
4334 _relay_root: tempfile::TempDir,
4335 }
4336
4337 #[cfg(unix)]
4339 async fn lease_a_live_actor() -> (LeasedActor, u64, StandaloneSession) {
4340 register_leased_relay_session();
4341 let relay_root = tempfile::tempdir().unwrap();
4342 let (commands_tx, commands_rx) = mpsc::channel(4);
4343 let (releases_tx, releases_rx) = mpsc::unbounded_channel();
4344 let (retirement_tx, retirement_rx) = watch::channel(false);
4345 let (view_tx, view_rx) = watch::channel(ManagedSessionView::default());
4346 let (updates_tx, updates_rx) = coalesced_update_channel();
4347 tokio::spawn(run_session_actor(
4348 leased_relay_target(relay_root.path()),
4349 commands_rx,
4350 releases_rx,
4351 retirement_rx,
4352 view_tx,
4353 updates_tx,
4354 ));
4355
4356 let (reply, response) = oneshot::channel();
4357 commands_tx
4358 .send(ActorCommand::Lease { reply })
4359 .await
4360 .unwrap();
4361 let (lease_id, connection) = response
4362 .await
4363 .expect("actor answered the lease request")
4364 .expect("actor leased its relay connection");
4365 (
4366 LeasedActor {
4367 commands: commands_tx,
4368 releases: releases_tx,
4369 retirement: retirement_tx,
4370 _views: view_rx,
4371 _updates: updates_rx,
4372 _relay_root: relay_root,
4373 },
4374 lease_id,
4375 connection,
4376 )
4377 }
4378
4379 #[cfg(unix)]
4380 async fn submit_a_deferred_prompt(
4381 actor: &LeasedActor,
4382 ) -> oneshot::Receiver<std::result::Result<u64, String>> {
4383 let (reply, mut response) = oneshot::channel();
4384 actor
4385 .commands
4386 .send(ActorCommand::Submit {
4387 command_id: new_command_id("prompt").unwrap(),
4388 command: RelayCommand::Prompt {
4389 prompt: vec![ContentBlock::Text(TextContent::new("hello"))],
4390 },
4391 reply,
4392 })
4393 .await
4394 .unwrap();
4395 assert!(
4396 tokio::time::timeout(Duration::from_millis(300), &mut response)
4397 .await
4398 .is_err(),
4399 "a leased actor must hold the prompt instead of answering it"
4400 );
4401 response
4402 }
4403
4404 #[cfg(unix)]
4405 #[tokio::test]
4406 async fn prompt_submitted_during_lease_is_delivered_after_release() {
4407 if std::env::var_os(DEFERRED_SUBMIT_TEST_CHILD).is_none() {
4408 run_in_isolated_child(
4409 DEFERRED_SUBMIT_TEST_CHILD,
4410 "prompt_submitted_during_lease_is_delivered_after_release",
4411 );
4412 return;
4413 }
4414 let _writer = hel::hel_database::install_isolated_test_writer();
4416 fail_if_the_actor_stalls("prompt deferred during a lease was never delivered");
4417
4418 let (actor, lease_id, connection) = lease_a_live_actor().await;
4419 let response = submit_a_deferred_prompt(&actor).await;
4420
4421 actor
4422 .releases
4423 .send(ReturnedConnection {
4424 lease_id,
4425 connection: Some(connection),
4426 })
4427 .unwrap();
4428
4429 let ordinal = response
4430 .await
4431 .expect("actor answered the deferred prompt")
4432 .expect("deferred prompt reached the relay");
4433 assert!(
4434 ordinal > 0,
4435 "relay accepted the prompt at ordinal {ordinal}"
4436 );
4437 }
4438
4439 #[cfg(unix)]
4440 #[tokio::test]
4441 async fn returned_lease_publishes_what_it_learned_while_it_held_the_connection() {
4442 if std::env::var_os(RETURNED_LEASE_VIEW_TEST_CHILD).is_none() {
4443 run_in_isolated_child(
4444 RETURNED_LEASE_VIEW_TEST_CHILD,
4445 "returned_lease_publishes_what_it_learned_while_it_held_the_connection",
4446 );
4447 return;
4448 }
4449 let _writer = hel::hel_database::install_isolated_test_writer();
4451 fail_if_the_actor_stalls("a returned lease never republished its session");
4452
4453 let (actor, lease_id, mut connection) = lease_a_live_actor().await;
4454 let mut views = actor._views.clone();
4455 let ordinal = connection
4458 .submit(
4459 new_command_id("prompt").unwrap(),
4460 RelayCommand::Prompt {
4461 prompt: vec![ContentBlock::Text(TextContent::new("hello"))],
4462 },
4463 )
4464 .await
4465 .unwrap();
4466 assert!(views.borrow_and_update().snapshot.is_none());
4467
4468 actor
4469 .releases
4470 .send(ReturnedConnection {
4471 lease_id,
4472 connection: Some(connection),
4473 })
4474 .unwrap();
4475
4476 views.changed().await.unwrap();
4477 let snapshot = views
4478 .borrow_and_update()
4479 .snapshot
4480 .clone()
4481 .expect("the returned connection republished its session");
4482 assert!(
4483 snapshot.materialized.applied_event_ordinal >= ordinal,
4484 "published frontier {} is behind the leased submission at {ordinal}",
4485 snapshot.materialized.applied_event_ordinal
4486 );
4487 }
4488
4489 #[cfg(unix)]
4490 #[tokio::test]
4491 async fn retirement_rejects_prompts_deferred_during_lease() {
4492 if std::env::var_os(RETIRED_SUBMIT_TEST_CHILD).is_none() {
4493 run_in_isolated_child(
4494 RETIRED_SUBMIT_TEST_CHILD,
4495 "retirement_rejects_prompts_deferred_during_lease",
4496 );
4497 return;
4498 }
4499 let _writer = hel::hel_database::install_isolated_test_writer();
4501 fail_if_the_actor_stalls("prompt deferred during a lease was never answered");
4502
4503 let (actor, lease_id, connection) = lease_a_live_actor().await;
4504 let response = submit_a_deferred_prompt(&actor).await;
4505
4506 actor.retirement.send(true).unwrap();
4507 actor
4508 .releases
4509 .send(ReturnedConnection {
4510 lease_id,
4511 connection: Some(connection),
4512 })
4513 .unwrap();
4514
4515 let error = response
4516 .await
4517 .expect("actor answered the deferred prompt")
4518 .expect_err("a retiring actor must not deliver the prompt");
4519 assert!(
4520 error.contains("session target is changing"),
4521 "unexpected rejection: {error}"
4522 );
4523 }
4524
4525 #[test]
4526 fn projection_integrity_failure_is_detected_only_for_integrity_errors() {
4527 let integrity = anyhow::Error::from(ProjectionIntegrityError(
4528 "transcript item \"tool:call-1\" changed immutable identity fields".into(),
4529 ))
4530 .context("apply projection event");
4531 assert!(projection_integrity_failure(&integrity));
4532
4533 let concurrent = anyhow::Error::from(ProjectionAdvancedError { event_ordinal: 7 });
4534 assert!(!projection_integrity_failure(&concurrent));
4535
4536 let unreachable = anyhow::anyhow!("connection refused").context("connect relay proxy");
4537 assert!(!projection_integrity_failure(&unreachable));
4538 }
4539
4540 #[test]
4541 fn dashboard_updates_keep_only_the_latest_view_per_session() {
4542 let (sender, mut receiver) = coalesced_update_channel();
4543 for revision in 0..1_000 {
4544 sender.send(SessionManagerUpdate {
4545 session_id: "session-1".into(),
4546 view: ManagedSessionView {
4547 error: Some(ViewError::Unreachable(format!("revision-{revision}"))),
4548 ..ManagedSessionView::default()
4549 },
4550 });
4551 }
4552 sender.send(SessionManagerUpdate {
4553 session_id: "session-2".into(),
4554 view: ManagedSessionView {
4555 error: Some(ViewError::Unreachable("other".into())),
4556 ..ManagedSessionView::default()
4557 },
4558 });
4559
4560 assert_eq!(
4561 sender
4562 .pending
4563 .lock()
4564 .expect("session update coalescer poisoned")
4565 .len(),
4566 2
4567 );
4568 let updates = [receiver.try_recv().unwrap(), receiver.try_recv().unwrap()]
4569 .into_iter()
4570 .map(|update| (update.session_id, update.view.error.unwrap()))
4571 .collect::<BTreeMap<_, _>>();
4572 assert_eq!(updates["session-1"].detail(), "revision-999");
4573 assert_eq!(updates["session-2"].detail(), "other");
4574 assert!(receiver.try_recv().is_err());
4575 }
4576
4577 #[tokio::test]
4578 async fn remote_session_manager_fans_out_views_and_forwards_commands() {
4579 let mut remote = spawn_remote_session_manager().unwrap();
4580 remote.targets.send_replace(vec![target("unused")]);
4581 remote
4582 .publisher
4583 .publish("session-1".into(), view_at_ordinal(7))
4584 .await
4585 .unwrap();
4586
4587 let session = remote
4588 .control
4589 .wait_for_session("session-1", Duration::from_secs(1))
4590 .await
4591 .unwrap();
4592 assert_eq!(
4593 session
4594 .view()
4595 .snapshot
4596 .as_ref()
4597 .unwrap()
4598 .materialized
4599 .applied_event_ordinal,
4600 7
4601 );
4602
4603 let submitted = session
4604 .enqueue_submit("prompt-1".into(), RelayCommand::Cancel)
4605 .await
4606 .unwrap();
4607 let request = remote.requests.recv().await.unwrap();
4608 match request {
4609 RemoteSessionRequest::Submit {
4610 session_id,
4611 command_id,
4612 command: RelayCommand::Cancel,
4613 reply,
4614 } => {
4615 assert_eq!(session_id, "session-1");
4616 assert_eq!(command_id, "prompt-1");
4617 reply.send(Ok(8)).unwrap();
4618 }
4619 _ => panic!("unexpected remote session request"),
4620 }
4621 assert_eq!(submitted.wait().await.unwrap(), 8);
4622 remote.shutdown.shutdown().await.unwrap();
4623 }
4624}