1use std::net::{Ipv4Addr, SocketAddr};
5use std::path::PathBuf;
6use std::sync::Arc;
7use std::sync::atomic::{AtomicBool, Ordering};
8use std::time::{Duration, Instant};
9
10use anyhow::{Context, Result, bail};
11mod api;
12mod api_activity;
13
14use mj_core::config::{Config, HarnessProfile, PhoneConfig, is_bare_project_target};
15use mj_core::remote_git::{default_branch, display_url, resolve_repository};
16use mj_core::state::{MaterializedSession, ProjectSourceIdentity, SessionRecord, State};
17
18use crate::controller::Controller;
19use crate::quota::ProfileQuota;
20use crate::server::{
21 ActionOutcome, BackgroundTaskStopFailure, BackgroundTaskStopRequest, BrowserTranscript,
22 ControllerAction, ControllerRequest, MovePreparationRequest, PreflightFailure,
23 ReadReceiptRequest, ResumeQueueDisposition, ServerOptions, ViewerActivityDetails,
24 ViewerActivityKind, ViewerBackgroundTask, ViewerMoveRecovery, ViewerQueuedPrompt, ViewerQuota,
25 ViewerSnapshot, ViewerUserShell,
26};
27use crate::session_manager::{SessionManagerChannels, SessionManagerControl, new_command_id};
28use crate::tailscale::TailscaleTls;
29#[cfg(test)]
30use crate::targets::ProcessExecutor;
31use crate::targets::{CancellableProcessExecutor, CommandExecutor};
32use crate::worker_client::CredentialSyncCoordinator;
33use mj_core::relay::RelayCommand;
34use mj_core::workspace::WorkspaceRecord;
35
36use crate::controller::config_only_controller;
37use crate::daemon::{
38 CreateSessionControl, CreateSessionRequest, ResumeSessionRequest, RuntimeState,
39};
40use crate::pollers::{
41 CredentialSyncNotices, CredentialSyncSignalTracker, QUOTA_STALE_AFTER, QuotaRefreshBatch,
42 QuotaUpdate, apply_worker_record_update, credential_sync_targets, dashboard_worker_targets,
43 projected_queued_prompts, queued_prompt_projection, quota_refresh_profiles,
44 schedule_due_credential_syncs, spawn_quota_refresher,
45};
46
47#[derive(Debug, Clone)]
48pub struct ServerArgs {
49 bind: String,
50 tailscale_detect: bool,
51 tls_cert: Option<PathBuf>,
52 tls_key: Option<PathBuf>,
53}
54
55impl From<&PhoneConfig> for ServerArgs {
56 fn from(config: &PhoneConfig) -> Self {
57 Self {
58 bind: config.bind.clone(),
59 tailscale_detect: config.tailscale_detect,
60 tls_cert: config.tls_cert.clone(),
61 tls_key: config.tls_key.clone(),
62 }
63 }
64}
65
66const TAILSCALE_COMMAND_TIMEOUT: Duration = Duration::from_secs(120);
67const TAILSCALE_RENEW_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60);
68
69const MAX_CONCURRENT_CONVERSATION_PROJECTIONS: usize = 2;
74const CONVERSATION_PROJECTION_CHANNEL_CAPACITY: usize = 16;
75
76#[derive(Debug, Clone, PartialEq, Eq)]
77struct ConversationProjectionKey {
78 ordinal: u64,
79 digest: String,
80}
81
82impl ConversationProjectionKey {
83 fn of(materialized: &MaterializedSession) -> Self {
84 Self {
85 ordinal: materialized.applied_event_ordinal,
86 digest: materialized.applied_event_digest.clone(),
87 }
88 }
89
90 fn is_newer_than(&self, other: &Self) -> bool {
94 self.ordinal > other.ordinal
95 || (self.ordinal == other.ordinal && self.digest != other.digest)
96 }
97}
98
99struct ConversationProjectionRequest {
100 materialized: MaterializedSession,
101 key: ConversationProjectionKey,
102 generation: u64,
103}
104
105struct ConversationProjectionResult {
106 session_id: String,
107 key: ConversationProjectionKey,
108 generation: u64,
109 result: std::result::Result<BrowserTranscript, String>,
110}
111
112struct ConversationProjectionDispatcher {
116 in_flight: std::collections::BTreeMap<String, (ConversationProjectionKey, u64)>,
117 pending: std::collections::BTreeMap<String, ConversationProjectionRequest>,
118 completed: std::collections::BTreeMap<String, ConversationProjectionKey>,
119 generations: std::collections::BTreeMap<String, u64>,
120 permits: Arc<tokio::sync::Semaphore>,
121 results: tokio::sync::mpsc::Sender<ConversationProjectionResult>,
122 shutdown: tokio_util::sync::CancellationToken,
123}
124
125impl ConversationProjectionDispatcher {
126 fn new(
127 results: tokio::sync::mpsc::Sender<ConversationProjectionResult>,
128 shutdown: tokio_util::sync::CancellationToken,
129 ) -> Self {
130 Self {
131 in_flight: std::collections::BTreeMap::new(),
132 pending: std::collections::BTreeMap::new(),
133 completed: std::collections::BTreeMap::new(),
134 generations: std::collections::BTreeMap::new(),
135 permits: Arc::new(tokio::sync::Semaphore::new(
136 MAX_CONCURRENT_CONVERSATION_PROJECTIONS,
137 )),
138 results,
139 shutdown,
140 }
141 }
142
143 #[cfg(test)]
144 fn with_permits(
145 results: tokio::sync::mpsc::Sender<ConversationProjectionResult>,
146 shutdown: tokio_util::sync::CancellationToken,
147 permits: usize,
148 ) -> Self {
149 let mut dispatcher = Self::new(results, shutdown);
150 dispatcher.permits = Arc::new(tokio::sync::Semaphore::new(permits));
151 dispatcher
152 }
153
154 fn enqueue(&mut self, materialized: MaterializedSession) {
157 let session_id = materialized.session_id.clone();
158 let key = ConversationProjectionKey::of(&materialized);
159 if self
160 .completed
161 .get(&session_id)
162 .is_some_and(|completed| !key.is_newer_than(completed))
163 {
164 return;
165 }
166 let generation = *self.generations.entry(session_id.clone()).or_default();
167 if let Some((in_flight, in_flight_generation)) = self.in_flight.get(&session_id) {
168 if generation != *in_flight_generation || key.is_newer_than(in_flight) {
169 let replace = self.pending.get(&session_id).is_none_or(|pending| {
170 pending.generation != generation || key.is_newer_than(&pending.key)
171 });
172 if replace {
173 self.pending.insert(
174 session_id,
175 ConversationProjectionRequest {
176 materialized,
177 key,
178 generation,
179 },
180 );
181 }
182 }
183 return;
184 }
185 self.in_flight
186 .insert(session_id.clone(), (key.clone(), generation));
187 self.start(ConversationProjectionRequest {
188 materialized,
189 key,
190 generation,
191 });
192 }
193
194 fn finish(
198 &mut self,
199 result: ConversationProjectionResult,
200 session_active: bool,
201 ) -> Option<(String, ConversationProjectionKey, BrowserTranscript)> {
202 let expected = self.in_flight.remove(&result.session_id);
203 if expected.as_ref() != Some(&(result.key.clone(), result.generation)) {
204 tracing::warn!(
205 session_id = %result.session_id,
206 "discarding an out-of-date browser transcript projection"
207 );
208 return None;
209 }
210 let session_id = result.session_id;
211 let key = result.key;
212 let current_generation = self
213 .generations
214 .get(&session_id)
215 .copied()
216 .unwrap_or_default();
217 let current = result.generation == current_generation;
218 let projected = match result.result {
219 Ok(transcript) if session_active && current => {
220 self.completed.insert(session_id.clone(), key.clone());
221 Some((session_id.clone(), key, transcript))
222 }
223 Ok(_) => {
224 self.completed.remove(&session_id);
228 None
229 }
230 Err(error) => {
231 tracing::warn!(
232 session_id = %session_id,
233 "browser transcript projection failed: {error}"
234 );
235 None
236 }
237 };
238 if session_active {
239 if let Some(pending) = self.pending.remove(&session_id) {
240 self.enqueue(pending.materialized);
241 }
242 } else {
243 self.pending.remove(&session_id);
244 self.completed.remove(&session_id);
245 }
246 projected
247 }
248
249 fn forget(&mut self, session_id: &str) {
253 self.pending.remove(session_id);
254 self.completed.remove(session_id);
255 let generation = self.generations.entry(session_id.to_owned()).or_default();
256 *generation = generation.wrapping_add(1);
257 }
258
259 fn session_ids(&self) -> std::collections::BTreeSet<String> {
260 self.in_flight
261 .keys()
262 .chain(self.pending.keys())
263 .chain(self.completed.keys())
264 .cloned()
265 .collect()
266 }
267
268 fn start(&self, request: ConversationProjectionRequest) {
269 let session_id = request.materialized.session_id.clone();
270 let key = request.key;
271 let generation = request.generation;
272 let permits = Arc::clone(&self.permits);
273 let results = self.results.clone();
274 let shutdown = self.shutdown.clone();
275 tokio::spawn(async move {
276 let result = match tokio::select! {
277 _ = shutdown.cancelled() => return,
278 result = permits.acquire_owned() => result,
279 } {
280 Ok(permit) => {
281 let projection = tokio::task::spawn_blocking(move || {
282 mj_client::transcript::materialized_browser_transcript(
283 &request.materialized,
284 )
285 })
286 .await;
287 drop(permit);
288 projection
289 .map_err(|error| format!("transcript projection task failed: {error}"))
290 }
291 Err(error) => Err(format!("transcript projection worker stopped: {error}")),
292 };
293 let message = ConversationProjectionResult {
294 session_id,
295 key,
296 generation,
297 result,
298 };
299 tokio::select! {
300 _ = shutdown.cancelled() => {}
301 result = results.send(message) => {
302 if let Err(error) = result {
303 tracing::debug!(%error, "browser transcript projection result dropped after server shutdown");
304 }
305 }
306 }
307 });
308 }
309}
310
311struct ResolvedServerArgs {
312 bind: SocketAddr,
313 viewer_url: String,
314 tls_files: Option<(PathBuf, PathBuf)>,
315 tailscale: Option<TailscaleTls>,
316 fallback_reason: Option<String>,
317}
318
319async fn resolve_server_args(
320 args: ServerArgs,
321 termination: tokio_util::sync::CancellationToken,
322) -> Result<ResolvedServerArgs> {
323 let configured_bind: SocketAddr = args.bind.parse().context("parse web viewer bind address")?;
324 match (args.tls_cert, args.tls_key) {
325 (Some(cert), Some(key)) => {
326 let scheme = "https";
327 return Ok(ResolvedServerArgs {
328 bind: configured_bind,
329 viewer_url: format!("{scheme}://{configured_bind}"),
330 tls_files: Some((cert, key)),
331 tailscale: None,
332 fallback_reason: None,
333 });
334 }
335 (None, None) => {}
336 _ => bail!("web viewer TLS requires both a certificate and private key"),
337 }
338
339 if !args.tailscale_detect {
340 return Ok(loopback_server_args(
341 configured_bind,
342 Some("automatic Tailscale detection is disabled".into()),
343 ));
344 }
345
346 let tls_root = mj_core::config::data_dir().join("viewer");
347 let prepared = run_tailscale_blocking(termination.clone(), move |executor| {
348 crate::tailscale::prepare_tailscale_tls(&tls_root, executor)
349 })
350 .await;
351 match prepared {
352 Ok(tailscale) => {
353 let bind = tailscale_bind(configured_bind);
354 let viewer_url = format!(
355 "https://{}:{}",
356 tailscale.cert_domain(),
357 configured_bind.port()
358 );
359 Ok(ResolvedServerArgs {
360 bind,
361 viewer_url,
362 tls_files: Some((
363 tailscale.cert_path().to_owned(),
364 tailscale.key_path().to_owned(),
365 )),
366 tailscale: Some(tailscale),
367 fallback_reason: None,
368 })
369 }
370 Err(error) if termination.is_cancelled() => Err(error),
371 Err(error) => {
372 let reason = format!("{error:#}");
373 tracing::debug!(error = reason, "Tailscale HTTPS unavailable for web viewer");
374 Ok(loopback_server_args(configured_bind, Some(reason)))
375 }
376 }
377}
378
379fn tailscale_bind(configured_bind: SocketAddr) -> SocketAddr {
380 SocketAddr::from((Ipv4Addr::UNSPECIFIED, configured_bind.port()))
381}
382
383fn loopback_server_args(bind: SocketAddr, fallback_reason: Option<String>) -> ResolvedServerArgs {
384 ResolvedServerArgs {
385 bind,
386 viewer_url: format!("http://{bind}"),
387 tls_files: None,
388 tailscale: None,
389 fallback_reason,
390 }
391}
392
393async fn run_tailscale_blocking<T>(
394 termination: tokio_util::sync::CancellationToken,
395 operation: impl FnOnce(&CancellableProcessExecutor) -> Result<T> + Send + 'static,
396) -> Result<T>
397where
398 T: Send + 'static,
399{
400 let cancelled = Arc::new(AtomicBool::new(false));
401 let executor_cancelled = cancelled.clone();
402 let mut task = tokio::task::spawn_blocking(move || {
403 let executor = CancellableProcessExecutor::new(executor_cancelled)
404 .with_deadline(TAILSCALE_COMMAND_TIMEOUT);
405 operation(&executor)
406 });
407 tokio::select! {
408 result = &mut task => result.context("Tailscale background task panicked")?,
409 _ = termination.cancelled() => {
410 cancelled.store(true, Ordering::Release);
411 let _ = task.await;
412 bail!("Tailscale operation cancelled during web viewer shutdown")
413 }
414 }
415}
416
417fn spawn_tailscale_cert_renewer(
418 tailscale: TailscaleTls,
419 rustls: axum_server::tls_rustls::RustlsConfig,
420 termination: tokio_util::sync::CancellationToken,
421) -> tokio::task::JoinHandle<()> {
422 tokio::spawn(async move {
423 let mut interval = tokio::time::interval(TAILSCALE_RENEW_INTERVAL);
424 interval.tick().await;
425 loop {
426 tokio::select! {
427 _ = termination.cancelled() => return,
428 _ = interval.tick() => {}
429 }
430 let renewing = tailscale.clone();
431 let result = run_tailscale_blocking(termination.clone(), move |executor| {
432 renewing.renew(executor)
433 })
434 .await;
435 if let Err(error) = result {
436 if !termination.is_cancelled() {
437 tracing::warn!(
438 error = format!("{error:#}"),
439 "Tailscale certificate renewal failed"
440 );
441 }
442 continue;
443 }
444 if let Err(error) = rustls
445 .reload_from_pem_file(tailscale.cert_path(), tailscale.key_path())
446 .await
447 {
448 tracing::warn!(%error, "could not activate renewed Tailscale certificate");
449 }
450 }
451 })
452}
453
454const MAX_CONCURRENT_PHONE_ACTIONS: usize = 4;
455const MAX_CONCURRENT_BUNDLE_CREATIONS: usize = 4;
456const MAX_CONCURRENT_PREFLIGHTS: usize = 4;
457
458struct PhoneActionStarted {
459 action_id: u64,
460 session: SessionRecord,
461 published: tokio::sync::oneshot::Sender<std::result::Result<(), String>>,
462}
463
464#[derive(Default)]
473struct PendingActionReplies(
474 std::collections::BTreeMap<u64, tokio::sync::oneshot::Sender<ActionOutcome>>,
475);
476
477impl PendingActionReplies {
478 fn accept(
479 &mut self,
480 action_id: u64,
481 action: &ControllerAction,
482 reply: tokio::sync::oneshot::Sender<ActionOutcome>,
483 ) {
484 if matches!(action, ControllerAction::New { .. }) {
485 self.0.insert(action_id, reply);
486 } else {
487 if reply.send(ActionOutcome::accepted()).is_err() {
488 tracing::debug!(
489 action_id,
490 "phone action acceptance reply dropped after client disconnect"
491 );
492 }
493 }
494 }
495
496 fn resolve(&mut self, action_id: u64, outcome: ActionOutcome) {
497 if let Some(reply) = self.0.remove(&action_id)
498 && reply.send(outcome).is_err()
499 {
500 tracing::debug!(
501 action_id,
502 "phone action completion reply dropped after client disconnect"
503 );
504 }
505 }
506}
507
508fn admit_phone_action(
512 action: &ControllerAction,
513 running_actions: usize,
514 active_sessions: &mut std::collections::BTreeSet<String>,
515) -> std::result::Result<Option<String>, ActionOutcome> {
516 let closing = matches!(action, ControllerAction::Close { .. });
517 if !closing && !phone_action_capacity_available(running_actions) {
518 return Err(ActionOutcome::Busy);
519 }
520 let session_id = controller_action_session_id(action);
521 if let Some(session_id) = &session_id
522 && !active_sessions.insert(session_id.clone())
523 && !closing
524 {
525 return Err(ActionOutcome::SessionBusy);
526 }
527 Ok(session_id)
528}
529
530struct ReadReceiptPersisted {
531 session_id: String,
532 result: std::result::Result<u64, String>,
533 reply: tokio::sync::oneshot::Sender<std::result::Result<(), String>>,
534}
535
536struct ControllerReloaded {
537 result: std::result::Result<Controller, String>,
538}
539
540struct BundleCreated {
541 result: std::result::Result<
542 crate::controller::QuickBundleCreation,
543 crate::controller::QuickBundleFailure,
544 >,
545 reply: tokio::sync::oneshot::Sender<std::result::Result<String, crate::server::BundleFailure>>,
546}
547
548struct MovePrepared {
549 result: std::result::Result<mj_core::state::MovePreparation, String>,
550 reply:
551 tokio::sync::oneshot::Sender<std::result::Result<mj_core::state::MovePreparation, String>>,
552}
553
554fn spawn_controller_reload(completed: tokio::sync::mpsc::UnboundedSender<ControllerReloaded>) {
558 spawn_controller_reload_with(completed, Controller::load);
559}
560
561fn spawn_controller_reload_with(
562 completed: tokio::sync::mpsc::UnboundedSender<ControllerReloaded>,
563 load: impl FnOnce() -> Result<Controller> + Send + 'static,
564) {
565 tokio::spawn(async move {
566 let result = match tokio::task::spawn_blocking(load).await {
567 Ok(result) => result.map_err(|error| format!("{error:#}")),
568 Err(error) => Err(format!("controller reload task failed: {error}")),
569 };
570 if completed.send(ControllerReloaded { result }).is_err() {
571 tracing::debug!("controller reload completed after the phone control loop stopped");
572 }
573 });
574}
575
576fn request_controller_reload(
577 in_flight: &mut bool,
578 requested: &mut bool,
579 completed: &tokio::sync::mpsc::UnboundedSender<ControllerReloaded>,
580) {
581 if *in_flight {
582 *requested = true;
583 } else {
584 *in_flight = true;
585 spawn_controller_reload(completed.clone());
586 }
587}
588
589fn request_daemon_controller_reload(daemon_runtime: Arc<RuntimeState>, reason: &'static str) {
590 tokio::spawn(async move {
591 if let Err(error) = daemon_runtime.reload_controller().await {
592 tracing::warn!(
593 error = format!("{error:#}"),
594 reason,
595 "phone operation could not refresh dashboard controller state"
596 );
597 }
598 });
599}
600
601#[derive(Debug, PartialEq, Eq)]
603#[cfg(test)]
604enum ReadReceiptPlan {
605 UnknownSession,
606 AlreadyRead,
608 Persist,
610}
611
612#[cfg(test)]
613fn plan_read_receipt(state: &State, session_id: &str, through: u64) -> ReadReceiptPlan {
614 let Some(session) = state.sessions.get(session_id) else {
615 return ReadReceiptPlan::UnknownSession;
616 };
617 if through > session.viewed_through_event_ordinal {
618 ReadReceiptPlan::Persist
619 } else {
620 ReadReceiptPlan::AlreadyRead
621 }
622}
623
624#[cfg(test)]
628fn apply_read_receipt(state: &mut State, session_id: &str, receipt: u64) -> bool {
629 let Some(session) = state.sessions.get_mut(session_id) else {
630 return false;
631 };
632 if receipt <= session.viewed_through_event_ordinal {
633 return false;
634 }
635 session.viewed_through_event_ordinal = receipt;
636 true
637}
638
639#[derive(Clone)]
640struct PhoneActionControl {
641 cancelled: Arc<AtomicBool>,
642 create: Option<CreateSessionControl>,
643}
644
645impl PhoneActionControl {
646 fn for_action(action: &ControllerAction) -> Self {
647 let create =
648 matches!(action, ControllerAction::New { .. }).then(CreateSessionControl::default);
649 let cancelled = create.as_ref().map_or_else(
650 || Arc::new(AtomicBool::new(false)),
651 |control| control.cancelled.clone(),
652 );
653 Self { cancelled, create }
654 }
655
656 fn request_cancel(&self) -> bool {
657 let accepted = self.create.as_ref().map_or_else(
658 || {
659 self.cancelled
660 .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
661 .is_ok()
662 },
663 |control| control.request_cancel(),
664 );
665 if accepted {
666 self.cancelled.store(true, Ordering::Release);
667 }
668 accepted
669 }
670
671 #[cfg(test)]
672 fn grant_new_commit(&self) -> bool {
673 self.create
674 .as_ref()
675 .is_some_and(|control| control.grant_commit())
676 }
677}
678
679pub async fn run_server(
680 args: ServerArgs,
681 termination: tokio_util::sync::CancellationToken,
682 worker: SessionManagerChannels,
683 daemon_runtime: Arc<RuntimeState>,
684 mut workspace_updates: tokio::sync::watch::Receiver<Vec<WorkspaceRecord>>,
685) -> Result<()> {
686 let resolved = resolve_server_args(args, termination.clone()).await?;
687 let bind = resolved.bind;
688 let mut controller = Controller::load()?;
689 let mut daemon_revisions = daemon_runtime.revisions();
690 daemon_revisions.borrow_and_update();
691 let mut phone_workspaces = workspace_updates.borrow_and_update().clone();
692 let mut quotas = std::collections::BTreeMap::new();
693 let subagent_quota_reports = Arc::new(std::sync::Mutex::new(quotas.clone()));
694 let (quota_profiles_tx, mut quota_updates_rx) = spawn_quota_refresher();
695 let mut quota_batch = QuotaRefreshBatch::default();
696 let mut published_quota_profiles = std::collections::BTreeMap::new();
697 republish_quota_profiles(
698 &controller,
699 &mut published_quota_profiles,
700 &mut quota_batch,
701 "a_profiles_tx,
702 );
703 let mut revision = daemon_runtime.allocate_revision();
704 let mut conversations = std::collections::BTreeMap::new();
705 let mut queued_prompts = projected_queued_prompts(&controller)?;
706 let mut active_user_shells = std::collections::BTreeMap::new();
707 let mut pending_elicitations = std::collections::BTreeMap::new();
708 let mut prompt_images = std::collections::BTreeSet::new();
709 let mut operational = std::collections::BTreeMap::new();
710 let mut materialized_activity = load_materialized_activity(&controller).await?;
711 let mut project_sources = PhoneProjectSources::default();
712 let (records, lifecycles) = daemon_runtime.session_projection();
713 controller.state.sessions = records;
714 let mut operations = lifecycles
715 .iter()
716 .map(|view| (view.session_id.clone(), viewer_operation(view)))
717 .collect::<std::collections::BTreeMap<_, _>>();
718 let mut move_recoveries = ViewerMoveRecoveries::new();
719 let (move_recovery_tx, mut move_recovery_rx) =
720 tokio::sync::mpsc::unbounded_channel::<Result<ViewerMoveRecoveries, String>>();
721 let mut move_recovery_load_in_flight = false;
722 let mut launch_failures = Vec::new();
723 let mut capacity_state: std::collections::BTreeMap<String, PhoneCapacity> =
727 std::collections::BTreeMap::new();
728 let (capacity_targets_tx, capacity_triggers_tx, mut capacity_updates_rx) =
729 crate::pollers::spawn_dashboard_capacity_poller();
730 let (snapshot_tx, snapshot_rx) = tokio::sync::watch::channel(viewer_snapshot(
731 &controller,
732 &phone_workspaces,
733 "as,
734 &PhoneSessionViews {
735 conversations: &conversations,
736 queued_prompts: &queued_prompts,
737 active_user_shells: &active_user_shells,
738 pending_elicitations: &pending_elicitations,
739 prompt_images: &prompt_images,
740 operational: &operational,
741 materialized_activity: &materialized_activity,
742 project_sources: &project_sources,
743 operations: &operations,
744 move_recoveries: &move_recoveries,
745 capacity: &viewer_capacity(&capacity_state),
746 launch_failures: &launch_failures,
747 reviews: &review_views(&daemon_runtime),
748 },
749 revision,
750 ));
751 let (conversation_tx, conversation_rx) = tokio::sync::watch::channel(conversations.clone());
752 let (action_tx, mut action_rx) = tokio::sync::mpsc::channel(32);
753 let (bundle_tx, mut bundle_rx) = tokio::sync::mpsc::channel(16);
754 let (receipt_tx, mut receipt_rx) = tokio::sync::mpsc::channel(32);
755 let (preflight_tx, mut preflight_rx) = tokio::sync::mpsc::channel(32);
756 let (move_preparation_tx, mut move_preparation_rx) = tokio::sync::mpsc::channel(32);
757 let (client_state_tx, mut client_state_rx) = tokio::sync::mpsc::channel(64);
758 let (dictation_tx, mut dictation_rx) =
759 tokio::sync::mpsc::channel::<crate::dictation::DictationRequest>(8);
760 let (background_task_stop_tx, mut background_task_stop_rx) =
761 tokio::sync::mpsc::channel::<BackgroundTaskStopRequest>(32);
762 let SessionManagerChannels {
763 targets: worker_targets_tx,
764 control: worker_commands_tx,
765 updates: mut worker_updates_rx,
766 shutdown: worker_shutdown,
767 } = worker;
768 worker_targets_tx.send_replace(dashboard_worker_targets(&controller));
769 publish_capacity_targets(&controller, &capacity_targets_tx, &mut capacity_state);
770 let mut credential_sync = CredentialSyncCoordinator::spawn();
771 let credential_sync_handle = credential_sync.handle();
772 credential_sync_handle.set_targets(credential_sync_targets(&controller));
773 let mut credential_sync_signals = CredentialSyncSignalTracker::default();
774 let mut credential_sync_notices = CredentialSyncNotices::default();
775 let options_session_ttl = crate::server::default_session_ttl();
777 let activity_snapshots = snapshot_rx.clone();
778 let mut options = ServerOptions::new(
779 bind,
780 snapshot_rx,
781 conversation_rx,
782 crate::server::ServerRequests {
783 action_tx,
784 bundle_tx,
785 receipt_tx,
786 preflight_tx,
787 move_preparation_tx,
788 client_state_tx,
789 dictation_tx,
790 },
791 )?;
792 options.set_background_task_stop_tx(background_task_stop_tx);
793 options.shutdown = termination.clone();
794 let cookie_key_path = crate::server::cookie_key_path();
798 options.set_cookie_key(crate::server::load_or_create_cookie_key(&cookie_key_path)?)?;
799 options.set_api_token(crate::server::load_or_create_api_token(
802 &crate::server::api_token_path(),
803 )?);
804 let api_runtime = daemon_runtime.clone();
805 let api_backend = Arc::new(
806 api::ApiBackend::new(
807 worker_commands_tx.client(),
808 Arc::new(move |session_id: &str| api_runtime.session_state(session_id)),
809 daemon_runtime.clone(),
810 )
811 .with_quota_reports(subagent_quota_reports.clone()),
812 );
813 options.set_subagent_backend(api_backend.clone());
814 let renewal_cancellation = termination.child_token();
815 let mut renewal_task = None;
816 if let Some((cert, key)) = resolved.tls_files {
817 let rustls = axum_server::tls_rustls::RustlsConfig::from_pem_file(cert, key)
818 .await
819 .context("load web viewer TLS certificate")?;
820 options.set_tls_config(rustls.clone());
821 if let Some(tailscale) = resolved.tailscale {
822 renewal_task = Some(spawn_tailscale_cert_renewer(
823 tailscale,
824 rustls,
825 renewal_cancellation.clone(),
826 ));
827 }
828 } else if bind.ip().is_loopback() {
829 options.secure_cookie = false;
830 } else {
831 anyhow::bail!("non-loopback web viewer requires TLS");
832 }
833 let fallback_reason = resolved.fallback_reason;
834 let qr_login_url = if fallback_reason.is_none() && resolved.viewer_url.starts_with("https://") {
835 let encoded = url::form_urlencoded::byte_serialize(options.login_token().as_bytes())
836 .collect::<String>();
837 Some(format!(
838 "{}/auth/login?token={encoded}",
839 resolved.viewer_url.trim_end_matches('/')
840 ))
841 } else {
842 None
843 };
844 let ready = crate::server::WebViewerAccess::Ready {
845 viewer_url: resolved.viewer_url,
846 viewer_code: options.viewer_code().to_owned(),
847 qr_login_url,
848 fallback_reason,
849 };
850
851 let serve = crate::web_viewer::serve(options, ready, &daemon_runtime.web_viewer, |access| {
852 daemon_runtime.publish_web_access(access);
853 });
854 let conversation_projection_shutdown = termination.child_token();
855 let control = async {
856 let mut credential_tick = tokio::time::interval(Duration::from_millis(250));
857 let mut prune_tick = tokio::time::interval(Duration::from_secs(60 * 60));
861 let client_state_retention = options_session_ttl;
862 let (action_done_tx, mut action_done_rx) = tokio::sync::mpsc::unbounded_channel::<(
863 u64,
864 Option<String>,
865 std::result::Result<(), String>,
866 )>();
867 let (action_started_tx, mut action_started_rx) =
868 tokio::sync::mpsc::unbounded_channel::<PhoneActionStarted>();
869 let (receipt_done_tx, mut receipt_done_rx) =
870 tokio::sync::mpsc::unbounded_channel::<ReadReceiptPersisted>();
871 let (controller_reload_tx, mut controller_reload_rx) =
872 tokio::sync::mpsc::unbounded_channel::<ControllerReloaded>();
873 let (bundle_done_tx, mut bundle_done_rx) =
874 tokio::sync::mpsc::unbounded_channel::<BundleCreated>();
875 let (move_prepared_tx, mut move_prepared_rx) =
876 tokio::sync::mpsc::unbounded_channel::<MovePrepared>();
877 let mut dictation_jobs = tokio::task::JoinSet::new();
878 let mut bundle_jobs = tokio::task::JoinSet::new();
879 let mut preflight_jobs = tokio::task::JoinSet::new();
880 let mut move_preparation_jobs = tokio::task::JoinSet::new();
881 let mut move_recovery_jobs = tokio::task::JoinSet::new();
882 let mut background_task_stop_jobs = tokio::task::JoinSet::new();
883 let mut background_task_stop_open = true;
884 let mut controller_reload_in_flight = false;
885 let mut controller_reload_requested = false;
886 let mut controller_reload_invalidated = false;
887 let mut pending_action_errors = std::collections::BTreeMap::<String, String>::new();
888 let mut active_actions = std::collections::BTreeSet::new();
889 let mut closing_actions = std::collections::BTreeMap::<String, u64>::new();
890 let mut next_action_id = 0_u64;
891 let mut action_cancellations = std::collections::BTreeMap::<u64, PhoneActionControl>::new();
892 let mut action_sessions = std::collections::BTreeMap::<u64, String>::new();
893 let mut action_replies = PendingActionReplies::default();
894 let mut launch_workspaces = std::collections::BTreeMap::new();
895 let mut subagent_jobs = tokio::task::JoinSet::new();
896 let mut subagent_completion_jobs = tokio::task::JoinSet::new();
897 let mut active_subagent_requests = std::collections::BTreeSet::new();
898 let (conversation_projection_tx, mut conversation_projection_rx) =
899 tokio::sync::mpsc::channel(CONVERSATION_PROJECTION_CHANNEL_CAPACITY);
900 let mut conversation_projections = ConversationProjectionDispatcher::new(
901 conversation_projection_tx,
902 conversation_projection_shutdown.clone(),
903 );
904 let mut quota_updates_open = true;
905 let mut failure: Option<anyhow::Error> = None;
909 request_move_recovery_reload(
910 &move_recovery_tx,
911 &mut move_recovery_load_in_flight,
912 &mut move_recovery_jobs,
913 );
914 macro_rules! publish_snapshot {
915 ($revision:expr) => {
916 let (records, lifecycles) = daemon_runtime.session_projection();
917 controller.state.sessions = records;
918 for (session_id, error) in &pending_action_errors {
919 if let Some(session) = controller.state.sessions.get_mut(session_id)
920 && session.last_error.is_none()
921 {
922 session.last_error = Some(error.clone());
923 }
924 }
925 operations = lifecycles.iter()
926 .map(|view| (view.session_id.clone(), viewer_operation(view)))
927 .collect();
928 if let Err(error) = snapshot_tx.send(viewer_snapshot(
929 &controller,
930 &phone_workspaces,
931 "as,
932 &PhoneSessionViews {
933 conversations: &conversations,
934 queued_prompts: &queued_prompts,
935 active_user_shells: &active_user_shells,
936 pending_elicitations: &pending_elicitations,
937 prompt_images: &prompt_images,
938 operational: &operational,
939 materialized_activity: &materialized_activity,
940 project_sources: &project_sources,
941 operations: &operations,
942 move_recoveries: &move_recoveries,
943 capacity: &viewer_capacity(&capacity_state),
944 launch_failures: &launch_failures,
945 reviews: &review_views(&daemon_runtime),
946 },
947 $revision,
948 )) {
949 tracing::debug!(revision = $revision, %error, "phone snapshot delivery failed; no viewer is subscribed");
950 }
951 };
952 }
953 loop {
954 project_sources.synchronize(&controller);
955 tokio::select! {
956 _ = termination.cancelled() => break,
957 request = background_task_stop_rx.recv(), if background_task_stop_open => {
958 let Some(request) = request else {
959 background_task_stop_open = false;
960 tracing::warn!("background-task stop request feed closed while the phone server was running");
961 continue;
962 };
963 let session_control = worker_commands_tx.clone();
964 background_task_stop_jobs.spawn(async move {
965 let result = match session_control.session(&request.session_id).await {
966 Ok(session) => session
967 .client()
968 .stop_background_task(request.background_task_id.clone())
969 .await
970 .map_err(|error| {
971 tracing::warn!(
972 session_id = %request.session_id,
973 background_task_id = %request.background_task_id,
974 %error,
975 "provider rejected background-task stop"
976 );
977 BackgroundTaskStopFailure::Provider
978 }),
979 Err(error) => {
980 tracing::warn!(
981 session_id = %request.session_id,
982 %error,
983 "could not resolve live session for background-task stop"
984 );
985 Err(BackgroundTaskStopFailure::SessionUnavailable)
986 }
987 };
988 if request.reply.send(result).is_err() {
989 tracing::debug!(
990 session_id = %request.session_id,
991 background_task_id = %request.background_task_id,
992 "background-task stop result dropped after viewer disconnected"
993 );
994 }
995 });
996 }
997 completed = background_task_stop_jobs.join_next(), if !background_task_stop_jobs.is_empty() => {
998 if let Some(Err(error)) = completed {
999 tracing::error!(%error, "background-task stop task failed unexpectedly");
1000 }
1001 }
1002 move_reloaded = move_recovery_rx.recv() => {
1003 let Some(result) = move_reloaded else {
1004 failure = feed_stopped(
1005 termination.is_cancelled(),
1006 "the Move recovery projection stopped while the phone server was running",
1007 );
1008 break;
1009 };
1010 move_recovery_load_in_flight = false;
1011 match result {
1012 Ok(recoveries) => {
1013 move_recoveries = recoveries;
1014 revision = daemon_runtime.allocate_revision();
1015 publish_snapshot!(revision);
1016 }
1017 Err(error) => tracing::warn!(%error, "could not refresh Move recovery projection"),
1018 }
1019 }
1020 resolved = project_sources.jobs.join_next(), if !project_sources.jobs.is_empty() => {
1021 match resolved {
1022 Some(Ok(resolved)) => project_sources.complete(resolved),
1023 Some(Err(error)) => {
1024 failure = Some(anyhow::anyhow!("web project source task failed: {error}"));
1025 break;
1026 }
1027 None => unreachable!("project source jobs were not empty"),
1028 }
1029 revision = daemon_runtime.allocate_revision();
1030 publish_snapshot!(revision);
1031 }
1032 changed = daemon_revisions.changed() => {
1033 if changed.is_err() {
1034 failure = feed_stopped(
1035 termination.is_cancelled(),
1036 "the daemon stopped publishing runtime revisions to the phone server",
1037 );
1038 break;
1039 }
1040 daemon_revisions.borrow_and_update();
1041 revision = daemon_runtime.allocate_revision();
1042 publish_snapshot!(revision);
1043 request_controller_reload(
1044 &mut controller_reload_in_flight,
1045 &mut controller_reload_requested,
1046 &controller_reload_tx,
1047 );
1048 }
1049 changed = workspace_updates.changed() => {
1050 if changed.is_err() {
1051 failure = feed_stopped(
1052 termination.is_cancelled(),
1053 "the daemon stopped publishing workspaces to the phone server",
1054 );
1055 break;
1056 }
1057 phone_workspaces = workspace_updates.borrow_and_update().clone();
1058 revision = daemon_runtime.allocate_revision();
1059 publish_snapshot!(revision);
1060 }
1061 update = capacity_updates_rx.recv() => {
1062 let Some(update) = update else {
1063 failure = feed_stopped(termination.is_cancelled(), "the capacity poller stopped while the phone server was running");
1064 break;
1065 };
1066 if let Some(entry) = capacity_state.get_mut(&update.target_id) {
1067 entry.refreshing = false;
1068 entry.sampled_at_epoch_seconds = Some(update.sampled_at_epoch_seconds);
1069 match update.result {
1070 Ok(usage) => {
1071 entry.on_demand = usage.is_none();
1075 entry.usage = usage;
1076 entry.failed = false;
1077 }
1078 Err(_) => entry.failed = true,
1082 }
1083 }
1084 revision = daemon_runtime.allocate_revision();
1085 publish_snapshot!(revision);
1086 }
1087 update = quota_updates_rx.recv(), if quota_updates_open => {
1088 match update {
1089 Some(QuotaUpdate::Report(outcome)) => {
1090 if outcome.credentials_changed {
1091 credential_sync_handle
1092 .sync_profile_now(&outcome.report.profile_id, None);
1093 }
1094 quotas.insert(outcome.report.profile_id.clone(), outcome.report.clone());
1095 subagent_quota_reports
1096 .lock()
1097 .expect("sub-agent quota reports lock poisoned")
1098 .insert(outcome.report.profile_id.clone(), outcome.report);
1099 revision = daemon_runtime.allocate_revision();
1100 publish_snapshot!(revision);
1101 }
1102 Some(QuotaUpdate::Refreshing { .. } | QuotaUpdate::Finished { .. }) => {}
1103 None => {
1104 quota_updates_open = false;
1105 tracing::warn!("quota refresher stopped while the phone server is running");
1106 }
1107 }
1108 }
1109 projected = conversation_projection_rx.recv() => {
1110 let Some(projected) = projected else {
1111 failure = feed_stopped(
1112 termination.is_cancelled(),
1113 "the browser transcript projection feed stopped",
1114 );
1115 break;
1116 };
1117 let session_id = projected.session_id.clone();
1118 let session_active = controller
1119 .state
1120 .sessions
1121 .get(&session_id)
1122 .is_some_and(|session| session.state.is_active());
1123 if !session_active {
1124 conversation_projections.forget(&session_id);
1129 }
1130 if let Some((session_id, _key, transcript)) =
1131 conversation_projections.finish(projected, session_active)
1132 {
1133 conversations.insert(session_id, transcript);
1134 revision = daemon_runtime.allocate_revision();
1135 conversation_tx.send_replace(conversations.clone());
1136 publish_snapshot!(revision);
1137 } else if !session_active && conversations.remove(&session_id).is_some() {
1138 revision = daemon_runtime.allocate_revision();
1143 conversation_tx.send_replace(conversations.clone());
1144 publish_snapshot!(revision);
1145 }
1146 tokio::task::yield_now().await;
1150 }
1151 update = worker_updates_rx.recv() => {
1152 let Some(update) = update else {
1153 failure = feed_stopped(termination.is_cancelled(), "the session manager stopped; the phone server can no longer follow sessions");
1154 break;
1155 };
1156 if let Some(snapshot) = update.view.snapshot.as_ref()
1157 && let Some(session) = controller.state.sessions.get(&update.session_id)
1158 && let Some(signal) = snapshot.latest_credential_sync_signal.clone()
1159 {
1160 credential_sync_signals.observe(
1161 &update.session_id,
1162 &session.last_profile,
1163 signal,
1164 );
1165 }
1166 schedule_due_credential_syncs(
1167 &mut credential_sync_signals,
1168 &credential_sync_handle,
1169 Instant::now(),
1170 );
1171 apply_worker_record_update(&mut controller, &update);
1172 if let Some(snapshot) = update.view.snapshot {
1173 for request in snapshot.subagent_requests.iter().cloned() {
1174 let identity = (update.session_id.clone(), request.request_id.clone());
1175 if !active_subagent_requests.insert(identity.clone()) {
1176 continue;
1177 }
1178 let backend = api_backend.clone();
1179 let runtime = daemon_runtime.clone();
1180 let parent_session_id = update.session_id.clone();
1181 subagent_jobs.spawn(async move {
1182 let result = backend
1183 .execute_subagent_tool(parent_session_id.clone(), request)
1184 .await;
1185 let outcome = async {
1186 backend
1187 .deliver_subagent_result(parent_session_id.clone(), &result)
1188 .await?;
1189 let handle = runtime
1190 .workspace_session_handle(&parent_session_id)
1191 .await?;
1192 let mut lease = handle.lease_connection().await?;
1193 lease
1194 .connection_mut()
1195 .complete_subagent_request(result)
1196 .await?;
1197 lease.release();
1198 anyhow::Ok(())
1199 }
1200 .await;
1201 (identity, outcome)
1202 });
1203 }
1204 if let Some(relation) = controller.state.subagents.get_mut(&update.session_id)
1205 && matches!(snapshot.materialized.execution, mj_core::state::MaterializedExecutionState::Idle)
1206 && let Some(outcome) = snapshot.materialized.last_turn_outcome.as_ref()
1207 && relation.delivered_turn != Some(outcome.completed_ordinal)
1208 {
1209 let turn = outcome.completed_ordinal;
1210 let output = snapshot
1211 .materialized
1212 .transcript
1213 .iter()
1214 .rev()
1215 .find_map(|item| match &item.body {
1216 mj_core::transcript::TranscriptBody::Agent { chunks, .. }
1217 if item.position >= outcome.turn_start_position.unwrap_or(0) =>
1218 {
1219 Some(mj_core::transcript::materialized_chunks_text(chunks))
1220 }
1221 _ => None,
1222 })
1223 .unwrap_or_else(|| "The sub-agent completed without a final text response.".to_owned());
1224 let child_id = relation.child_session_id.clone();
1225 let parent_id = relation.parent_session_id.clone();
1226 let task_name = relation.task_name.clone();
1227 let outcome_name = format!("{:?}", outcome.outcome).to_lowercase();
1228 relation.delivered_turn = Some(turn);
1229 let backend = api_backend.clone();
1230 subagent_completion_jobs.spawn(async move {
1231 let result = async {
1232 backend
1233 .deliver_subagent_completion(
1234 parent_id,
1235 &child_id,
1236 &task_name,
1237 turn,
1238 &outcome_name,
1239 &output,
1240 )
1241 .await?;
1242 tokio::task::spawn_blocking({
1243 let child_id = child_id.clone();
1244 move || crate::database::mark_subagent_turn_delivered(&child_id, turn)
1245 })
1246 .await??;
1247 anyhow::Ok(())
1248 }
1249 .await;
1250 (child_id, turn, result)
1251 });
1252 }
1253 if snapshot.operational.native_session_is_ready()
1254 && operational.get(&update.session_id).is_none_or(|old: &mj_core::relay::RelayOperationalState| old.config_options != snapshot.operational.config_options)
1255 && let Some(session) = controller.state.sessions.get(&update.session_id)
1256 && matches!(session.target, Some(mj_core::state::TargetLocator::LocalBare { .. } | mj_core::state::TargetLocator::SshBare { .. } | mj_core::state::TargetLocator::AwsEc2 { .. }))
1257 && let Some(build) = snapshot.worker_build.clone()
1258 {
1259 let profile = session.last_profile.clone();
1260 let state = snapshot.operational.clone();
1261 tokio::spawn(async move {
1262 if let Err(error) = crate::controller::profile_config::observe(profile, build, state).await {
1263 tracing::warn!(%error, "could not cache observed profile choices");
1264 }
1265 });
1266 }
1267 let materialized = snapshot.materialized;
1268 let operational_state = snapshot.operational;
1269 materialized_activity.insert(
1270 update.session_id.clone(),
1271 materialized.last_activity_at_ms,
1272 );
1273 let queued = queued_prompt_projection(&materialized);
1274 let pending = materialized.pending_elicitations.clone();
1275 let active_shells = operational_state.active_user_shells.clone();
1276 let prompt_images_supported =
1277 agent_accepts_prompt_images(&operational_state);
1278 active_user_shells.insert(
1279 update.session_id.clone(),
1280 active_shells,
1281 );
1282 conversation_projections.enqueue(materialized);
1283 queued_prompts.insert(
1284 update.session_id.clone(),
1285 queued,
1286 );
1287 pending_elicitations.insert(
1288 update.session_id.clone(),
1289 pending,
1290 );
1291 if prompt_images_supported {
1292 prompt_images.insert(update.session_id.clone());
1293 } else {
1294 prompt_images.remove(&update.session_id);
1295 }
1296 operational.insert(
1297 update.session_id.clone(),
1298 operational_state,
1299 );
1300 revision = daemon_runtime.allocate_revision();
1301 conversation_tx.send_replace(conversations.clone());
1302 publish_snapshot!(revision);
1303 }
1304 tokio::task::yield_now().await;
1309 }
1310 completed = subagent_jobs.join_next(), if !subagent_jobs.is_empty() => {
1311 match completed {
1312 Some(Ok((identity, Ok(())))) => {
1313 active_subagent_requests.remove(&identity);
1314 }
1315 Some(Ok((identity, Err(error)))) => {
1316 active_subagent_requests.remove(&identity);
1317 tracing::warn!(
1318 parent_session_id = %identity.0,
1319 request_id = %identity.1,
1320 error = %format!("{error:#}"),
1321 "sub-agent tool request failed"
1322 );
1323 }
1324 Some(Err(error)) => tracing::warn!(%error, "sub-agent tool task panicked"),
1325 None => {}
1326 }
1327 }
1328 completed = subagent_completion_jobs.join_next(), if !subagent_completion_jobs.is_empty() => {
1329 match completed {
1330 Some(Ok((_, _, Ok(())))) => {}
1331 Some(Ok((child_id, turn, Err(error)))) => {
1332 if let Some(relation) = controller.state.subagents.get_mut(&child_id)
1333 && relation.delivered_turn == Some(turn)
1334 {
1335 relation.delivered_turn = None;
1336 }
1337 tracing::warn!(%child_id, turn, error = %format!("{error:#}"), "could not deliver sub-agent completion");
1338 }
1339 Some(Err(error)) => tracing::warn!(%error, "sub-agent completion task panicked"),
1340 None => {}
1341 }
1342 }
1343 _ = prune_tick.tick() => {
1344 tokio::spawn(async move {
1348 let pruned = tokio::task::spawn_blocking(move || {
1349 crate::database::prune_phone_client_state(client_state_retention)
1350 })
1351 .await;
1352 match pruned {
1353 Ok(Ok(0)) => {}
1354 Ok(Ok(rows)) => tracing::debug!(rows, "pruned expired phone viewer state"),
1355 Ok(Err(error)) => tracing::warn!(%error, "could not prune phone viewer state"),
1356 Err(error) => tracing::warn!(%error, "phone viewer state pruning task failed"),
1357 }
1358 });
1359 }
1360 _ = credential_tick.tick() => {
1361 schedule_due_credential_syncs(
1362 &mut credential_sync_signals,
1363 &credential_sync_handle,
1364 Instant::now(),
1365 );
1366 while let Some(result) = credential_sync.try_result() {
1367 crate::pollers::log_credential_sync_actions(&result);
1368 let harness = controller
1369 .config
1370 .profiles
1371 .get(&result.profile_id)
1372 .map(|profile| profile.kind);
1373 if let Some(notice) = credential_sync_notices.notice(&result, harness) {
1374 eprintln!("Mjolnir: {notice}");
1375 }
1376 }
1377 }
1378 request = dictation_rx.recv() => {
1379 let Some(request) = request else {
1380 failure = feed_stopped(termination.is_cancelled(), "the phone HTTP server stopped delivering dictation requests");
1381 break;
1382 };
1383 let paths = controller.state.sessions.get(&request.session_id).map(|session| {
1384 crate::dictation::auth_paths(&controller.config, &session.last_profile)
1385 });
1386 dictation_jobs.spawn(crate::dictation::execute(
1387 request, paths, termination.clone(),
1388 ));
1389 }
1390 job = dictation_jobs.join_next(), if !dictation_jobs.is_empty() => {
1391 if let Some(Err(error)) = job {
1392 tracing::warn!(%error, "web dictation task failed");
1393 }
1394 }
1395 stored = client_state_rx.recv() => {
1396 let Some(stored) = stored else {
1397 failure = feed_stopped(termination.is_cancelled(), "the phone HTTP server stopped delivering viewer state requests");
1398 break;
1399 };
1400 let workspace_of = |session_id: &str| {
1404 controller
1405 .state
1406 .sessions
1407 .get(session_id)
1408 .map(|session| session.workspace_id.clone())
1409 };
1410 let bundle_of = |session_id: &str| {
1411 controller
1412 .state
1413 .sessions
1414 .get(session_id)
1415 .map(|session| session.bundle_id.clone())
1416 };
1417 match stored {
1418 crate::server::ClientStateRequest::Read { client_id, session_id, reply } => {
1419 let workspace = workspace_of(&session_id);
1420 tokio::spawn(async move {
1421 let answer = tokio::task::spawn_blocking(move || {
1422 let workspace = workspace.context("unknown session")?;
1423 let state = crate::database::client_session_state(
1424 &client_id, &workspace, &session_id,
1425 )?;
1426 anyhow::Ok(crate::server::ViewerClientState {
1427 draft: state.draft,
1428 through_event_ordinal: state.through_event_ordinal,
1429 })
1430 })
1431 .await;
1432 reply.send(flatten_stored(answer)).ok();
1433 });
1434 }
1435 crate::server::ClientStateRequest::SaveDraft { client_id, session_id, draft, reply } => {
1436 let workspace = workspace_of(&session_id);
1437 tokio::spawn(async move {
1438 let answer = tokio::task::spawn_blocking(move || {
1439 let workspace = workspace.context("unknown session")?;
1440 crate::database::persist_client_draft(
1441 &client_id, &workspace, &session_id, &draft,
1442 )
1443 })
1444 .await;
1445 reply.send(flatten_stored(answer)).ok();
1446 });
1447 }
1448 crate::server::ClientStateRequest::MarkWorkspaceRead { client_id, workspace_id, reply } => {
1449 let sessions = controller
1450 .state
1451 .sessions
1452 .values()
1453 .filter(|session| session.workspace_id == workspace_id)
1454 .map(|session| (session.id.clone(), session.viewed_through_event_ordinal))
1455 .collect::<Vec<_>>();
1456 tokio::spawn(async move {
1457 let answer = tokio::task::spawn_blocking(move || {
1458 for (session_id, through) in sessions {
1459 crate::database::persist_read_receipt(
1463 &client_id, &workspace_id, &session_id, through,
1464 )
1465 .ok();
1466 }
1467 anyhow::Ok(())
1468 })
1469 .await;
1470 reply.send(flatten_stored(answer)).ok();
1471 });
1472 }
1473 crate::server::ClientStateRequest::History { session_id, query, scope, reply } => {
1474 let bundle = bundle_of(&session_id);
1475 tokio::spawn(async move {
1476 let answer = tokio::task::spawn_blocking(move || {
1477 let bundle = bundle.context("unknown session")?;
1478 let scope = match scope.as_str() {
1479 "session" => crate::database::HistoryScope::Session,
1480 "all" => crate::database::HistoryScope::All,
1481 _ => crate::database::HistoryScope::Project,
1482 };
1483 let found = crate::database::search_prompts_bounded(
1484 &session_id,
1485 &bundle,
1486 scope,
1487 &query,
1488 crate::server::MAX_HISTORY_MATCHES,
1489 )?;
1490 anyhow::Ok(crate::server::ViewerPromptHistory {
1491 entries: found
1492 .entries
1493 .into_iter()
1494 .map(|entry| entry.text)
1495 .collect(),
1496 truncated: found.truncated,
1497 })
1498 })
1499 .await;
1500 reply.send(flatten_stored(answer)).ok();
1501 });
1502 }
1503 }
1504 }
1505 bundle = bundle_rx.recv(), if bundle_jobs.len() < MAX_CONCURRENT_BUNDLE_CREATIONS => {
1506 let Some(crate::server::BundleRequest { source, reply }) = bundle else {
1507 failure = feed_stopped(termination.is_cancelled(), "the phone HTTP server stopped delivering bundle requests");
1508 break;
1509 };
1510 let done = bundle_done_tx.clone();
1515 let daemon_runtime = daemon_runtime.clone();
1516 bundle_jobs.spawn(async move {
1517 let result = daemon_runtime
1518 .create_quick_bundle(source)
1519 .await;
1520 if let Err(error) = done.send(BundleCreated { result, reply }) {
1521 tracing::debug!(%error, "bundle creation finished after the server stopped");
1522 }
1523 });
1524 }
1525 bundle_done = bundle_done_rx.recv() => {
1526 let Some(BundleCreated { result, reply }) = bundle_done else {
1527 failure = feed_stopped(termination.is_cancelled(), "the bundle creation pipeline stopped while the phone server was running");
1528 break;
1529 };
1530 match result {
1531 Ok(created) => {
1532 let bundle_id = created.bundle_id;
1533 let Some(bundle) = created.config.bundles.get(&bundle_id) else {
1539 tracing::error!(%bundle_id, "bundle creation returned a config without its bundle");
1540 if reply.send(Err(crate::server::BundleFailure::Controller)).is_err() {
1541 tracing::debug!("bundle creation failure reply dropped after client disconnect");
1542 }
1543 continue;
1544 };
1545 controller
1546 .config
1547 .bundles
1548 .insert(bundle_id.clone(), bundle.clone());
1549 controller_reload_invalidated |= controller_reload_in_flight;
1553 revision = daemon_runtime.allocate_revision();
1554 publish_snapshot!(revision);
1555 request_daemon_controller_reload(
1556 daemon_runtime.clone(),
1557 "new bundle publication",
1558 );
1559 if reply.send(Ok(bundle_id)).is_err() {
1560 tracing::debug!("bundle creation reply dropped after client disconnect");
1561 }
1562 }
1563 Err(error) => {
1564 let failure = match error {
1565 crate::controller::QuickBundleFailure::InvalidSource(
1566 detail,
1567 ) => {
1568 tracing::debug!(error = %detail, "phone bundle source was invalid");
1569 crate::server::BundleFailure::InvalidSource
1570 }
1571 crate::controller::QuickBundleFailure::Persistence(
1572 detail,
1573 ) => {
1574 tracing::warn!(error = %detail, "phone bundle creation failed");
1575 crate::server::BundleFailure::Controller
1576 }
1577 };
1578 if reply.send(Err(failure)).is_err() {
1579 tracing::debug!("bundle creation failure reply dropped after client disconnect");
1580 }
1581 }
1582 }
1583 }
1584 bundle_job = bundle_jobs.join_next(), if !bundle_jobs.is_empty() => {
1585 if let Some(Err(error)) = bundle_job {
1586 tracing::warn!(%error, "bundle creation task panicked");
1587 }
1588 }
1589 preflight = preflight_rx.recv(), if preflight_jobs.len() < MAX_CONCURRENT_PREFLIGHTS => {
1590 let Some(crate::server::PreflightRequest {
1591 bundle_id,
1592 target_id,
1593 project_directory,
1594 mut reply,
1595 remote_repairs,
1596 }) = preflight else {
1597 failure = feed_stopped(termination.is_cancelled(), "the phone HTTP server stopped delivering preflight requests");
1598 break;
1599 };
1600 let config = controller.config.clone();
1605 let project_validation = project_directory.is_some();
1606 let task_termination = termination.clone();
1607 preflight_jobs.spawn(async move {
1608 let cancelled = Arc::new(AtomicBool::new(false));
1609 let cancellation_guard = ProcessCancellationGuard(cancelled.clone());
1610 let mut blocking = tokio::task::spawn_blocking(move || {
1611 run_new_preflight_with_cancellation(
1612 config,
1613 bundle_id,
1614 target_id,
1615 project_directory,
1616 cancelled,
1617 remote_repairs,
1618 )
1619 });
1620 let answer = tokio::select! {
1621 biased;
1622 _ = task_termination.cancelled() => None,
1623 _ = reply.closed() => None,
1624 answer = &mut blocking => Some(answer),
1625 };
1626 let Some(answer) = answer else {
1627 drop(cancellation_guard);
1628 match blocking.await {
1629 Err(error) => tracing::warn!(%error, "cancelled phone preflight task failed"),
1630 Ok(Err(error)) => tracing::debug!(%error, "phone preflight cancelled"),
1631 Ok(Ok(_)) => {}
1632 }
1633 return;
1634 };
1635 let answer = match answer {
1636 Ok(Ok(answer)) => Ok(answer),
1637 Ok(Err(error)) => {
1638 tracing::debug!(
1639 error = %error,
1640 project_validation,
1641 "phone preflight check failed"
1642 );
1643 Err(if project_validation {
1644 PreflightFailure::Validation
1645 } else {
1646 PreflightFailure::InvalidRepository(format!("{error:#}"))
1647 })
1648 }
1649 Err(error) => {
1650 tracing::warn!(%error, "phone preflight task failed");
1651 Err(PreflightFailure::Controller(format!(
1652 "preflight task failed: {error}"
1653 )))
1654 }
1655 };
1656 if reply.send(answer).is_err() {
1657 tracing::debug!("phone preflight reply dropped after client disconnect");
1658 }
1659 });
1660 }
1661 preflight_job = preflight_jobs.join_next(), if !preflight_jobs.is_empty() => {
1662 if let Some(Err(error)) = preflight_job {
1663 tracing::warn!(%error, "phone preflight task panicked");
1664 }
1665 }
1666 preparation = move_preparation_rx.recv() => {
1667 let Some(MovePreparationRequest { selection, reply }) = preparation else {
1668 failure = feed_stopped(termination.is_cancelled(), "the phone HTTP server stopped delivering move preparation requests");
1669 break;
1670 };
1671 let done = move_prepared_tx.clone();
1676 let daemon_runtime = daemon_runtime.clone();
1677 move_preparation_jobs.spawn(async move {
1678 let result = daemon_runtime
1679 .prepare_move_session(selection)
1680 .await
1681 .map_err(|error| format!("{error:#}"));
1682 if let Err(error) = done.send(MovePrepared { result, reply }) {
1683 tracing::debug!(%error, "move preparation finished after the server stopped");
1684 }
1685 });
1686 }
1687 prepared = move_prepared_rx.recv() => {
1688 let Some(MovePrepared { result, reply }) = prepared else {
1689 failure = feed_stopped(termination.is_cancelled(), "the move preparation pipeline stopped while the phone server was running");
1690 break;
1691 };
1692 if reply.send(result).is_err() {
1693 tracing::debug!("move preparation reply dropped after client disconnect");
1694 }
1695 }
1696 move_preparation_job = move_preparation_jobs.join_next(), if !move_preparation_jobs.is_empty() => {
1697 if let Some(Err(error)) = move_preparation_job {
1698 tracing::warn!(%error, "move preparation task failed");
1699 }
1700 }
1701 move_recovery_job = move_recovery_jobs.join_next(), if !move_recovery_jobs.is_empty() => {
1702 if let Some(Err(error)) = move_recovery_job {
1703 move_recovery_load_in_flight = false;
1704 tracing::warn!(%error, "Move recovery projection task failed");
1705 }
1706 }
1707 receipt = receipt_rx.recv() => {
1708 let Some(ReadReceiptRequest { client_id, session_id, through, reply }) = receipt else {
1709 failure = feed_stopped(termination.is_cancelled(), "the phone HTTP server stopped delivering read receipts");
1710 break;
1711 };
1712 match controller.state.sessions.get(&session_id) {
1713 None => {
1714 if reply.send(Err("unknown session".into())).is_err() {
1715 tracing::debug!(%session_id, "unknown-session read receipt reply dropped after client disconnect");
1716 }
1717 }
1718 Some(session) => {
1719 let workspace_id = session.workspace_id.clone();
1720 let done = receipt_done_tx.clone();
1721 let persisted_session_id = session_id.clone();
1722 tokio::spawn(async move {
1723 let joined = tokio::task::spawn_blocking(move || {
1724 crate::database::persist_read_receipt(
1725 &client_id,
1726 &workspace_id,
1727 &persisted_session_id,
1728 through,
1729 )
1730 })
1731 .await;
1732 let result = match joined {
1733 Ok(result) => result.map_err(|error| format!("{error:#}")),
1734 Err(error) => Err(format!("phone read receipt task failed: {error}")),
1735 };
1736 if let Err(error) = done.send(ReadReceiptPersisted { session_id, result, reply }) {
1737 tracing::debug!(%error, "phone read receipt finished after the server stopped");
1738 }
1739 });
1740 }
1741 }
1742 }
1743 persisted = receipt_done_rx.recv() => {
1744 let Some(ReadReceiptPersisted { session_id, result, reply }) = persisted else { continue };
1745 match result {
1746 Ok(receipt) => {
1747 let _ = receipt;
1748 if reply.send(Ok(())).is_err() {
1749 tracing::debug!(%session_id, "phone read receipt reply dropped after client disconnect");
1750 }
1751 }
1752 Err(error) => {
1753 tracing::warn!(%session_id, "could not persist a phone read receipt: {error}");
1754 if reply.send(Err(error)).is_err() {
1755 tracing::debug!(%session_id, "failed phone read receipt reply dropped after client disconnect");
1756 }
1757 }
1758 }
1759 }
1760 action = action_rx.recv() => {
1761 let Some(request) = action else {
1762 failure = feed_stopped(termination.is_cancelled(), "the phone HTTP server stopped delivering actions");
1763 break;
1764 };
1765 match &request.action {
1769 ControllerAction::RefreshCapacity { target_id } => {
1770 let known = capacity_state.contains_key(target_id);
1771 let accepted = known && match capacity_triggers_tx.try_send(()) {
1774 Ok(()) | Err(tokio::sync::mpsc::error::TrySendError::Full(())) => true,
1775 Err(tokio::sync::mpsc::error::TrySendError::Closed(())) => {
1776 tracing::warn!("phone capacity refresh rejected: poller stopped");
1777 false
1778 }
1779 };
1780 if known {
1781 if let Some(entry) = capacity_state.get_mut(target_id) {
1782 entry.refreshing = accepted;
1783 if !accepted {
1784 entry.failed = true;
1785 }
1786 }
1787 revision = daemon_runtime.allocate_revision();
1788 publish_snapshot!(revision);
1789 }
1790 let outcome = if accepted {
1791 ActionOutcome::accepted()
1792 } else {
1793 ActionOutcome::Failed
1794 };
1795 if request.reply.send(outcome).is_err() {
1796 tracing::debug!(%target_id, "phone capacity refresh reply dropped after client disconnect");
1797 }
1798 tokio::task::yield_now().await;
1799 continue;
1800 }
1801 ControllerAction::RefreshQuota { profile_id } => {
1802 let known = controller.config.enabled_profile(profile_id).is_some();
1803 if known {
1804 quota_batch.generation = quota_batch.generation.saturating_add(1);
1808 quota_batch.profiles = quota_refresh_profiles(&controller);
1809 quota_profiles_tx.send_replace(quota_batch.clone());
1810 }
1811 let outcome = if known {
1812 ActionOutcome::accepted()
1813 } else {
1814 ActionOutcome::Failed
1815 };
1816 if request.reply.send(outcome).is_err() {
1817 tracing::debug!(%profile_id, "phone quota refresh reply dropped after client disconnect");
1818 }
1819 tokio::task::yield_now().await;
1820 continue;
1821 }
1822 _ => {}
1823 }
1824 if let ControllerAction::Cancel { session_id } = &request.action {
1825 let outcome = if request_phone_action_cancellation(
1826 session_id,
1827 &action_sessions,
1828 &action_cancellations,
1829 ) {
1830 daemon_runtime.cancel_lifecycle_if_active(session_id);
1831 ActionOutcome::accepted()
1832 } else {
1833 ActionOutcome::NotCancellable
1834 };
1835 if request.reply.send(outcome).is_err() {
1836 tracing::debug!(%session_id, "phone cancellation reply dropped after client disconnect");
1837 }
1838 tokio::task::yield_now().await;
1839 continue;
1840 }
1841 if let ControllerAction::Close { session_id } = &request.action {
1842 if closing_actions.contains_key(session_id) {
1843 if request.reply.send(ActionOutcome::accepted()).is_err() { tracing::debug!(%session_id, "repeated close reply dropped"); }
1844 continue;
1845 }
1846 request_phone_action_cancellation(session_id, &action_sessions, &action_cancellations);
1847 daemon_runtime.request_close(session_id);
1848 }
1849 let session_id = match admit_phone_action(
1850 &request.action,
1851 action_cancellations.len(),
1852 &mut active_actions,
1853 ) {
1854 Ok(session_id) => session_id,
1855 Err(refusal) => {
1856 if request.reply.send(refusal).is_err() {
1857 tracing::debug!("phone action refusal reply dropped after client disconnect");
1858 }
1859 tokio::task::yield_now().await;
1860 continue;
1861 }
1862 };
1863 let ControllerRequest { action, reply } = request;
1864 let done = action_done_tx.clone();
1865 let session_control = worker_commands_tx.clone();
1866 let daemon_runtime = daemon_runtime.clone();
1867 let started = action_started_tx.clone();
1868 next_action_id = next_action_id.wrapping_add(1).max(1);
1869 let action_id = next_action_id;
1870 if let ControllerAction::Close { session_id } = &action { closing_actions.insert(session_id.clone(), action_id); }
1871 if let ControllerAction::New { workspace_id, .. } = &action {
1872 let workspace_id = if workspace_id.is_empty() && phone_workspaces.len() == 1 {
1873 phone_workspaces[0].id.clone()
1874 } else {
1875 workspace_id.clone()
1876 };
1877 launch_workspaces.insert(action_id, workspace_id);
1878 }
1879 let control = PhoneActionControl::for_action(&action);
1880 action_cancellations.insert(action_id, control.clone());
1881 if let Some(session_id) = &session_id {
1882 action_sessions.insert(action_id, session_id.clone());
1883 }
1884 action_replies.accept(action_id, &action, reply);
1885 let runtime = tokio::runtime::Handle::current();
1886 tokio::spawn(async move {
1887 let joined = tokio::task::spawn_blocking(move || {
1888 let result = (|| -> Result<()> {
1889 if control.cancelled.load(Ordering::Acquire) {
1890 bail!("phone action cancelled");
1891 }
1892 let mut operation_controller = Controller::load()?;
1893 let executor =
1894 CancellableProcessExecutor::new(control.cancelled.clone());
1895 runtime.block_on(apply_phone_action(
1896 &mut operation_controller,
1897 PhoneActionServices {
1898 sessions: &session_control,
1899 daemon_runtime: &daemon_runtime,
1900 },
1901 action,
1902 &executor,
1903 action_id,
1904 &started,
1905 &control,
1906 ))
1907 })();
1908 result.map_err(|error| format!("{error:#}"))
1909 })
1910 .await;
1911 let result = match joined {
1912 Ok(result) => result,
1913 Err(error) => Err(format!("phone action task failed: {error}")),
1914 };
1915 if let Err(error) = done.send((action_id, session_id, result)) {
1916 tracing::debug!(action_id, %error, "phone action finished after the server stopped");
1917 }
1918 });
1919 }
1920 started = action_started_rx.recv() => {
1921 let Some(started) = started else {
1922 tokio::task::yield_now().await;
1923 continue;
1924 };
1925 let started_session_id = started.session.id.clone();
1926 let publication = if !action_cancellations.contains_key(&started.action_id) {
1927 Err("phone action completed before its provisional session was published".into())
1928 } else {
1929 track_started_phone_session(
1930 &mut controller.state,
1931 &mut active_actions,
1932 &mut action_sessions,
1933 started.action_id,
1934 started.session,
1935 )
1936 };
1937 if publication.is_ok() {
1938 revision = daemon_runtime.allocate_revision();
1939 publish_snapshot!(revision);
1940 request_daemon_controller_reload(
1941 daemon_runtime.clone(),
1942 "new session publication",
1943 );
1944 };
1945 if publication.is_err()
1946 && let Some(control) = action_cancellations.get(&started.action_id)
1947 {
1948 control.request_cancel();
1949 }
1950 action_replies.resolve(
1953 started.action_id,
1954 if publication.is_ok() {
1955 ActionOutcome::Accepted {
1956 session_id: Some(started_session_id),
1957 }
1958 } else {
1959 ActionOutcome::Failed
1960 },
1961 );
1962 if started.published.send(publication).is_err() {
1963 tracing::debug!(action_id = started.action_id, "phone new-session publication reply dropped after client disconnect");
1964 }
1965 }
1966 completed = action_done_rx.recv() => {
1967 let Some((action_id, session_id, result)) = completed else {
1968 failure = feed_stopped(termination.is_cancelled(), "the phone action pipeline stopped reporting completions");
1969 break;
1970 };
1971 action_cancellations.remove(&action_id);
1972 let session_id = action_sessions.remove(&action_id).or(session_id);
1973 if closing_actions.values().any(|closing_id| *closing_id == action_id) && let Some(id) = &session_id { daemon_runtime.clear_close_request(id); }
1974 closing_actions.retain(|_, closing_id| *closing_id != action_id);
1975 if let Some(session_id) = &session_id && !action_sessions.values().any(|active| active == session_id) {
1976 active_actions.remove(session_id);
1977 }
1978 action_replies.resolve(action_id, ActionOutcome::Failed);
1982 if let Some(workspace_id) = launch_workspaces.remove(&action_id)
1983 && result.is_err()
1984 && !session_id.as_ref().is_some_and(|id| closing_actions.contains_key(id))
1985 {
1986 record_launch_failure(
1987 &mut launch_failures,
1988 action_id,
1989 workspace_id,
1990 session_id.clone(),
1991 );
1992 revision = daemon_runtime.allocate_revision();
1993 publish_snapshot!(revision);
1994 }
1995 if let Err(error) = &result {
1996 tracing::warn!(action_id, %error, "phone action failed");
1997 }
1998 record_action_result(
1999 &mut pending_action_errors,
2000 session_id.as_deref(),
2001 &result,
2002 );
2003 request_controller_reload(
2004 &mut controller_reload_in_flight,
2005 &mut controller_reload_requested,
2006 &controller_reload_tx,
2007 );
2008 request_move_recovery_reload(
2009 &move_recovery_tx,
2010 &mut move_recovery_load_in_flight,
2011 &mut move_recovery_jobs,
2012 );
2013 request_daemon_controller_reload(
2014 daemon_runtime.clone(),
2015 "phone action completion",
2016 );
2017 }
2018 reloaded = controller_reload_rx.recv() => {
2019 let Some(ControllerReloaded { result }) = reloaded else {
2020 failure = feed_stopped(
2021 termination.is_cancelled(),
2022 "the controller reload pipeline stopped while the phone server was running",
2023 );
2024 break;
2025 };
2026 controller_reload_in_flight = false;
2027 if std::mem::take(&mut controller_reload_invalidated) {
2028 if let Err(error) = &result {
2029 tracing::warn!(%error, "superseded controller reload failed");
2030 }
2031 controller_reload_requested = false;
2032 request_controller_reload(
2033 &mut controller_reload_in_flight,
2034 &mut controller_reload_requested,
2035 &controller_reload_tx,
2036 );
2037 continue;
2038 }
2039 match result {
2040 Ok(mut reloaded) => {
2041 for (session_id, error) in &pending_action_errors {
2042 if let Some(session) = reloaded.state.sessions.get_mut(session_id)
2043 && session.last_error.is_none()
2044 {
2045 session.last_error = Some(error.clone());
2046 }
2047 }
2048 controller = reloaded;
2049 quotas.retain(|id, _| controller.config.enabled_profile(id).is_some());
2050 subagent_quota_reports
2051 .lock()
2052 .expect("sub-agent quota reports lock poisoned")
2053 .retain(|id, _| controller.config.enabled_profile(id).is_some());
2054 worker_targets_tx.send_replace(dashboard_worker_targets(&controller));
2055 publish_capacity_targets(
2056 &controller,
2057 &capacity_targets_tx,
2058 &mut capacity_state,
2059 );
2060 credential_sync_handle.set_targets(credential_sync_targets(&controller));
2061 republish_quota_profiles(
2062 &controller,
2063 &mut published_quota_profiles,
2064 &mut quota_batch,
2065 "a_profiles_tx,
2066 );
2067 queued_prompts.retain(|session_id, _| {
2068 controller.state.sessions.contains_key(session_id)
2069 });
2070 pending_elicitations.retain(|session_id, _| {
2071 controller.state.sessions.contains_key(session_id)
2072 });
2073 prompt_images.retain(|session_id| {
2074 controller.state.sessions.contains_key(session_id)
2075 });
2076 operational.retain(|session_id, _| {
2077 controller.state.sessions.contains_key(session_id)
2078 });
2079 materialized_activity.retain(|session_id, _| {
2080 controller.state.sessions.contains_key(session_id)
2081 });
2082 request_move_recovery_reload(
2083 &move_recovery_tx,
2084 &mut move_recovery_load_in_flight,
2085 &mut move_recovery_jobs,
2086 );
2087 conversations.retain(|id, _| {
2088 controller.state.sessions.get(id).is_some_and(|session| session.state.is_active())
2089 });
2090 for session_id in conversation_projections.session_ids() {
2091 if !controller
2092 .state
2093 .sessions
2094 .get(&session_id)
2095 .is_some_and(|session| session.state.is_active())
2096 {
2097 conversation_projections.forget(&session_id);
2098 }
2099 }
2100 revision = daemon_runtime.allocate_revision();
2101 conversation_tx.send_replace(conversations.clone());
2102 publish_snapshot!(revision);
2103 }
2104 Err(error) => {
2105 tracing::warn!(%error, "completed phone operation could not reload controller state");
2106 }
2107 }
2108 if controller_reload_requested {
2109 controller_reload_requested = false;
2110 controller_reload_in_flight = true;
2111 spawn_controller_reload(controller_reload_tx.clone());
2112 }
2113 }
2114 }
2115 }
2116 dictation_jobs.shutdown().await;
2118 bundle_jobs.shutdown().await;
2121 preflight_jobs.shutdown().await;
2125 move_preparation_jobs.shutdown().await;
2128 move_recovery_jobs.shutdown().await;
2129 crate::controller::profile_config::cancel_all();
2131 for control in action_cancellations.values() {
2132 control.request_cancel();
2133 }
2134 match failure {
2135 Some(failure) => Err(failure),
2136 None => Ok::<(), anyhow::Error>(()),
2137 }
2138 };
2139 let result = tokio::select! {
2140 result = api_activity::record_activity_stream(activity_snapshots) => result.context("native API activity recorder stopped"),
2141 result = serve => result,
2142 result = control => result,
2143 };
2144 conversation_projection_shutdown.cancel();
2145 renewal_cancellation.cancel();
2146 if let Some(task) = renewal_task
2147 && let Err(error) = task.await
2148 {
2149 tracing::warn!(%error, "Tailscale certificate renewal task failed");
2150 }
2151 worker_shutdown
2152 .shutdown()
2153 .await
2154 .context("shut down phone server session manager")?;
2155 result?;
2156 Ok(())
2157}
2158
2159fn feed_stopped(shutting_down: bool, reason: &'static str) -> Option<anyhow::Error> {
2165 (!shutting_down).then(|| anyhow::anyhow!(reason))
2166}
2167
2168fn agent_accepts_prompt_images(operational: &mj_core::relay::RelayOperationalState) -> bool {
2172 operational
2173 .agent_capabilities
2174 .as_ref()
2175 .is_some_and(|capabilities| capabilities.prompt_capabilities.image)
2176}
2177
2178fn controller_action_session_id(action: &ControllerAction) -> Option<String> {
2179 match action {
2180 ControllerAction::New { .. } => None,
2181 ControllerAction::Prompt { session_id, .. }
2182 | ControllerAction::RunShell { session_id, .. }
2183 | ControllerAction::CancelShell { session_id, .. }
2184 | ControllerAction::Close { session_id }
2185 | ControllerAction::Resume { session_id, .. }
2186 | ControllerAction::Open { session_id }
2187 | ControllerAction::Cancel { session_id }
2188 | ControllerAction::RemoveQueuedPrompt { session_id, .. }
2189 | ControllerAction::RespondElicitation { session_id, .. }
2190 | ControllerAction::Rename { session_id, .. }
2191 | ControllerAction::CancelTurn { session_id }
2192 | ControllerAction::SetConfig { session_id, .. }
2193 | ControllerAction::SetPlanMode { session_id, .. }
2194 | ControllerAction::StartReview { session_id }
2195 | ControllerAction::ResolveReview { session_id, .. } => Some(session_id.clone()),
2196 ControllerAction::Move { request } => {
2197 Some(request.preparation.selection.session_id.clone())
2198 }
2199 ControllerAction::RefreshQuota { .. } | ControllerAction::RefreshCapacity { .. } => None,
2202 }
2203}
2204
2205fn flatten_stored<T>(
2207 joined: std::result::Result<Result<T>, tokio::task::JoinError>,
2208) -> std::result::Result<T, String> {
2209 match joined {
2210 Ok(Ok(value)) => Ok(value),
2211 Ok(Err(error)) => Err(format!("{error:#}")),
2212 Err(error) => Err(format!("viewer state task failed: {error}")),
2213 }
2214}
2215
2216struct ProcessCancellationGuard(Arc<AtomicBool>);
2218
2219impl Drop for ProcessCancellationGuard {
2220 fn drop(&mut self) {
2221 self.0.store(true, Ordering::Release);
2222 }
2223}
2224
2225#[cfg(test)]
2233fn run_new_preflight(
2234 config: Config,
2235 bundle_id: String,
2236 target_id: String,
2237 project_directory: Option<PathBuf>,
2238) -> Result<crate::server::PreflightNew> {
2239 run_new_preflight_with_cancellation(
2240 config,
2241 bundle_id,
2242 target_id,
2243 project_directory,
2244 Arc::new(AtomicBool::new(false)),
2245 Vec::new(),
2246 )
2247}
2248
2249fn run_new_preflight_with_cancellation(
2250 config: Config,
2251 bundle_id: String,
2252 target_id: String,
2253 project_directory: Option<PathBuf>,
2254 cancelled: Arc<AtomicBool>,
2255 remote_repairs: Vec<mj_core::local_git::LocalRemoteRepair>,
2256) -> Result<crate::server::PreflightNew> {
2257 let executor =
2258 CancellableProcessExecutor::new(cancelled).with_deadline(Duration::from_secs(30));
2259 if !remote_repairs.is_empty() {
2260 let target = config.targets.get(&target_id).context("unknown target")?;
2261 anyhow::ensure!(
2262 !is_bare_project_target(target) && project_directory.is_none(),
2263 "remote repair requires an isolated target"
2264 );
2265 let bundle = config.bundles.get(&bundle_id).context("unknown bundle")?;
2266 mj_core::local_git::apply_repository_remote_repairs(bundle, &remote_repairs, &executor)?;
2267 }
2268 run_new_preflight_with_executor(config, bundle_id, target_id, project_directory, &executor)
2269}
2270
2271fn run_new_preflight_with_executor(
2272 config: Config,
2273 bundle_id: String,
2274 target_id: String,
2275 project_directory: Option<PathBuf>,
2276 executor: &impl CommandExecutor,
2277) -> Result<crate::server::PreflightNew> {
2278 let target_is_bare = config
2279 .targets
2280 .get(&target_id)
2281 .with_context(|| format!("unknown target template {target_id:?}"))
2282 .map(is_bare_project_target)?;
2283 if target_is_bare {
2284 let directory =
2285 project_directory.context("project directory is required for a bare target")?;
2286 let controller = config_only_controller(config);
2287 let directory = controller.resolve_project_directory(&target_id, &directory, executor)?;
2288 let managed_worktree =
2289 controller.managed_worktree_options(&target_id, &directory, executor)?;
2290 return Ok(crate::server::PreflightNew {
2291 managed_worktree,
2292 project_directory: Some(directory),
2293 remote_repairs: Vec::new(),
2294 dirty_repositories: Vec::new(),
2295 remote_repositories: Vec::new(),
2296 local_changes_excluded: false,
2297 });
2298 }
2299 if project_directory.is_some() {
2300 bail!("project directory is unsupported for this target");
2301 }
2302
2303 let bundle = config.bundles.get(&bundle_id).context("unknown bundle")?;
2304 let repairs = mj_core::local_git::repository_remote_repairs(bundle, executor)?;
2305 if !repairs.is_empty() {
2306 return Ok(crate::server::PreflightNew {
2307 managed_worktree: Default::default(),
2308 project_directory: None,
2309 remote_repairs: repairs,
2310 dirty_repositories: Vec::new(),
2311 remote_repositories: Vec::new(),
2312 local_changes_excluded: true,
2313 });
2314 }
2315 let remote_repositories = bundle
2316 .repositories
2317 .iter()
2318 .map(|repository| {
2319 let source = resolve_repository(repository, executor)
2320 .with_context(|| format!("repository {:?}", repository.id))?;
2321 let default_branch = default_branch(&source, executor)
2322 .with_context(|| format!("repository {:?}", repository.id))?;
2323 Ok(crate::server::PreflightRepository {
2324 id: repository.id.clone(),
2325 fetch_url: display_url(&source.fetch_url),
2326 default_branch,
2327 push_urls: source
2328 .push_urls
2329 .iter()
2330 .map(|url| display_url(url))
2331 .collect(),
2332 })
2333 })
2334 .collect::<Result<Vec<_>>>()?;
2335 Ok(crate::server::PreflightNew {
2336 managed_worktree: Default::default(),
2337 project_directory: None,
2338 remote_repairs: Vec::new(),
2339 dirty_repositories: Vec::new(),
2340 remote_repositories,
2341 local_changes_excluded: true,
2342 })
2343}
2344
2345fn phone_action_capacity_available(active_actions: usize) -> bool {
2346 active_actions < MAX_CONCURRENT_PHONE_ACTIONS
2347}
2348
2349fn republish_quota_profiles(
2359 controller: &Controller,
2360 published: &mut std::collections::BTreeMap<String, HarnessProfile>,
2361 batch: &mut QuotaRefreshBatch,
2362 profiles_tx: &tokio::sync::watch::Sender<QuotaRefreshBatch>,
2363) -> bool {
2364 if *published == controller.config.profiles {
2365 return false;
2366 }
2367 published.clone_from(&controller.config.profiles);
2368 batch.generation = batch.generation.saturating_add(1);
2369 batch.profiles = quota_refresh_profiles(controller);
2370 profiles_tx.send_replace(batch.clone());
2371 true
2372}
2373
2374fn request_phone_action_cancellation(
2375 session_id: &str,
2376 action_sessions: &std::collections::BTreeMap<u64, String>,
2377 action_cancellations: &std::collections::BTreeMap<u64, PhoneActionControl>,
2378) -> bool {
2379 let control = action_sessions
2380 .iter()
2381 .find_map(|(action_id, active_session_id)| {
2382 (active_session_id == session_id)
2383 .then(|| action_cancellations.get(action_id))
2384 .flatten()
2385 });
2386 if let Some(control) = control {
2387 return control.request_cancel();
2388 }
2389 false
2390}
2391
2392fn track_started_phone_session(
2393 state: &mut State,
2394 active_actions: &mut std::collections::BTreeSet<String>,
2395 action_sessions: &mut std::collections::BTreeMap<u64, String>,
2396 action_id: u64,
2397 session: SessionRecord,
2398) -> std::result::Result<(), String> {
2399 let session_id = session.id.clone();
2400 if !active_actions.insert(session_id.clone()) {
2401 return Err("another operation is already running for the new session".into());
2402 }
2403 action_sessions.insert(action_id, session_id.clone());
2404 state.sessions.insert(session_id, session);
2405 Ok(())
2406}
2407
2408fn record_action_result(
2418 pending_action_errors: &mut std::collections::BTreeMap<String, String>,
2419 session_id: Option<&str>,
2420 result: &std::result::Result<(), String>,
2421) {
2422 let Some(session_id) = session_id else {
2423 return;
2424 };
2425 match result {
2426 Err(error) => {
2427 pending_action_errors.insert(session_id.to_owned(), error.clone());
2428 }
2429 Ok(()) => {
2430 pending_action_errors.remove(session_id);
2431 }
2432 }
2433}
2434
2435fn record_launch_failure(
2438 failures: &mut Vec<crate::server::ViewerLaunchFailure>,
2439 action_id: u64,
2440 workspace_id: String,
2441 session_id: Option<String>,
2442) {
2443 failures.push(crate::server::ViewerLaunchFailure {
2444 id: format!("{}-{action_id}", std::process::id()),
2445 workspace_id,
2446 session_id,
2447 });
2448 if failures.len() > 16 {
2449 failures.remove(0);
2450 }
2451}
2452
2453struct PhoneActionServices<'a> {
2454 sessions: &'a SessionManagerControl,
2455 daemon_runtime: &'a Arc<RuntimeState>,
2456}
2457
2458async fn apply_phone_action(
2459 controller: &mut Controller,
2460 services: PhoneActionServices<'_>,
2461 action: ControllerAction,
2462 _executor: &(impl CommandExecutor + Sync),
2463 action_id: u64,
2464 started: &tokio::sync::mpsc::UnboundedSender<PhoneActionStarted>,
2465 control: &PhoneActionControl,
2466) -> Result<()> {
2467 match action {
2468 ControllerAction::New {
2469 workspace_id,
2470 profile_id,
2471 bundle_id,
2472 target_id,
2473 title,
2474 project_directory,
2475 create_managed_worktree,
2476 dirty_ack: _dirty_ack,
2477 } => {
2478 let workspace_id = if workspace_id.is_empty() {
2479 let workspaces = crate::database::list_workspaces()?;
2480 match workspaces.as_slice() {
2481 [workspace] => workspace.id.clone(),
2482 [] => bail!("create a workspace before starting a phone session"),
2483 _ => bail!("phone session creation requires a workspace_id"),
2484 }
2485 } else {
2486 workspace_id
2487 };
2488 let title = title.unwrap_or_else(|| {
2492 let project = project_directory
2493 .as_ref()
2494 .and_then(|path| path.file_name())
2495 .map(|name| name.to_string_lossy().into_owned())
2496 .unwrap_or_else(|| bundle_id.clone());
2497 format!("{project} via {profile_id}")
2498 });
2499 let session_title_override = Some(title.clone());
2500 let allow_dirty_local = false;
2504 let (published, publication) = tokio::sync::oneshot::channel();
2505 let registered = services
2506 .daemon_runtime
2507 .start_create_session_controlled(
2508 CreateSessionRequest {
2509 create_managed_worktree,
2510 initial_prompt: None,
2511 workspace_id,
2512 profile_id,
2513 bundle_id,
2514 project_directory,
2515 target_template_id: target_id,
2516 additional_mounts: Vec::new(),
2517 allow_dirty_local,
2518 resource_allocation: None,
2519 title,
2520 session_title_override,
2521 },
2522 control
2523 .create
2524 .clone()
2525 .expect("New action has a daemon create control"),
2526 publication,
2527 )
2528 .await?;
2529 let registered_session_id = registered.session.id.clone();
2530 started
2531 .send(PhoneActionStarted {
2532 action_id,
2533 session: registered.session,
2534 published,
2535 })
2536 .map_err(|_| anyhow::anyhow!("phone server stopped before publishing session"))?;
2537 services
2538 .daemon_runtime
2539 .wait_create_session(®istered_session_id)
2540 .await
2541 }
2542 ControllerAction::Prompt {
2543 session_id,
2544 text,
2545 images,
2546 } => {
2547 services
2548 .sessions
2549 .wait_for_session(&session_id, Duration::from_secs(5))
2550 .await?
2551 .submit(
2552 new_command_id("phone-prompt")?,
2553 RelayCommand::Prompt {
2554 prompt: phone_prompt_blocks(text, images),
2555 },
2556 )
2557 .await?;
2558 Ok(())
2559 }
2560 ControllerAction::RunShell {
2561 session_id,
2562 command,
2563 } => {
2564 services
2565 .sessions
2566 .wait_for_session(&session_id, Duration::from_secs(5))
2567 .await?
2568 .submit(
2569 new_command_id("phone-shell")?,
2570 RelayCommand::RunUserShell { command },
2571 )
2572 .await?;
2573 Ok(())
2574 }
2575 ControllerAction::CancelShell {
2576 session_id,
2577 shell_command_id,
2578 } => {
2579 services
2580 .sessions
2581 .wait_for_session(&session_id, Duration::from_secs(5))
2582 .await?
2583 .submit(
2584 new_command_id("phone-cancel-shell")?,
2585 RelayCommand::CancelUserShell { shell_command_id },
2586 )
2587 .await?;
2588 Ok(())
2589 }
2590 ControllerAction::Close { session_id } => {
2591 services.daemon_runtime.close_session(session_id).await
2592 }
2593 ControllerAction::Resume {
2594 session_id,
2595 workspace_id,
2596 profile_id,
2597 target_id,
2598 queue,
2599 additional_mounts,
2600 resource_allocation,
2601 } => services
2602 .daemon_runtime
2603 .resume_session(ResumeSessionRequest {
2604 session_id,
2605 workspace_id,
2606 profile_id,
2607 target_template_id: target_id,
2608 additional_mounts,
2609 resource_allocation,
2610 discard_queue: queue == ResumeQueueDisposition::Discard,
2611 repository_preflight: None,
2612 })
2613 .await
2614 .map(|_| ()),
2615 ControllerAction::Move { request } => {
2616 let outcome = services.daemon_runtime.move_session(request).await?;
2617 match outcome.outcome.as_str() {
2618 "completed" | "unchanged" => Ok(()),
2619 "cancelled" | "failed" => {
2620 let recovery = outcome.recovery.unwrap_or_else(|| {
2625 "Inspect the session and retry Move or Resume with previous settings."
2626 .into()
2627 });
2628 bail!("Move {}: {recovery}", outcome.outcome)
2629 }
2630 status => bail!("Move returned an unknown outcome: {status}"),
2631 }
2632 }
2633 ControllerAction::Open { .. } => Ok(()),
2634 ControllerAction::Cancel { .. } => {
2635 bail!("cancel actions must be handled by the phone control loop")
2636 }
2637 ControllerAction::RemoveQueuedPrompt {
2638 session_id,
2639 queue_id,
2640 } => {
2641 services
2642 .sessions
2643 .session(&session_id)
2644 .await?
2645 .submit(
2646 new_command_id("phone-remove-prompt")?,
2647 RelayCommand::RemoveQueuedPrompt {
2648 queued_command_id: queue_id,
2649 },
2650 )
2651 .await?;
2652 Ok(())
2653 }
2654 ControllerAction::RespondElicitation {
2655 session_id,
2656 elicitation_id,
2657 response,
2658 } => {
2659 services
2660 .sessions
2661 .session(&session_id)
2662 .await?
2663 .respond_elicitation(elicitation_id, response)
2664 .await
2665 }
2666 ControllerAction::Rename { session_id, title } => {
2667 controller.rename_session(&session_id, &title)?;
2668 Ok(())
2669 }
2670 ControllerAction::StartReview { session_id } => {
2671 services
2675 .daemon_runtime
2676 .review_host()
2677 .start(&session_id, true)
2678 .await
2679 .map_err(|refusal| anyhow::anyhow!("{refusal}"))?;
2680 Ok(())
2681 }
2682 ControllerAction::ResolveReview {
2683 session_id,
2684 resolution,
2685 } => {
2686 let resolution = crate::server::resolution_from_name(&resolution)
2687 .context("a review is resolved by forward, dismiss, or cancel")?;
2688 services
2689 .daemon_runtime
2690 .review_host()
2691 .resolve(&session_id, resolution)
2692 .await
2693 .map_err(|error| anyhow::anyhow!("{error}"))?;
2694 Ok(())
2695 }
2696 ControllerAction::CancelTurn { session_id } => {
2697 services
2698 .sessions
2699 .session(&session_id)
2700 .await?
2701 .submit(new_command_id("phone-cancel-turn")?, RelayCommand::Cancel)
2702 .await?;
2703 Ok(())
2704 }
2705 ControllerAction::SetConfig {
2706 session_id,
2707 key,
2708 value,
2709 } => {
2710 services
2711 .sessions
2712 .session(&session_id)
2713 .await?
2714 .submit(
2715 new_command_id("phone-set-config")?,
2716 RelayCommand::SetConfig { key, value },
2717 )
2718 .await?;
2719 Ok(())
2720 }
2721 ControllerAction::SetPlanMode { session_id, active } => {
2722 let harness_kind = controller
2726 .state
2727 .sessions
2728 .get(&session_id)
2729 .with_context(|| format!("unknown session {session_id}"))?
2730 .harness_kind;
2731 let handle = services.sessions.session(&session_id).await?;
2732 let operational = handle
2733 .view()
2734 .snapshot
2735 .map(|snapshot| snapshot.operational)
2736 .context("the session has not reported what it supports yet")?;
2737 let facts = mj_core::acp::AcpSessionFacts::from_operational(
2738 harness_kind,
2739 &operational.config,
2740 &operational.config_options,
2741 operational.modes.as_ref(),
2742 );
2743 let command = match facts.plan_control(active) {
2744 Ok(mj_core::acp::PlanControl::SetConfig { key, value }) => {
2745 RelayCommand::SetConfig { key, value }
2746 }
2747 Ok(mj_core::acp::PlanControl::SetSessionMode { mode_id }) => {
2748 RelayCommand::SetSessionMode { mode_id }
2749 }
2750 Err(reason) => bail!("{reason}"),
2751 };
2752 handle
2753 .submit(new_command_id("phone-plan-mode")?, command)
2754 .await?;
2755 Ok(())
2756 }
2757 ControllerAction::RefreshQuota { .. } | ControllerAction::RefreshCapacity { .. } => {
2760 bail!("refresh actions must be handled by the phone control loop")
2761 }
2762 }
2763}
2764
2765#[derive(Clone, PartialEq, Eq)]
2767struct ProjectSourceKey {
2768 directory: Option<PathBuf>,
2769 worktree: Option<mj_core::state::ManagedWorktree>,
2770 target: Option<mj_core::config::TargetTemplate>,
2771 fallback: ProjectSourceIdentity,
2772}
2773
2774impl ProjectSourceKey {
2775 fn of(session: &SessionRecord, config: &Config) -> Self {
2776 Self {
2777 directory: session.project_directory.clone(),
2778 worktree: session.managed_worktree.clone(),
2779 target: config.targets.get(&session.target_template_id).cloned(),
2780 fallback: session.project_source(config),
2781 }
2782 }
2783}
2784
2785struct ProjectSourceEntry {
2786 key: ProjectSourceKey,
2787 source: Option<ProjectSourceIdentity>,
2788 retry_at: Option<Instant>,
2789 cancelled: Arc<AtomicBool>,
2790}
2791
2792struct ProjectSourceResolved {
2793 cancelled: Arc<AtomicBool>,
2794 session_id: String,
2795 key: ProjectSourceKey,
2796 result: Result<ProjectSourceIdentity, String>,
2797}
2798
2799#[derive(Default)]
2802struct PhoneProjectSources {
2803 entries: std::collections::BTreeMap<String, ProjectSourceEntry>,
2804 jobs: tokio::task::JoinSet<ProjectSourceResolved>,
2805}
2806
2807impl PhoneProjectSources {
2808 fn synchronize(&mut self, controller: &Controller) {
2809 self.entries.retain(|id, entry| {
2810 let keep = controller.state.sessions.get(id).is_some_and(|session| {
2811 session.project_directory.is_some()
2812 && entry.key == ProjectSourceKey::of(session, &controller.config)
2813 });
2814 if !keep {
2815 entry.cancelled.store(true, Ordering::Release);
2816 }
2817 keep
2818 });
2819 for session in controller.state.sessions.values() {
2820 if self.jobs.len() >= 8 {
2821 break;
2822 }
2823 if session.project_directory.is_none()
2824 || self.entries.get(&session.id).is_some_and(|entry| {
2825 entry
2826 .retry_at
2827 .is_none_or(|deadline| Instant::now() < deadline)
2828 })
2829 {
2830 continue;
2831 }
2832 let key = ProjectSourceKey::of(session, &controller.config);
2833 let cancelled = Arc::new(AtomicBool::new(false));
2834 self.entries.insert(
2835 session.id.clone(),
2836 ProjectSourceEntry {
2837 key: key.clone(),
2838 source: None,
2839 retry_at: None,
2840 cancelled: cancelled.clone(),
2841 },
2842 );
2843 let source_controller = Controller {
2844 config: controller.config.clone(),
2845 state: State {
2846 sessions: [(session.id.clone(), session.clone())]
2847 .into_iter()
2848 .collect(),
2849 ..State::default()
2850 },
2851 };
2852 let session_id = session.id.clone();
2853 self.jobs.spawn_blocking(move || {
2854 let executor = CancellableProcessExecutor::new(cancelled.clone())
2855 .with_deadline(Duration::from_secs(8));
2856 let result = source_controller
2857 .resolve_session_project_source(&session_id, &executor)
2858 .map_err(|error| format!("{error:#}"));
2859 ProjectSourceResolved {
2860 cancelled,
2861 session_id,
2862 key,
2863 result,
2864 }
2865 });
2866 }
2867 }
2868
2869 fn complete(&mut self, resolved: ProjectSourceResolved) {
2870 let Some(entry) = self.entries.get_mut(&resolved.session_id) else {
2871 return;
2872 };
2873 if entry.key != resolved.key || !Arc::ptr_eq(&entry.cancelled, &resolved.cancelled) {
2874 return;
2875 }
2876 match resolved.result {
2877 Ok(source) => entry.source = Some(source),
2878 Err(error) => {
2879 tracing::warn!(session_id = %resolved.session_id, %error, "could not resolve web project source");
2880 entry.retry_at = Some(Instant::now() + Duration::from_secs(30));
2881 }
2882 }
2883 }
2884
2885 fn source(&self, session: &SessionRecord, config: &Config) -> Option<&ProjectSourceIdentity> {
2886 self.entries
2887 .get(&session.id)
2888 .filter(|entry| entry.key == ProjectSourceKey::of(session, config))
2889 .and_then(|entry| entry.source.as_ref())
2890 }
2891}
2892
2893impl Drop for PhoneProjectSources {
2894 fn drop(&mut self) {
2895 for entry in self.entries.values() {
2896 entry.cancelled.store(true, Ordering::Release);
2897 }
2898 }
2899}
2900
2901struct PhoneSessionViews<'a> {
2905 conversations: &'a std::collections::BTreeMap<String, crate::server::BrowserTranscript>,
2906 queued_prompts: &'a std::collections::BTreeMap<String, Vec<mj_core::relay::QueuedPrompt>>,
2907 active_user_shells:
2908 &'a std::collections::BTreeMap<String, Vec<mj_core::relay::ActiveUserShell>>,
2909 pending_elicitations:
2910 &'a std::collections::BTreeMap<String, Vec<mj_core::elicitation::ElicitationRequest>>,
2911 prompt_images: &'a std::collections::BTreeSet<String>,
2913 operational: &'a std::collections::BTreeMap<String, mj_core::relay::RelayOperationalState>,
2917 materialized_activity: &'a std::collections::BTreeMap<String, Option<i64>>,
2921 project_sources: &'a PhoneProjectSources,
2922 operations: &'a std::collections::BTreeMap<String, crate::server::ViewerOperation>,
2924 move_recoveries: &'a std::collections::BTreeMap<String, ViewerMoveRecovery>,
2926 capacity: &'a [crate::server::ViewerTargetCapacity],
2928 launch_failures: &'a [crate::server::ViewerLaunchFailure],
2929 reviews: &'a std::collections::BTreeMap<String, crate::review_host::RuntimeReviewView>,
2932}
2933
2934#[derive(Debug, Clone)]
2940struct PhoneCapacity {
2941 target: crate::targets::DeploymentCapacityTarget,
2942 usage: Option<crate::targets::DeploymentCapacityUsage>,
2943 on_demand: bool,
2944 sampled_at_epoch_seconds: Option<u64>,
2945 refreshing: bool,
2946 failed: bool,
2947}
2948
2949const CAPACITY_STALE_AFTER: Duration = Duration::from_secs(120);
2951
2952fn publish_capacity_targets(
2954 controller: &Controller,
2955 targets_tx: &tokio::sync::watch::Sender<Vec<crate::targets::DeploymentCapacityTarget>>,
2956 state: &mut std::collections::BTreeMap<String, PhoneCapacity>,
2957) {
2958 let targets = controller.deployment_capacity_targets();
2959 state.retain(|id, _| targets.iter().any(|target| target.id == *id));
2960 for target in &targets {
2961 state
2962 .entry(target.id.clone())
2963 .and_modify(|entry| entry.target = target.clone())
2964 .or_insert_with(|| PhoneCapacity {
2965 target: target.clone(),
2966 usage: None,
2967 on_demand: false,
2968 sampled_at_epoch_seconds: None,
2969 refreshing: true,
2971 failed: false,
2972 });
2973 }
2974 if targets_tx.borrow().as_slice() != targets.as_slice() {
2975 targets_tx.send_replace(targets);
2976 }
2977}
2978
2979fn viewer_capacity(
2981 state: &std::collections::BTreeMap<String, PhoneCapacity>,
2982) -> Vec<crate::server::ViewerTargetCapacity> {
2983 let now = std::time::SystemTime::now()
2984 .duration_since(std::time::UNIX_EPOCH)
2985 .unwrap_or_default()
2986 .as_secs();
2987 state
2988 .values()
2989 .map(|entry| {
2990 let usage = entry.usage.as_ref();
2991 crate::server::ViewerTargetCapacity {
2992 id: entry.target.id.clone(),
2993 label: entry.target.host.clone(),
2994 target_ids: entry.target.target_ids.clone(),
2995 cpu_percent: usage.and_then(|usage| usage.cpu_percent),
2996 memory_used_bytes: usage.map(|usage| usage.memory_used_bytes),
2997 memory_total_bytes: usage.map(|usage| usage.memory_total_bytes),
2998 logical_cores: usage.map(|usage| usage.logical_cores),
2999 disk_total_bytes: usage.and_then(|usage| usage.disk_total_bytes),
3000 virtual_machines: matches!(
3003 entry.target.kind,
3004 crate::targets::DeploymentCapacityKind::AwsFleet
3005 )
3006 .then(|| u64::from(!entry.on_demand)),
3007 sampled_at_epoch_seconds: entry.sampled_at_epoch_seconds,
3008 refreshing: entry.refreshing,
3009 stale: entry.sampled_at_epoch_seconds.is_some_and(|sampled| {
3010 now.saturating_sub(sampled) > CAPACITY_STALE_AFTER.as_secs()
3011 }),
3012 has_error: entry.failed,
3013 }
3014 })
3015 .collect()
3016}
3017
3018fn viewer_operation(view: &crate::daemon::RuntimeLifecycleView) -> crate::server::ViewerOperation {
3020 use crate::server::{ViewerOperationKind, ViewerOperationStage};
3021
3022 crate::server::ViewerOperation {
3023 id: view.operation_id.clone(),
3027 session_id: view.session_id.clone(),
3028 kind: match view.kind {
3029 crate::daemon::RuntimeLifecycleKind::Create => ViewerOperationKind::Create,
3030 crate::daemon::RuntimeLifecycleKind::Resume => ViewerOperationKind::Resume,
3031 crate::daemon::RuntimeLifecycleKind::Move => ViewerOperationKind::Move,
3032 crate::daemon::RuntimeLifecycleKind::Close
3035 | crate::daemon::RuntimeLifecycleKind::ForceStop => ViewerOperationKind::Stop,
3036 crate::daemon::RuntimeLifecycleKind::DestroyStopped
3037 | crate::daemon::RuntimeLifecycleKind::ForceDestroy => ViewerOperationKind::Destroy,
3038 crate::daemon::RuntimeLifecycleKind::Cleanup => ViewerOperationKind::Cleanup,
3039 },
3040 started_at_epoch_seconds: view.started_at_epoch_seconds,
3041 stages: view
3042 .active_stages
3043 .iter()
3044 .map(|(stage, started_at)| ViewerOperationStage {
3045 label: stage.label(),
3046 started_at_epoch_seconds: *started_at,
3047 })
3048 .collect(),
3049 notice: view.notice.clone(),
3050 cancellable: view.cancellable,
3051 }
3052}
3053
3054fn session_capabilities(
3060 session: &crate::server::ViewerSession,
3061 operational: Option<&mj_core::relay::RelayOperationalState>,
3062 operation: Option<&crate::server::ViewerOperation>,
3063 facts: Option<&mj_core::acp::AcpSessionFacts>,
3064) -> crate::server::ViewerSessionCapabilities {
3065 use crate::server::ViewerLifecycleCategory;
3066
3067 let live = session.lifecycle == ViewerLifecycleCategory::Live;
3068 let attached = operational.is_some();
3071 let transition_busy = session.transitioning && !session.has_error;
3076 let busy = operation.is_some() || transition_busy;
3077 let partial_move_queue = session.move_recovery.as_ref().is_some_and(|recovery| {
3081 recovery.queue_admission_started && !recovery.queue_admission_finished
3082 });
3083 let mutation_busy = busy || partial_move_queue;
3084 let idle = operational
3085 .is_some_and(|state| state.execution == mj_core::relay::RelayExecutionState::Idle);
3086 crate::server::ViewerSessionCapabilities {
3087 open: session.conversation_available
3088 && !session.transitioning
3089 && session.lifecycle == ViewerLifecycleCategory::Live,
3090 prompt: live && attached && !mutation_busy,
3091 run_shell: live && attached && !mutation_busy,
3092 cancel_turn: live
3093 && !mutation_busy
3094 && operational.is_some_and(|state| {
3095 state.active_prompt.is_some() || state.capacity_retry.is_some()
3096 }),
3097 cancel_operation: operation.is_some_and(|operation| operation.cancellable),
3098 stop: session.lifecycle.is_dashboard_visible() && !mutation_busy,
3101 rename: !session.transitioning,
3102 resume: !session.lifecycle.is_dashboard_visible() && !mutation_busy,
3103 move_session: live && !busy,
3104 set_config: live && attached && facts.is_some() && !mutation_busy,
3105 set_plan_mode: live
3108 && !mutation_busy
3109 && idle
3110 && facts.is_some_and(mj_core::acp::AcpSessionFacts::supports_plan_mode),
3111 }
3112}
3113
3114fn phone_prompt_blocks(
3117 text: String,
3118 images: Vec<crate::server::ViewerPromptImage>,
3119) -> Vec<agent_client_protocol::schema::v1::ContentBlock> {
3120 use agent_client_protocol::schema::v1::{ContentBlock, ImageContent, TextContent};
3121
3122 let mut prompt = Vec::with_capacity(images.len() + 1);
3123 if !text.is_empty() {
3124 prompt.push(ContentBlock::Text(TextContent::new(text)));
3125 }
3126 prompt.extend(images.into_iter().map(|image| match image.attachment {
3127 Some(reference) => reference.content_block(),
3128 None => ContentBlock::Image(ImageContent::new(image.data_base64, image.mime_type)),
3129 }));
3130 prompt
3131}
3132
3133fn phone_commands(
3139 session: &crate::server::ViewerSession,
3140 operational: Option<&mj_core::relay::RelayOperationalState>,
3141) -> Vec<crate::server::ViewerMjCommand> {
3142 use crate::server::ViewerCommandSource;
3143 use agent_client_protocol::schema::v1::AvailableCommandInput;
3144
3145 let command =
3146 |name: &str, description: &str, argument: Option<&str>| crate::server::ViewerMjCommand {
3147 name: name.to_owned(),
3148 description: description.to_owned(),
3149 source: ViewerCommandSource::Mj,
3150 argument: argument.map(str::to_owned),
3151 };
3152 let mut commands = vec![
3153 command("help", "show available Mjolnir and agent commands", None),
3154 command(
3155 "detach",
3156 "leave the conversation without stopping the worker",
3157 None,
3158 ),
3159 ];
3160 let option = |key: &str| {
3161 session
3162 .config_options
3163 .iter()
3164 .any(|option| option.key == key)
3165 };
3166 if option("model") {
3167 commands.push(command("model", "change the active model", Some("value")));
3168 commands.push(command("fast", "toggle Codex Fast mode", None));
3169 }
3170 if option("effort") {
3171 commands.push(command(
3172 "effort",
3173 "change the active reasoning effort",
3174 Some("value"),
3175 ));
3176 }
3177 if session.plan_mode_active.is_some() && session.capabilities.set_plan_mode {
3178 commands.push(command("plan", "toggle plan mode", Some("message")));
3179 commands.push(command(
3180 "implement",
3181 "leave plan mode and implement",
3182 Some("instruction"),
3183 ));
3184 }
3185 if session.capabilities.prompt || session.turn_review.is_some() {
3186 commands.push(command(
3187 "review",
3188 "review the finished turn now, or report how review is configured",
3189 Some("status"),
3190 ));
3191 }
3192 let reserved = [
3196 "help",
3197 "detach",
3198 "model",
3199 "fast",
3200 "effort",
3201 "plan",
3202 "implement",
3203 "review",
3204 ];
3205 for advertised in operational
3206 .into_iter()
3207 .flat_map(|state| state.available_commands.iter())
3208 {
3209 let name = advertised.name.trim();
3210 if name.is_empty()
3211 || reserved
3212 .iter()
3213 .any(|local| name.eq_ignore_ascii_case(local))
3214 || commands
3215 .iter()
3216 .any(|existing| existing.name.eq_ignore_ascii_case(name))
3217 {
3218 continue;
3219 }
3220 let argument = advertised.input.as_ref().and_then(|input| match input {
3221 AvailableCommandInput::Unstructured(input) => {
3222 let hint = input.hint.trim();
3223 (!hint.is_empty()).then(|| hint.to_owned())
3224 }
3225 _ => None,
3226 });
3227 commands.push(crate::server::ViewerMjCommand {
3228 name: name.to_owned(),
3229 description: advertised.description.trim().to_owned(),
3230 source: ViewerCommandSource::Agent,
3231 argument,
3232 });
3233 }
3234 commands
3235}
3236
3237fn review_views(
3239 daemon_runtime: &Arc<RuntimeState>,
3240) -> std::collections::BTreeMap<String, crate::review_host::RuntimeReviewView> {
3241 daemon_runtime
3242 .review_host()
3243 .views()
3244 .into_iter()
3245 .map(|review| (review.session_id.clone(), review))
3246 .collect()
3247}
3248
3249type ViewerMoveRecoveries = std::collections::BTreeMap<String, crate::server::ViewerMoveRecovery>;
3250
3251fn request_move_recovery_reload(
3255 completed: &tokio::sync::mpsc::UnboundedSender<Result<ViewerMoveRecoveries, String>>,
3256 in_flight: &mut bool,
3257 jobs: &mut tokio::task::JoinSet<()>,
3258) {
3259 if *in_flight {
3260 return;
3261 }
3262 *in_flight = true;
3263 let completed = completed.clone();
3264 jobs.spawn(async move {
3265 let result = match tokio::task::spawn_blocking(|| {
3266 crate::database::load_move_operations()
3267 .map(|operations| {
3268 operations
3269 .into_iter()
3270 .filter_map(|operation| {
3271 crate::server::ViewerMoveRecovery::from_operation(&operation)
3272 .map(|recovery| (operation.selection.session_id.clone(), recovery))
3273 })
3274 .collect()
3275 })
3276 .map_err(|error| format!("{error:#}"))
3277 })
3278 .await
3279 {
3280 Ok(result) => result,
3281 Err(error) => Err(format!("move recovery projection task failed: {error}")),
3282 };
3283 let _ = completed.send(result);
3284 });
3285}
3286
3287fn viewer_snapshot(
3288 controller: &Controller,
3289 workspaces: &[mj_core::workspace::WorkspaceRecord],
3290 quotas: &std::collections::BTreeMap<String, ProfileQuota>,
3291 views: &PhoneSessionViews<'_>,
3292 revision: u64,
3293) -> ViewerSnapshot {
3294 let PhoneSessionViews {
3295 conversations,
3296 reviews,
3297 queued_prompts,
3298 active_user_shells,
3299 pending_elicitations,
3300 prompt_images,
3301 operational,
3302 materialized_activity,
3303 project_sources,
3304 operations,
3305 move_recoveries,
3306 capacity,
3307 launch_failures,
3308 } = views;
3309 let mut snapshot =
3310 ViewerSnapshot::from_config_state(&controller.config, &controller.state, revision);
3311 snapshot.launch_failures = launch_failures.to_vec();
3312 snapshot.workspaces = workspaces
3313 .iter()
3314 .map(|workspace| crate::server::ViewerWorkspace {
3315 id: workspace.id.clone(),
3316 name: workspace.name.clone(),
3317 })
3318 .collect();
3319 let now = std::time::SystemTime::now()
3320 .duration_since(std::time::UNIX_EPOCH)
3321 .unwrap_or_default()
3322 .as_secs();
3323 for profile in &mut snapshot.profiles {
3324 let Some(quota) = quotas.get(&profile.id) else {
3325 continue;
3326 };
3327 profile.quota = Some(ViewerQuota {
3328 summary: quota.compact(),
3329 windows: quota
3330 .windows
3331 .iter()
3332 .map(|window| crate::server::ViewerQuotaWindow {
3333 label: window.label.clone(),
3334 percent_used: window
3337 .remaining_percent
3338 .map(|left| 100_u8.saturating_sub(left)),
3339 resets_at: window.resets.clone(),
3340 projects_exhaustion_before_reset: crate::quota::projects_exhaustion(
3341 window,
3342 quota.refreshed_at_epoch_seconds,
3343 ),
3344 })
3345 .collect(),
3346 resets_at: quota
3347 .windows
3348 .iter()
3349 .find_map(|window| window.resets.clone()),
3350 stale: now.saturating_sub(quota.refreshed_at_epoch_seconds)
3351 > QUOTA_STALE_AFTER.as_secs(),
3352 refreshed_at_epoch_seconds: quota.refreshed_at_epoch_seconds,
3353 has_error: quota.error.is_some(),
3354 });
3355 }
3356 for session in &mut snapshot.sessions {
3357 session.move_recovery = move_recoveries.get(&session.id).cloned();
3358 if let Some(record) = controller.state.sessions.get(&session.id)
3359 && let Some(source) = project_sources.source(record, &controller.config)
3360 {
3361 session.set_project_source(source);
3362 }
3363 session.last_activity_at_ms = materialized_activity.get(&session.id).copied().flatten();
3364 session.queued_prompts = queued_prompts
3365 .get(&session.id)
3366 .into_iter()
3367 .flatten()
3368 .map(|prompt| ViewerQueuedPrompt {
3369 id: prompt.id.clone(),
3370 text: prompt.text.clone(),
3371 created_at: prompt.created_at_ms.to_string(),
3372 })
3373 .collect();
3374 session.active_user_shells = active_user_shells
3375 .get(&session.id)
3376 .into_iter()
3377 .flatten()
3378 .map(|shell| ViewerUserShell {
3379 id: shell.command_id.clone(),
3380 command: shell.command.clone(),
3381 started_at_ms: shell.started_at_ms,
3382 })
3383 .collect();
3384 session.background_tasks = operational
3385 .get(&session.id)
3386 .map(|state| {
3387 state
3388 .background_commands
3389 .iter()
3390 .map(|task| ViewerBackgroundTask {
3391 id: task.id.clone(),
3392 command: task.command.clone(),
3393 started_at_ms: task.started_at_ms,
3394 can_stop: task.can_stop,
3395 })
3396 .collect()
3397 })
3398 .unwrap_or_default();
3399 session.pending_elicitations = pending_elicitations
3400 .get(&session.id)
3401 .cloned()
3402 .unwrap_or_default();
3403 session.prompt_images_supported = prompt_images.contains(&session.id);
3404 session.operation = operations.get(&session.id).cloned();
3405 if let Some(operation) = session.operation.as_ref()
3410 && operation.kind.transition_kind().is_some()
3411 {
3412 session.transitioning = true;
3413 }
3414 let live = operational.get(&session.id);
3415 let facts = live.map(|state| {
3416 mj_core::acp::AcpSessionFacts::from_operational(
3417 controller
3418 .state
3419 .sessions
3420 .get(&session.id)
3421 .map_or(mj_core::config::HarnessKind::Codex, |record| {
3422 record.harness_kind
3423 }),
3424 &state.config,
3425 &state.config_options,
3426 state.modes.as_ref(),
3427 )
3428 });
3429 if let Some(state) = live {
3430 session.latest_event_ordinal = state.latest_ordinal;
3431 session.chat_phase = match state.execution {
3432 mj_core::relay::RelayExecutionState::Idle => crate::server::ViewerChatPhase::Idle,
3433 mj_core::relay::RelayExecutionState::Running => {
3434 crate::server::ViewerChatPhase::Running
3435 }
3436 mj_core::relay::RelayExecutionState::Closing => {
3437 crate::server::ViewerChatPhase::Closing
3438 }
3439 mj_core::relay::RelayExecutionState::Closed => {
3440 crate::server::ViewerChatPhase::Closed
3441 }
3442 };
3443 session.config_options = crate::server::viewer_config_options(
3444 &state.config_options,
3445 facts
3446 .as_ref()
3447 .expect("live operational state always has ACP session facts"),
3448 );
3449 let turn_started_at_ms = state
3452 .active_prompt
3453 .as_ref()
3454 .map(|prompt| prompt.started_at_ms)
3455 .or_else(|| state.harness_turn.map(|turn| turn.started_at_ms));
3456 let turn_started_at = turn_started_at_ms
3457 .and_then(|started_at_ms| u64::try_from(started_at_ms).ok())
3458 .map(|started_at_ms| started_at_ms / 1_000);
3459 session.capacity_retry = state.capacity_retry.clone();
3460 let activity = mj_client::usage_format::SessionActivity::of(state);
3461 let activity_details =
3462 activity.details(turn_started_at_ms, state.current_step_started_at_ms);
3463 session.activity_details = Some(viewer_activity_details(&activity_details));
3464 session.is_idle = controller
3465 .state
3466 .sessions
3467 .get(&session.id)
3468 .is_some_and(|record| record.state == mj_core::state::SessionState::Running)
3469 && session.operation.is_none()
3470 && activity.is_idle(turn_started_at);
3471 session.activity = mj_client::usage_format::format_activity_columns(
3472 now,
3473 turn_started_at,
3474 state
3475 .current_step_started_at_ms
3476 .and_then(|value| u64::try_from(value).ok()),
3477 &activity,
3478 )
3479 .join(" ")
3480 .trim()
3481 .to_owned();
3482 }
3483 session.plan_mode_active = facts
3484 .as_ref()
3485 .filter(|facts| facts.supports_plan_mode())
3486 .map(mj_core::acp::AcpSessionFacts::plan_mode_active);
3487 session.turn_review = reviews
3488 .get(&session.id)
3489 .map(crate::server::ViewerTurnReview::from_runtime);
3490 session.capabilities =
3491 session_capabilities(session, live, operations.get(&session.id), facts.as_ref());
3492 session.available_commands = phone_commands(session, live);
3493 if let Some(transcript) = conversations.get(&session.id) {
3494 session.conversation_available = true;
3495 if !session.transitioning {
3496 let mut lines = transcript
3497 .entries
3498 .iter()
3499 .flat_map(|entry| {
3500 entry
3501 .lines
3502 .iter()
3503 .enumerate()
3504 .filter_map(move |(index, line)| {
3505 let line = line.trim();
3506 (!line.is_empty()).then(|| {
3507 if index == 0 {
3508 format!("{}: {line}", entry.label)
3509 } else {
3510 line.to_owned()
3511 }
3512 })
3513 })
3514 })
3515 .collect::<Vec<_>>();
3516 session.preview = lines.split_off(lines.len().saturating_sub(4));
3517 }
3518 }
3519 session.capabilities.open = session.conversation_available && !session.transitioning;
3522 }
3523 snapshot.capacity = capacity.to_vec();
3524 snapshot
3525}
3526
3527fn viewer_activity_details(
3528 details: &mj_client::usage_format::SessionActivityDetails,
3529) -> ViewerActivityDetails {
3530 ViewerActivityDetails {
3531 kind: match details.kind {
3532 mj_client::usage_format::SessionActivityKind::Turn => ViewerActivityKind::Turn,
3533 mj_client::usage_format::SessionActivityKind::Step => ViewerActivityKind::Step,
3534 mj_client::usage_format::SessionActivityKind::Background => {
3535 ViewerActivityKind::Background
3536 }
3537 mj_client::usage_format::SessionActivityKind::Idle => ViewerActivityKind::Idle,
3538 mj_client::usage_format::SessionActivityKind::Lifecycle
3539 | mj_client::usage_format::SessionActivityKind::Goal => ViewerActivityKind::Lifecycle,
3540 },
3541 turn_started_at_ms: details.turn_started_at_ms,
3542 step_started_at_ms: details.step_started_at_ms,
3543 background_started_at_ms: details.background_started_at_ms,
3544 idle_since_ms: details.idle_since_ms,
3545 label: details.label.clone(),
3546 }
3547}
3548
3549async fn load_materialized_activity(
3553 controller: &Controller,
3554) -> Result<std::collections::BTreeMap<String, Option<i64>>> {
3555 let session_ids = controller
3556 .state
3557 .sessions
3558 .keys()
3559 .cloned()
3560 .collect::<Vec<_>>();
3561 tokio::task::spawn_blocking(move || {
3562 session_ids
3563 .into_iter()
3564 .map(|session_id| {
3565 crate::database::load_materialized_session_summary(&session_id).map(|summary| {
3566 (
3567 session_id,
3568 summary.and_then(|summary| summary.last_activity_at_ms),
3569 )
3570 })
3571 })
3572 .collect()
3573 })
3574 .await
3575 .context("materialized activity startup task failed")?
3576}
3577
3578#[cfg(test)]
3579mod tests {
3580 use super::*;
3581 use crate::pollers::QUOTA_REFRESH_INTERVAL;
3582 use std::collections::BTreeMap;
3583
3584 use agent_client_protocol::schema::v1::{
3585 SessionConfigOption, SessionConfigOptionCategory, SessionConfigSelectOption,
3586 SessionConfigSelectOptions,
3587 };
3588 use mj_core::config::{
3589 CONFIG_VERSION, Config, HarnessKind, ProjectBundle, ProjectRepository, TargetTemplate,
3590 };
3591 use mj_core::state::SessionState;
3592
3593 #[test]
3594 fn viewer_config_options_publish_current_advertised_values() {
3595 let make_options = |model, effort| {
3596 vec![
3597 SessionConfigOption::select(
3598 "model_selector",
3599 "Model",
3600 model,
3601 SessionConfigSelectOptions::Ungrouped(vec![
3602 SessionConfigSelectOption::new("sonnet", "Claude Sonnet"),
3603 SessionConfigSelectOption::new("opus", "Claude Opus"),
3604 ]),
3605 )
3606 .category(SessionConfigOptionCategory::Model),
3607 SessionConfigOption::select(
3608 "reasoning_effort",
3609 "Effort",
3610 effort,
3611 SessionConfigSelectOptions::Ungrouped(vec![
3612 SessionConfigSelectOption::new("high", "High"),
3613 SessionConfigSelectOption::new("max", "Maximum"),
3614 ]),
3615 ),
3616 ]
3617 };
3618 let options = make_options("sonnet", "high");
3619 let defaults = mj_core::acp::AcpSessionFacts::from_operational(
3620 HarnessKind::Claude,
3621 &BTreeMap::new(),
3622 &options,
3623 None,
3624 );
3625 let projected = crate::server::viewer_config_options(&options, &defaults);
3626 assert_eq!(
3627 projected
3628 .iter()
3629 .map(|option| (option.key.as_str(), option.current.as_deref()))
3630 .collect::<Vec<_>>(),
3631 [("model", Some("sonnet")), ("effort", Some("high"))]
3632 );
3633
3634 let options = make_options("opus", "max");
3635 let updated = mj_core::acp::AcpSessionFacts::from_operational(
3636 HarnessKind::Claude,
3637 &BTreeMap::new(),
3638 &options,
3639 None,
3640 );
3641 let projected = crate::server::viewer_config_options(&options, &updated);
3642 assert_eq!(projected[0].current.as_deref(), Some("opus"));
3643 assert_eq!(projected[1].current.as_deref(), Some("max"));
3644 }
3645
3646 #[tokio::test]
3647 async fn explicit_tls_takes_precedence_over_tailscale_detection() {
3648 let resolved = resolve_server_args(
3649 ServerArgs {
3650 bind: "0.0.0.0:4443".into(),
3651 tailscale_detect: true,
3652 tls_cert: Some(PathBuf::from("configured-cert.pem")),
3653 tls_key: Some(PathBuf::from("configured-key.pem")),
3654 },
3655 tokio_util::sync::CancellationToken::new(),
3656 )
3657 .await
3658 .unwrap();
3659
3660 assert_eq!(resolved.bind, "0.0.0.0:4443".parse().unwrap());
3661 assert_eq!(resolved.viewer_url, "https://0.0.0.0:4443");
3662 assert_eq!(
3663 resolved.tls_files,
3664 Some((
3665 PathBuf::from("configured-cert.pem"),
3666 PathBuf::from("configured-key.pem")
3667 ))
3668 );
3669 assert!(resolved.tailscale.is_none());
3670 assert!(resolved.fallback_reason.is_none());
3671 }
3672
3673 #[tokio::test]
3674 async fn disabling_detection_keeps_the_viewer_on_loopback() {
3675 let resolved = resolve_server_args(
3676 ServerArgs {
3677 bind: "127.0.0.1:4765".into(),
3678 tailscale_detect: false,
3679 tls_cert: None,
3680 tls_key: None,
3681 },
3682 tokio_util::sync::CancellationToken::new(),
3683 )
3684 .await
3685 .unwrap();
3686
3687 assert_eq!(resolved.bind, "127.0.0.1:4765".parse().unwrap());
3688 assert_eq!(resolved.viewer_url, "http://127.0.0.1:4765");
3689 assert!(
3690 resolved
3691 .fallback_reason
3692 .unwrap()
3693 .contains("detection is disabled")
3694 );
3695 }
3696
3697 fn bare_preflight_config() -> Config {
3698 let mut config = Config::default();
3699 config
3700 .targets
3701 .insert("raw".into(), TargetTemplate::LocalBare);
3702 config
3703 }
3704
3705 #[test]
3706 fn move_recovery_projection_exposes_safe_retry_settings_only() {
3707 let operation = mj_core::state::MoveOperation {
3708 source_checkpoint_only: false,
3709 operation_id: "move-1".into(),
3710 selection: mj_core::state::MoveSelection {
3711 clear_resource_allocation: true,
3712 session_id: "session-1".into(),
3713 profile_id: Some("destination-profile".into()),
3714 target_template_id: Some("destination-target".into()),
3715 additional_mounts: Some(vec![crate::targets::AdditionalMount {
3716 source: "/destination/source".into(),
3717 destination: "/destination/target".into(),
3718 read_only: true,
3719 }]),
3720 resource_allocation: None,
3721 },
3722 source_profile_id: "source-profile".into(),
3723 source_target_template_id: "source-target".into(),
3724 source_target: None,
3725 source_native_session_id: Some("private-native-id".into()),
3726 source_additional_mounts: vec![crate::targets::AdditionalMount {
3727 source: "/source/source".into(),
3728 destination: "/source/target".into(),
3729 read_only: false,
3730 }],
3731 source_resource_allocation: Some(
3732 mj_core::state::SessionResourceAllocation::Container {
3733 cpus: 2,
3734 memory_bytes: 4096,
3735 },
3736 ),
3737 destination_target: None,
3738 destination_native_session_id: None,
3739 destination_store_id: None,
3740 configuration_fingerprint: "private-fingerprint".into(),
3741 checkpoint: None,
3742 recovery_session: None,
3743 queue: mj_core::state::ResumeQueueDisposition::Start,
3744 phase: mj_core::state::MovePhase::Cancelled,
3745 queue_admission_started: false,
3746 queue_admission_finished: false,
3747 cancellation_requested: true,
3748 created_at: "now".into(),
3749 updated_at: "now".into(),
3750 error: Some("private path and token".into()),
3751 };
3752 let recovery = ViewerMoveRecovery::from_operation(&operation).unwrap();
3753 assert_eq!(recovery.phase, "cancelled");
3754 assert_eq!(recovery.source_profile_id, "source-profile");
3755 assert!(recovery.clear_resource_allocation);
3756 assert_eq!(recovery.source_additional_mounts.len(), 1);
3757 assert!(recovery.source_resource_allocation.is_some());
3758 assert_eq!(recovery.destination_additional_mounts.len(), 1);
3759 assert!(recovery.destination_resource_allocation.is_none());
3760 let json = serde_json::to_string(&recovery).unwrap();
3761 assert!(!json.contains("private-native-id"));
3762 assert!(!json.contains("private-fingerprint"));
3763 assert!(!json.contains("private path and token"));
3764 }
3765
3766 #[test]
3767 fn new_preflight_rejects_a_bare_project_without_a_git_head() {
3768 let error = run_new_preflight(
3769 bare_preflight_config(),
3770 "hel".into(),
3771 "raw".into(),
3772 Some(PathBuf::from("/definitely/not/a/project")),
3773 )
3774 .expect_err("a missing project directory must fail preflight");
3775
3776 assert!(
3777 error
3778 .to_string()
3779 .contains("project directory does not exist or is not a directory")
3780 );
3781 }
3782
3783 #[test]
3784 fn new_preflight_accepts_a_git_project_for_a_bare_target() {
3785 let directory = std::env::current_dir().expect("the test has a working directory");
3786 let answer = run_new_preflight(
3787 bare_preflight_config(),
3788 "hel".into(),
3789 "raw".into(),
3790 Some(directory.clone()),
3791 )
3792 .expect("the repository running the test has a valid Git HEAD");
3793
3794 assert!(answer.dirty_repositories.is_empty());
3795 assert_eq!(answer.project_directory, Some(directory));
3796 }
3797
3798 #[test]
3799 fn new_preflight_requires_network_sources_for_isolated_targets() {
3800 let mut config = Config::default();
3801 config.targets.insert(
3802 "podman".into(),
3803 TargetTemplate::LocalPodman {
3804 container: mj_core::config::ContainerTemplate {
3805 image: "test-image".into(),
3806 pull_policy: Default::default(),
3807 platform: None,
3808 cpus: None,
3809 memory: None,
3810 environment: Default::default(),
3811 workspace_storage: Default::default(),
3812 },
3813 },
3814 );
3815 config.bundles.insert(
3816 "hel".into(),
3817 ProjectBundle {
3818 primary_repo: "hel".into(),
3819 repositories: vec![ProjectRepository {
3820 id: "hel".into(),
3821 github: None,
3822 local: Some(PathBuf::from("/definitely/not/a/repository")),
3823 destination: "hel".into(),
3824 git_ref: None,
3825 }],
3826 },
3827 );
3828 let error = run_new_preflight(config, "hel".into(), "podman".into(), None)
3829 .expect_err("an isolated bundle cannot use a local source");
3830 assert!(error.to_string().contains("repository"));
3831 }
3832
3833 #[test]
3834 fn a_phone_prompt_becomes_its_text_then_its_images() {
3835 use agent_client_protocol::schema::v1::ContentBlock;
3836
3837 let image = |data: &str| crate::server::ViewerPromptImage {
3838 attachment: None,
3839 data_base64: data.into(),
3840 mime_type: "image/png".into(),
3841 width: 32,
3842 height: 24,
3843 };
3844 let blocks = phone_prompt_blocks(
3845 "look at this".into(),
3846 vec![image("aW1hZ2U="), image("c2Vjb25k")],
3847 );
3848 let ContentBlock::Text(text) = &blocks[0] else {
3849 panic!("the prompt leads with its text");
3850 };
3851 assert_eq!(text.text, "look at this");
3852 let ContentBlock::Image(first) = &blocks[1] else {
3853 panic!("each attachment travels as an image block");
3854 };
3855 assert_eq!(first.data, "aW1hZ2U=");
3856 assert_eq!(first.mime_type, "image/png");
3857 assert!(matches!(blocks[2], ContentBlock::Image(_)));
3858 assert_eq!(blocks.len(), 3);
3859
3860 let images_only = phone_prompt_blocks(String::new(), vec![image("aW1hZ2U=")]);
3863 assert_eq!(images_only.len(), 1);
3864 assert!(matches!(images_only[0], ContentBlock::Image(_)));
3865 }
3866
3867 #[test]
3868 fn image_prompts_are_offered_only_after_the_agent_advertises_them() {
3869 use agent_client_protocol::schema::v1::AgentCapabilities;
3870 use mj_core::relay::{RelayExecutionState, RelayOperationalState};
3871
3872 let operational = |agent_capabilities| RelayOperationalState {
3873 goal: Default::default(),
3874 capacity_retry: None,
3875 activity_turn_started_at_ms: None,
3876 session_id: "session-1".into(),
3877 store_id: None,
3878 idle_since_ms: None,
3879 execution: RelayExecutionState::Idle,
3880 latest_ordinal: 0,
3881 latest_digest: String::new(),
3882 acknowledged_through: 0,
3883 acknowledged_digest: String::new(),
3884 recovery_floor_ordinal: 0,
3885 recovery_floor_digest: String::new(),
3886 native_session_id: None,
3887 checkpoint_only: false,
3888 acp_ready: None,
3889 agent_capabilities,
3890 agent_info: None,
3891 steering_supported: None,
3892 config_options: Vec::new(),
3893 modes: None,
3894 available_commands: Vec::new(),
3895 config: std::collections::BTreeMap::new(),
3896 active_prompt: None,
3897 queued_prompts: Vec::new(),
3898 active_user_shells: Vec::new(),
3899 active_agent_terminals: Vec::new(),
3900 checkpoint_barrier: None,
3901 checkpoint_ready: None,
3902 last_acp_activity_at_ms: None,
3903 current_step_started_at_ms: None,
3904 foreground_tool_started_at_ms: None,
3905 harness_turn: None,
3906 last_harness_turn_started_ordinal: None,
3907 background_commands: Vec::new(),
3908 background_work_known: None,
3909 };
3910
3911 assert!(!agent_accepts_prompt_images(&operational(None)));
3914 assert!(!agent_accepts_prompt_images(&operational(Some(Box::new(
3915 AgentCapabilities::default()
3916 )))));
3917 let mut capabilities = AgentCapabilities::default();
3918 capabilities.prompt_capabilities.image = true;
3919 assert!(agent_accepts_prompt_images(&operational(Some(Box::new(
3920 capabilities
3921 )))));
3922 }
3923
3924 #[test]
3925 fn phone_snapshot_projects_capability_gated_and_agent_commands_with_provenance() {
3926 use agent_client_protocol::schema::v1::{
3927 AvailableCommand, AvailableCommandInput, SessionMode, SessionModeState,
3928 UnstructuredCommandInput,
3929 };
3930 use mj_core::relay::{RelayExecutionState, RelayOperationalState};
3931
3932 use crate::server::ViewerCommandSource;
3933
3934 let mut controller = controller_with_profiles(&["claude"]);
3935 let mut record = phone_session("session-1", 0);
3936 record.harness_kind = HarnessKind::Claude;
3937 record.last_profile = "claude".into();
3938 record.state = SessionState::Running;
3939 controller.state.sessions.insert(record.id.clone(), record);
3940 let operational = RelayOperationalState {
3941 goal: Default::default(),
3942 capacity_retry: None,
3943 activity_turn_started_at_ms: None,
3944 session_id: "session-1".into(),
3945 store_id: None,
3946 idle_since_ms: None,
3947 execution: RelayExecutionState::Idle,
3948 latest_ordinal: 0,
3949 latest_digest: String::new(),
3950 acknowledged_through: 0,
3951 acknowledged_digest: String::new(),
3952 recovery_floor_ordinal: 0,
3953 recovery_floor_digest: String::new(),
3954 native_session_id: None,
3955 checkpoint_only: false,
3956 acp_ready: None,
3957 agent_capabilities: None,
3958 agent_info: None,
3959 steering_supported: None,
3960 config_options: Vec::new(),
3961 modes: Some(SessionModeState::new(
3962 "default",
3963 vec![
3964 SessionMode::new("default", "Default"),
3965 SessionMode::new("plan", "Plan"),
3966 ],
3967 )),
3968 available_commands: vec![
3969 AvailableCommand::new("inspect", " Inspect the workspace ").input(
3970 AvailableCommandInput::Unstructured(UnstructuredCommandInput::new(" query ")),
3971 ),
3972 AvailableCommand::new("Review", "agent collision"),
3973 AvailableCommand::new("INSPECT", "duplicate agent command"),
3974 ],
3975 config: std::collections::BTreeMap::new(),
3976 active_prompt: None,
3977 queued_prompts: Vec::new(),
3978 active_user_shells: Vec::new(),
3979 active_agent_terminals: Vec::new(),
3980 checkpoint_barrier: None,
3981 checkpoint_ready: None,
3982 last_acp_activity_at_ms: None,
3983 current_step_started_at_ms: None,
3984 foreground_tool_started_at_ms: None,
3985 harness_turn: None,
3986 last_harness_turn_started_ordinal: None,
3987 background_commands: Vec::new(),
3988 background_work_known: None,
3989 };
3990 let mut operational = std::collections::BTreeMap::from([("session-1".into(), operational)]);
3991 let materialized_activity =
3992 std::collections::BTreeMap::from([("session-1".into(), Some(7_777_i64))]);
3993 let project = |operational: &std::collections::BTreeMap<String, RelayOperationalState>| {
3994 viewer_snapshot(
3995 &controller,
3996 &[],
3997 &std::collections::BTreeMap::new(),
3998 &PhoneSessionViews {
3999 conversations: &std::collections::BTreeMap::new(),
4000 queued_prompts: &std::collections::BTreeMap::new(),
4001 active_user_shells: &std::collections::BTreeMap::new(),
4002 pending_elicitations: &std::collections::BTreeMap::new(),
4003 prompt_images: &std::collections::BTreeSet::new(),
4004 operational,
4005 materialized_activity: &materialized_activity,
4006 project_sources: &PhoneProjectSources::default(),
4007 operations: &std::collections::BTreeMap::new(),
4008 move_recoveries: &std::collections::BTreeMap::new(),
4009 capacity: &[],
4010 launch_failures: &[],
4011 reviews: &std::collections::BTreeMap::new(),
4012 },
4013 1,
4014 )
4015 };
4016 let snapshot = project(&operational);
4017 let session = &snapshot.sessions[0];
4018
4019 assert_eq!(session.display_location, "podman");
4020 assert_eq!(session.last_activity_at_ms, Some(7_777));
4021 assert_eq!(
4022 session.activity_details,
4023 Some(crate::server::ViewerActivityDetails {
4024 kind: ViewerActivityKind::Idle,
4025 turn_started_at_ms: None,
4026 step_started_at_ms: None,
4027 background_started_at_ms: None,
4028 idle_since_ms: None,
4029 label: None,
4030 })
4031 );
4032
4033 assert!(session.capabilities.prompt);
4034 assert!(session.capabilities.set_plan_mode);
4035 assert_eq!(
4036 session
4037 .available_commands
4038 .iter()
4039 .map(|command| (command.name.as_str(), command.source))
4040 .collect::<Vec<_>>(),
4041 vec![
4042 ("help", ViewerCommandSource::Mj),
4043 ("detach", ViewerCommandSource::Mj),
4044 ("plan", ViewerCommandSource::Mj),
4045 ("implement", ViewerCommandSource::Mj),
4046 ("review", ViewerCommandSource::Mj),
4047 ("inspect", ViewerCommandSource::Agent),
4048 ]
4049 );
4050 let inspect = session.available_commands.last().unwrap();
4051 assert_eq!(inspect.description, "Inspect the workspace");
4052 assert_eq!(inspect.argument.as_deref(), Some("query"));
4053
4054 assert!(session.is_idle);
4055 assert_eq!(session.activity, "[idle]");
4056 let state = operational.get_mut("session-1").unwrap();
4057 state
4058 .background_commands
4059 .push(mj_core::relay::BackgroundCommand {
4060 id: "background-1".into(),
4061 started_at_ms: 1_000,
4062 command: "background check".into(),
4063 can_stop: true,
4064 });
4065 let background = project(&operational);
4066 assert_eq!(
4067 background.sessions[0].background_tasks,
4068 vec![ViewerBackgroundTask {
4069 id: "background-1".into(),
4070 command: "background check".into(),
4071 started_at_ms: 1_000,
4072 can_stop: true,
4073 }]
4074 );
4075 assert!(!background.sessions[0].is_idle);
4076 assert!(background.sessions[0].activity.starts_with("BG "));
4077 let state = operational.get_mut("session-1").unwrap();
4078 state.background_commands.clear();
4079 state.execution = RelayExecutionState::Running;
4081 let stale_running = project(&operational);
4082 assert!(stale_running.sessions[0].is_idle);
4083 operational
4084 .get_mut("session-1")
4085 .unwrap()
4086 .current_step_started_at_ms = Some(1_000);
4087 let running = project(&operational);
4088 assert!(!running.sessions[0].is_idle);
4089 assert_ne!(running.sessions[0].activity, "[idle]");
4090 operational.get_mut("session-1").unwrap().execution = RelayExecutionState::Idle;
4091 let idle_again = project(&operational);
4092 assert!(idle_again.sessions[0].is_idle);
4093 assert_eq!(idle_again.sessions[0].activity, "[idle]");
4094 let unknown = project(&std::collections::BTreeMap::new());
4095 assert!(!unknown.sessions[0].is_idle);
4096 assert!(unknown.sessions[0].activity.is_empty());
4097 assert!(unknown.sessions[0].activity_details.is_none());
4098 }
4099
4100 #[test]
4101 fn tailscale_listener_preserves_the_configured_port() {
4102 assert_eq!(
4103 tailscale_bind("127.0.0.1:4765".parse().unwrap()),
4104 "0.0.0.0:4765".parse().unwrap()
4105 );
4106 }
4107
4108 fn controller_with_profiles(ids: &[&str]) -> Controller {
4109 Controller {
4110 config: Config {
4111 subagents: Default::default(),
4112 version: CONFIG_VERSION,
4113 sessions_side: Default::default(),
4114 advanced: Default::default(),
4115 show_stopped_sessions: false,
4116 newer_config_version: None,
4117 spinner: Default::default(),
4118 theme: Default::default(),
4119 phone: Default::default(),
4120 review: Default::default(),
4121 legacy_startup: (),
4122 profiles: ids
4123 .iter()
4124 .map(|id| {
4125 (
4126 (*id).to_owned(),
4127 HarnessProfile {
4128 enabled: true,
4129 context_window_bytes: None,
4130 kind: HarnessKind::Codex,
4131 home: PathBuf::from("/home/agent").join(id),
4132 environment: std::collections::BTreeMap::new(),
4133 },
4134 )
4135 })
4136 .collect(),
4137 bundles: std::collections::BTreeMap::new(),
4138 targets: std::collections::BTreeMap::new(),
4139 },
4140 state: State::default(),
4141 }
4142 }
4143
4144 fn snapshot_with_project_sources(
4145 controller: &Controller,
4146 sources: &PhoneProjectSources,
4147 ) -> ViewerSnapshot {
4148 viewer_snapshot(
4149 controller,
4150 &[],
4151 &Default::default(),
4152 &PhoneSessionViews {
4153 conversations: &Default::default(),
4154 queued_prompts: &Default::default(),
4155 active_user_shells: &Default::default(),
4156 pending_elicitations: &Default::default(),
4157 prompt_images: &Default::default(),
4158 operational: &Default::default(),
4159 materialized_activity: &Default::default(),
4160 project_sources: sources,
4161 operations: &Default::default(),
4162 move_recoveries: &Default::default(),
4163 capacity: &[],
4164 launch_failures: &[],
4165 reviews: &Default::default(),
4166 },
4167 1,
4168 )
4169 }
4170
4171 #[tokio::test]
4172 async fn phone_projects_resolve_origins_and_discard_results_after_location_changes() {
4173 let root = tempfile::tempdir().unwrap();
4174 let mut controller = controller_with_profiles(&["codex"]);
4175 controller
4176 .config
4177 .targets
4178 .insert("local".into(), TargetTemplate::LocalBare);
4179 for (id, origin) in [
4180 ("first-checkout", "git@github.com:BrokkAi/hel.git"),
4181 ("second-checkout", "https://github.com/BrokkAi/hel.git"),
4182 ] {
4183 let directory = root.path().join(id);
4184 std::fs::create_dir(&directory).unwrap();
4185 for args in [vec!["init"], vec!["remote", "add", "origin", origin]] {
4186 let command = crate::targets::CommandSpec::new(
4187 "git",
4188 ["-C".to_owned(), directory.to_string_lossy().into_owned()]
4189 .into_iter()
4190 .chain(args.into_iter().map(str::to_owned)),
4191 );
4192 assert_eq!(ProcessExecutor.execute(&command).unwrap().status, 0);
4193 }
4194 let mut record = phone_session(id, 0);
4195 record.project_directory = Some(directory);
4196 record.target_template_id = "local".into();
4197 controller.state.sessions.insert(id.into(), record);
4198 }
4199 let mut sources = PhoneProjectSources::default();
4200 sources.synchronize(&controller);
4201 assert_eq!(sources.jobs.len(), 2);
4202 tokio::time::timeout(Duration::from_secs(10), async {
4203 while let Some(result) = sources.jobs.join_next().await {
4204 sources.complete(result.unwrap());
4205 }
4206 })
4207 .await
4208 .unwrap();
4209 let snapshot = snapshot_with_project_sources(&controller, &sources);
4210 assert_eq!(
4211 snapshot.sessions[0].project_key,
4212 snapshot.sessions[1].project_key
4213 );
4214 assert!(
4215 snapshot
4216 .sessions
4217 .iter()
4218 .all(|session| session.project_label == "hel")
4219 );
4220 assert!(
4221 !serde_json::to_string(&snapshot)
4222 .unwrap()
4223 .contains(&root.path().to_string_lossy().to_string())
4224 );
4225 sources.synchronize(&controller);
4226 assert!(
4227 sources.jobs.is_empty(),
4228 "unchanged inputs reuse the resolved origin"
4229 );
4230
4231 let previous = &sources.entries["first-checkout"];
4232 let late = ProjectSourceResolved {
4233 cancelled: previous.cancelled.clone(),
4234 session_id: "first-checkout".into(),
4235 key: previous.key.clone(),
4236 result: Ok(ProjectSourceIdentity::git_remote("old/wrong").unwrap()),
4237 };
4238 controller
4239 .state
4240 .sessions
4241 .get_mut("first-checkout")
4242 .unwrap()
4243 .project_directory = None;
4244 let snapshot = snapshot_with_project_sources(&controller, &sources);
4245 assert_ne!(
4246 snapshot.sessions[0].project_key,
4247 snapshot.sessions[1].project_key
4248 );
4249 sources.synchronize(&controller);
4250 assert!(late.cancelled.load(Ordering::Acquire));
4251 sources.complete(late);
4252 assert!(!sources.entries.contains_key("first-checkout"));
4253 assert_eq!(snapshot.sessions[0].project_label, "project");
4254 }
4255
4256 #[test]
4257 fn capacity_target_publication_skips_unchanged_targets_and_preserves_readings() {
4258 let mut controller = controller_with_profiles(&[]);
4259 controller
4260 .config
4261 .targets
4262 .insert("raw".into(), TargetTemplate::LocalBare);
4263 let (targets_tx, mut targets_rx) = tokio::sync::watch::channel(Vec::new());
4264 let mut state = std::collections::BTreeMap::new();
4265
4266 publish_capacity_targets(&controller, &targets_tx, &mut state);
4267 assert!(targets_rx.has_changed().expect("target sender is alive"));
4268 assert_eq!(targets_rx.borrow_and_update().len(), 1);
4269
4270 let usage = crate::targets::DeploymentCapacityUsage {
4271 cpu_percent: Some(37),
4272 memory_used_bytes: 3,
4273 memory_total_bytes: 4,
4274 logical_cores: 8,
4275 disk_total_bytes: Some(5),
4276 };
4277 let local = state.get_mut("local").expect("local capacity state");
4278 local.usage = Some(usage.clone());
4279 local.on_demand = true;
4280 local.sampled_at_epoch_seconds = Some(42);
4281 local.refreshing = false;
4282
4283 publish_capacity_targets(&controller, &targets_tx, &mut state);
4284 assert!(!targets_rx.has_changed().expect("target sender is alive"));
4285 let local_capacity = viewer_capacity(&state)
4286 .into_iter()
4287 .find(|capacity| capacity.id == "local")
4288 .expect("local viewer capacity");
4289 assert_eq!(local_capacity.cpu_percent, usage.cpu_percent);
4290 assert_eq!(
4291 local_capacity.memory_used_bytes,
4292 Some(usage.memory_used_bytes)
4293 );
4294 assert_eq!(local_capacity.logical_cores, Some(usage.logical_cores));
4295 assert_eq!(local_capacity.sampled_at_epoch_seconds, Some(42));
4296
4297 controller
4298 .config
4299 .targets
4300 .insert("second-local".into(), TargetTemplate::LocalBare);
4301 publish_capacity_targets(&controller, &targets_tx, &mut state);
4302 assert!(targets_rx.has_changed().expect("target sender is alive"));
4303 assert_eq!(targets_rx.borrow_and_update().len(), 1);
4304 assert_eq!(state["local"].usage, Some(usage.clone()));
4305
4306 controller.config.targets.insert(
4307 "fleet".into(),
4308 TargetTemplate::AwsEc2 {
4309 aws_profile: None,
4310 region: "us-east-1".into(),
4311 launch_template: "hel-runson".into(),
4312 launch_template_version: None,
4313 ssh_user: "ubuntu".into(),
4314 address_source: Default::default(),
4315 identity_file: None,
4316 ssh_args: Vec::new(),
4317 },
4318 );
4319 publish_capacity_targets(&controller, &targets_tx, &mut state);
4320 assert!(targets_rx.has_changed().expect("target sender is alive"));
4321 assert_eq!(targets_rx.borrow_and_update().len(), 2);
4322 assert!(state.contains_key("aws:fleet"));
4323 assert_eq!(state["local"].usage, Some(usage.clone()));
4324
4325 controller.config.targets.remove("fleet");
4326 publish_capacity_targets(&controller, &targets_tx, &mut state);
4327 assert!(targets_rx.has_changed().expect("target sender is alive"));
4328 assert_eq!(targets_rx.borrow_and_update().len(), 1);
4329 assert!(!state.contains_key("aws:fleet"));
4330 assert_eq!(state["local"].usage, Some(usage));
4331 }
4332
4333 fn prompt_action() -> ControllerAction {
4334 ControllerAction::Prompt {
4335 session_id: "session-1".into(),
4336 text: "ship it".into(),
4337 images: Vec::new(),
4338 }
4339 }
4340
4341 fn new_action() -> ControllerAction {
4342 ControllerAction::New {
4343 create_managed_worktree: None,
4344 workspace_id: String::new(),
4345 profile_id: "codex".into(),
4346 bundle_id: "project".into(),
4347 target_id: "podman".into(),
4348 title: Some("Phone launch".into()),
4349 project_directory: None,
4350 dirty_ack: Vec::new(),
4351 }
4352 }
4353
4354 fn phone_session(id: &str, viewed_through_event_ordinal: u64) -> SessionRecord {
4355 SessionRecord {
4356 create_managed_worktree: None,
4357 workspace_id: mj_core::workspace::DEFAULT_WORKSPACE_ID.to_owned(),
4358 archived: false,
4359 container_cpus: None,
4360 container_memory: None,
4361 id: id.into(),
4362 title: "Phone launch".into(),
4363 harness_kind: mj_core::config::HarnessKind::Codex,
4364 last_profile: "codex".into(),
4365 bundle_id: "project".into(),
4366 project_directory: None,
4367 managed_worktree: None,
4368 target_template_id: "podman".into(),
4369 resource_allocation: None,
4370 additional_mounts: Vec::new(),
4371 state: SessionState::Provisioning,
4372 target: None,
4373 native_session_id: None,
4374 acp_session_title: None,
4375 session_title_override: Some("Phone launch".into()),
4376 created_at: "2026-08-14T00:00:00Z".into(),
4377 updated_at: "2026-08-14T00:00:00Z".into(),
4378 viewed_through_event_ordinal,
4379 draft_input: String::new(),
4380 last_error: None,
4381 last_checkpoint_error: None,
4382 checkpoint: None,
4383 }
4384 }
4385
4386 #[tokio::test]
4387 async fn controller_reload_does_not_block_the_phone_control_loop() {
4388 let (completed_tx, mut completed_rx) = tokio::sync::mpsc::unbounded_channel();
4389 let (started_tx, started_rx) = std::sync::mpsc::channel();
4390 let (release_tx, release_rx) = std::sync::mpsc::channel();
4391 let loaded = controller_with_profiles(&[]);
4392 let release = std::thread::spawn(move || {
4393 started_rx.recv().unwrap();
4394 std::thread::sleep(Duration::from_millis(250));
4395 release_tx.send(()).unwrap();
4396 });
4397
4398 let started = Instant::now();
4399 spawn_controller_reload_with(completed_tx, move || {
4400 started_tx.send(()).unwrap();
4401 release_rx.recv().unwrap();
4402 Ok(loaded)
4403 });
4404 assert!(
4405 started.elapsed() < Duration::from_millis(100),
4406 "scheduling a controller reload occupied the control loop for {:?}",
4407 started.elapsed()
4408 );
4409
4410 let completed = tokio::time::timeout(Duration::from_secs(1), completed_rx.recv())
4411 .await
4412 .expect("background reload timed out")
4413 .expect("background reload channel closed");
4414 assert!(completed.result.is_ok());
4415 release.join().unwrap();
4416 }
4417
4418 #[test]
4419 fn read_receipt_only_persists_and_refreshes_when_the_cursor_advances() {
4420 let session_id = "0123456789abcdef0123456789abcdef";
4421 let mut state = State::default();
4422 state
4423 .sessions
4424 .insert(session_id.into(), phone_session(session_id, 5));
4425
4426 assert_eq!(
4429 plan_read_receipt(&state, session_id, 5),
4430 ReadReceiptPlan::AlreadyRead
4431 );
4432 assert_eq!(
4433 plan_read_receipt(&state, session_id, 4),
4434 ReadReceiptPlan::AlreadyRead
4435 );
4436 assert_eq!(
4437 plan_read_receipt(&state, "missing", 9),
4438 ReadReceiptPlan::UnknownSession
4439 );
4440 assert_eq!(
4441 plan_read_receipt(&state, session_id, 9),
4442 ReadReceiptPlan::Persist
4443 );
4444
4445 assert!(apply_read_receipt(&mut state, session_id, 9));
4446 assert_eq!(state.sessions[session_id].viewed_through_event_ordinal, 9);
4447 assert!(!apply_read_receipt(&mut state, session_id, 9));
4448 assert!(!apply_read_receipt(&mut state, session_id, 7));
4449 assert!(!apply_read_receipt(&mut state, "missing", 9));
4450 assert_eq!(
4451 plan_read_receipt(&state, session_id, 9),
4452 ReadReceiptPlan::AlreadyRead
4453 );
4454 }
4455
4456 #[tokio::test]
4457 async fn an_admitted_action_answers_its_phone_before_the_work_runs() {
4458 let mut replies = PendingActionReplies::default();
4459 let (reply, answer) = tokio::sync::oneshot::channel();
4460
4461 replies.accept(1, &prompt_action(), reply);
4462
4463 assert_eq!(answer.await.unwrap(), ActionOutcome::accepted());
4467 }
4468
4469 #[tokio::test]
4470 async fn a_new_action_answers_once_its_provisional_session_is_published() {
4471 let mut replies = PendingActionReplies::default();
4472 let (reply, mut answer) = tokio::sync::oneshot::channel();
4473
4474 replies.accept(7, &new_action(), reply);
4475 assert!(
4476 matches!(
4477 answer.try_recv(),
4478 Err(tokio::sync::oneshot::error::TryRecvError::Empty)
4479 ),
4480 "a new session has no id to report before it is published"
4481 );
4482
4483 replies.resolve(7, ActionOutcome::accepted());
4484 assert_eq!(answer.await.unwrap(), ActionOutcome::accepted());
4485 }
4486
4487 #[tokio::test]
4488 async fn a_new_action_that_never_publishes_still_answers_its_phone() {
4489 let mut replies = PendingActionReplies::default();
4490 let (reply, answer) = tokio::sync::oneshot::channel();
4491 replies.accept(7, &new_action(), reply);
4492
4493 replies.resolve(7, ActionOutcome::Failed);
4496
4497 assert_eq!(answer.await.unwrap(), ActionOutcome::Failed);
4498 replies.resolve(7, ActionOutcome::accepted());
4501 }
4502
4503 #[test]
4504 fn close_is_admitted_while_provisioning_occupies_a_full_action_pool() {
4505 let mut active = std::collections::BTreeSet::from(["session-1".to_owned()]);
4506 let close = ControllerAction::Close {
4507 session_id: "session-1".into(),
4508 };
4509 assert_eq!(
4510 admit_phone_action(&close, MAX_CONCURRENT_PHONE_ACTIONS, &mut active),
4511 Ok(Some("session-1".into()))
4512 );
4513 assert_eq!(
4514 admit_phone_action(&prompt_action(), 0, &mut active),
4515 Err(ActionOutcome::SessionBusy)
4516 );
4517 }
4518
4519 #[test]
4520 fn a_refused_action_reports_the_reason_the_phone_can_act_on() {
4521 let mut active = std::collections::BTreeSet::new();
4522
4523 assert_eq!(
4524 admit_phone_action(&prompt_action(), 0, &mut active),
4525 Ok(Some("session-1".to_owned()))
4526 );
4527 assert_eq!(
4528 admit_phone_action(&prompt_action(), 1, &mut active),
4529 Err(ActionOutcome::SessionBusy)
4530 );
4531 assert_eq!(
4532 admit_phone_action(&new_action(), MAX_CONCURRENT_PHONE_ACTIONS, &mut active),
4533 Err(ActionOutcome::Busy)
4534 );
4535 assert_eq!(active.len(), 1);
4537 assert_eq!(admit_phone_action(&new_action(), 1, &mut active), Ok(None));
4538 }
4539
4540 #[test]
4541 fn a_feed_that_ends_outside_shutdown_names_the_failure() {
4542 assert!(feed_stopped(true, "the session manager stopped").is_none());
4543 let failure = feed_stopped(false, "the session manager stopped").expect("named failure");
4544 assert!(failure.to_string().contains("session manager"));
4545 }
4546
4547 #[test]
4548 fn a_profile_added_while_the_server_runs_reaches_the_quota_refresher() {
4549 let (profiles_tx, profiles_rx) = tokio::sync::watch::channel(QuotaRefreshBatch::default());
4550 let mut published = std::collections::BTreeMap::new();
4551 let mut batch = QuotaRefreshBatch::default();
4552 let controller = controller_with_profiles(&["codex"]);
4553
4554 assert!(republish_quota_profiles(
4555 &controller,
4556 &mut published,
4557 &mut batch,
4558 &profiles_tx
4559 ));
4560 assert_eq!(
4561 profiles_rx
4562 .borrow()
4563 .profiles
4564 .iter()
4565 .map(|profile| profile.profile_id.clone())
4566 .collect::<Vec<_>>(),
4567 vec!["codex".to_owned()]
4568 );
4569 let first_generation = profiles_rx.borrow().generation;
4570
4571 assert!(!republish_quota_profiles(
4574 &controller,
4575 &mut published,
4576 &mut batch,
4577 &profiles_tx
4578 ));
4579 assert_eq!(profiles_rx.borrow().generation, first_generation);
4580
4581 let grown = controller_with_profiles(&["claude", "codex"]);
4582 assert!(republish_quota_profiles(
4583 &grown,
4584 &mut published,
4585 &mut batch,
4586 &profiles_tx
4587 ));
4588 assert_eq!(
4589 profiles_rx
4590 .borrow()
4591 .profiles
4592 .iter()
4593 .map(|profile| profile.profile_id.clone())
4594 .collect::<Vec<_>>(),
4595 vec!["claude".to_owned(), "codex".to_owned()]
4596 );
4597 assert!(profiles_rx.borrow().generation > first_generation);
4598 }
4599
4600 #[test]
4601 fn a_quota_reads_stale_only_once_its_next_refresh_is_overdue() {
4602 let controller = controller_with_profiles(&["codex"]);
4603 let now = std::time::SystemTime::now()
4604 .duration_since(std::time::UNIX_EPOCH)
4605 .unwrap()
4606 .as_secs();
4607 let quota_refreshed = |age: Duration| {
4608 let quotas = std::collections::BTreeMap::from([(
4609 "codex".to_owned(),
4610 ProfileQuota {
4611 profile_id: "codex".into(),
4612 harness: HarnessKind::Codex,
4613 windows: Vec::new(),
4614 extra: None,
4615 error: None,
4616 refreshed_at_epoch_seconds: now - age.as_secs(),
4617 },
4618 )]);
4619 viewer_snapshot(
4620 &controller,
4621 &[],
4622 "as,
4623 &PhoneSessionViews {
4624 conversations: &std::collections::BTreeMap::new(),
4625 queued_prompts: &std::collections::BTreeMap::new(),
4626 active_user_shells: &std::collections::BTreeMap::new(),
4627 pending_elicitations: &std::collections::BTreeMap::new(),
4628 prompt_images: &std::collections::BTreeSet::new(),
4629 operational: &std::collections::BTreeMap::new(),
4630 materialized_activity: &std::collections::BTreeMap::new(),
4631 project_sources: &PhoneProjectSources::default(),
4632 operations: &std::collections::BTreeMap::new(),
4633 move_recoveries: &std::collections::BTreeMap::new(),
4634 capacity: &[],
4635 launch_failures: &[],
4636 reviews: &std::collections::BTreeMap::new(),
4637 },
4638 1,
4639 )
4640 .profiles[0]
4641 .quota
4642 .as_ref()
4643 .expect("the profile carries its quota")
4644 .stale
4645 };
4646
4647 assert!(!quota_refreshed(QUOTA_REFRESH_INTERVAL));
4650 assert!(!quota_refreshed(QUOTA_STALE_AFTER));
4651 assert!(quota_refreshed(QUOTA_STALE_AFTER + Duration::from_secs(1)));
4652 }
4653
4654 #[test]
4655 fn phone_action_capacity_is_bounded() {
4656 assert!(phone_action_capacity_available(
4657 MAX_CONCURRENT_PHONE_ACTIONS - 1
4658 ));
4659 assert!(!phone_action_capacity_available(
4660 MAX_CONCURRENT_PHONE_ACTIONS
4661 ));
4662 }
4663
4664 #[test]
4665 fn started_phone_session_is_visible_and_mapped_before_provisioning() {
4666 let session_id = "0123456789abcdef0123456789abcdef";
4667 let session = phone_session(session_id, 0);
4668 let mut state = State::default();
4669 let mut active_actions = std::collections::BTreeSet::new();
4670 let mut action_sessions = std::collections::BTreeMap::new();
4671
4672 track_started_phone_session(
4673 &mut state,
4674 &mut active_actions,
4675 &mut action_sessions,
4676 7,
4677 session,
4678 )
4679 .unwrap();
4680
4681 assert_eq!(state.sessions[session_id].state, SessionState::Provisioning);
4682 assert_eq!(state.sessions[session_id].display_title(), "Phone launch");
4683 assert!(active_actions.contains(session_id));
4684 assert_eq!(
4685 action_sessions.get(&7).map(String::as_str),
4686 Some(session_id)
4687 );
4688 }
4689
4690 #[test]
4691 fn failed_launch_notice_survives_session_rollback_and_history_is_bounded() {
4692 let controller = controller_with_profiles(&["codex"]);
4693 let mut failures = Vec::new();
4694 for index in 0..20 {
4695 record_launch_failure(
4696 &mut failures,
4697 index,
4698 format!("workspace-{index}"),
4699 Some(format!("session-{index}")),
4700 );
4701 }
4702 let snapshot = viewer_snapshot(
4703 &controller,
4704 &[],
4705 &std::collections::BTreeMap::new(),
4706 &PhoneSessionViews {
4707 conversations: &std::collections::BTreeMap::new(),
4708 queued_prompts: &std::collections::BTreeMap::new(),
4709 active_user_shells: &std::collections::BTreeMap::new(),
4710 pending_elicitations: &std::collections::BTreeMap::new(),
4711 prompt_images: &std::collections::BTreeSet::new(),
4712 operational: &std::collections::BTreeMap::new(),
4713 materialized_activity: &std::collections::BTreeMap::new(),
4714 project_sources: &PhoneProjectSources::default(),
4715 operations: &std::collections::BTreeMap::new(),
4716 move_recoveries: &std::collections::BTreeMap::new(),
4717 capacity: &[],
4718 launch_failures: &failures,
4719 reviews: &std::collections::BTreeMap::new(),
4720 },
4721 1,
4722 );
4723 assert!(snapshot.sessions.is_empty());
4724 assert_eq!(failures.len(), 16);
4725 assert_eq!(failures[0].workspace_id, "workspace-4");
4726 assert_eq!(failures[15].workspace_id, "workspace-19");
4727 assert_ne!(failures[0].id, failures[1].id);
4728 assert_eq!(
4729 failures[15].session_id.as_deref(),
4730 Some("session-19"),
4731 "a wait on that session has to be able to recognize its own launch failure"
4732 );
4733 let json = serde_json::to_value(snapshot).unwrap();
4734 assert_eq!(json["launch_failures"][15]["workspace_id"], "workspace-19");
4735 assert_eq!(json["launch_failures"][15].as_object().unwrap().len(), 3);
4736 }
4737
4738 #[test]
4739 fn a_later_successful_action_clears_a_session_s_recorded_failure() {
4740 let mut pending = std::collections::BTreeMap::new();
4741
4742 record_action_result(&mut pending, Some("session-1"), &Err("relay hiccup".into()));
4743 assert_eq!(
4744 pending.get("session-1").map(String::as_str),
4745 Some("relay hiccup")
4746 );
4747
4748 record_action_result(&mut pending, Some("session-2"), &Ok(()));
4749 record_action_result(&mut pending, Some("session-1"), &Ok(()));
4750 assert!(
4751 pending.is_empty(),
4752 "the overlay has no other expiry, so a stale error would badge the session forever"
4753 );
4754
4755 record_action_result(&mut pending, None, &Err("orphaned".into()));
4757 assert!(pending.is_empty());
4758 }
4759
4760 #[test]
4761 fn phone_cancel_targets_the_matching_background_action() {
4762 let first = PhoneActionControl {
4763 cancelled: Arc::new(AtomicBool::new(false)),
4764 create: None,
4765 };
4766 let second = PhoneActionControl {
4767 cancelled: Arc::new(AtomicBool::new(false)),
4768 create: None,
4769 };
4770 let action_sessions =
4771 std::collections::BTreeMap::from([(1, "session-1".into()), (2, "session-2".into())]);
4772 let cancellations =
4773 std::collections::BTreeMap::from([(1, first.clone()), (2, second.clone())]);
4774
4775 assert!(request_phone_action_cancellation(
4776 "session-2",
4777 &action_sessions,
4778 &cancellations,
4779 ));
4780 assert!(!first.cancelled.load(Ordering::Acquire));
4781 assert!(second.cancelled.load(Ordering::Acquire));
4782 assert!(!request_phone_action_cancellation(
4783 "missing",
4784 &action_sessions,
4785 &cancellations,
4786 ));
4787 }
4788
4789 #[test]
4790 fn phone_new_cancel_and_running_commit_have_one_atomic_winner() {
4791 for _ in 0..100 {
4792 let create = CreateSessionControl::default();
4793 let control = PhoneActionControl {
4794 cancelled: create.cancelled.clone(),
4795 create: Some(create),
4796 };
4797 let cancelling = control.clone();
4798 let committing = control.clone();
4799 let (cancelled, committed) = std::thread::scope(|scope| {
4800 let cancel = scope.spawn(move || cancelling.request_cancel());
4801 let commit = scope.spawn(move || committing.grant_new_commit());
4802 (cancel.join().unwrap(), commit.join().unwrap())
4803 });
4804
4805 assert_ne!(cancelled, committed);
4806 assert_eq!(control.cancelled.load(Ordering::Acquire), cancelled);
4807 assert!(!control.request_cancel());
4808 assert!(!control.grant_new_commit());
4809 }
4810 }
4811
4812 fn materialized_at(ordinal: u64) -> MaterializedSession {
4813 let mut materialized = MaterializedSession::empty("session-1");
4814 materialized.applied_event_ordinal = ordinal;
4815 materialized.applied_event_digest = format!("digest-{ordinal}");
4816 materialized
4817 }
4818
4819 #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
4820 async fn browser_projection_enqueue_does_not_wait_for_a_blocking_permit() {
4821 let (results, mut completed) = tokio::sync::mpsc::channel(1);
4822 let shutdown = tokio_util::sync::CancellationToken::new();
4823 let mut dispatcher = ConversationProjectionDispatcher::with_permits(results, shutdown, 0);
4824
4825 dispatcher.enqueue(materialized_at(1));
4826 let (progress, progress_done) = tokio::sync::oneshot::channel();
4827 tokio::spawn(async move {
4828 progress.send(()).expect("control task is still alive");
4829 });
4830
4831 tokio::time::timeout(Duration::from_millis(100), progress_done)
4832 .await
4833 .expect("control progress was starved by transcript projection")
4834 .expect("progress task did not report");
4835 assert!(completed.try_recv().is_err());
4836 }
4837
4838 #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
4839 async fn browser_projection_keeps_only_the_newest_pending_snapshot() {
4840 let (results, mut completed) = tokio::sync::mpsc::channel(4);
4841 let shutdown = tokio_util::sync::CancellationToken::new();
4842 let mut dispatcher = ConversationProjectionDispatcher::with_permits(results, shutdown, 0);
4843
4844 dispatcher.enqueue(materialized_at(1));
4845 dispatcher.enqueue(materialized_at(3));
4846 dispatcher.enqueue(materialized_at(2));
4847 assert_eq!(dispatcher.pending["session-1"].key.ordinal, 3);
4848
4849 dispatcher.permits.add_permits(1);
4850 let first = tokio::time::timeout(Duration::from_secs(1), completed.recv())
4851 .await
4852 .expect("first projection did not complete")
4853 .expect("projection result channel closed");
4854 assert_eq!(first.key.ordinal, 1);
4855 assert!(dispatcher.finish(first, true).is_some());
4856
4857 let second = tokio::time::timeout(Duration::from_secs(1), completed.recv())
4858 .await
4859 .expect("coalesced projection did not complete")
4860 .expect("projection result channel closed");
4861 assert_eq!(second.key.ordinal, 3);
4862 assert!(dispatcher.finish(second, true).is_some());
4863 assert!(dispatcher.pending.is_empty());
4864 assert!(dispatcher.in_flight.is_empty());
4865 }
4866
4867 #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
4868 async fn forgotten_projection_cannot_repopulate_after_a_same_cursor_resume() {
4869 let (results, mut completed) = tokio::sync::mpsc::channel(4);
4870 let shutdown = tokio_util::sync::CancellationToken::new();
4871 let mut dispatcher = ConversationProjectionDispatcher::with_permits(results, shutdown, 0);
4872 let snapshot = materialized_at(7);
4873
4874 dispatcher.enqueue(snapshot.clone());
4875 dispatcher.forget("session-1");
4876 dispatcher.enqueue(snapshot);
4877 dispatcher.permits.add_permits(1);
4878
4879 let old = tokio::time::timeout(Duration::from_secs(1), completed.recv())
4880 .await
4881 .expect("old projection did not complete")
4882 .expect("projection result channel closed");
4883 assert!(dispatcher.finish(old, true).is_none());
4884
4885 let current = tokio::time::timeout(Duration::from_secs(1), completed.recv())
4886 .await
4887 .expect("resumed projection did not complete")
4888 .expect("projection result channel closed");
4889 assert!(dispatcher.finish(current, true).is_some());
4890 assert!(dispatcher.in_flight.is_empty());
4891 }
4892}