Skip to main content

mj_controller/
server_runtime.rs

1//! The phone-oriented remote-control server: its HTTP surface, the controller
2//! actions phones request, and the concurrency limits that keep them safe.
3
4use std::net::{Ipv4Addr, SocketAddr};
5use std::path::PathBuf;
6use std::sync::Arc;
7use std::sync::atomic::{AtomicBool, Ordering};
8use std::time::{Duration, Instant};
9
10use anyhow::{Context, Result, bail};
11mod api;
12mod api_activity;
13
14use mj_core::config::{Config, HarnessProfile, PhoneConfig, is_bare_project_target};
15use mj_core::remote_git::{default_branch, display_url, resolve_repository};
16use mj_core::state::{MaterializedSession, ProjectSourceIdentity, SessionRecord, State};
17
18use crate::controller::Controller;
19use crate::quota::ProfileQuota;
20use crate::server::{
21    ActionOutcome, BackgroundTaskStopFailure, BackgroundTaskStopRequest, BrowserTranscript,
22    ControllerAction, ControllerRequest, MovePreparationRequest, PreflightFailure,
23    ReadReceiptRequest, ResumeQueueDisposition, ServerOptions, ViewerActivityDetails,
24    ViewerActivityKind, ViewerBackgroundTask, ViewerMoveRecovery, ViewerQueuedPrompt, ViewerQuota,
25    ViewerSnapshot, ViewerUserShell,
26};
27use crate::session_manager::{SessionManagerChannels, SessionManagerControl, new_command_id};
28use crate::tailscale::TailscaleTls;
29#[cfg(test)]
30use crate::targets::ProcessExecutor;
31use crate::targets::{CancellableProcessExecutor, CommandExecutor};
32use crate::worker_client::CredentialSyncCoordinator;
33use mj_core::relay::RelayCommand;
34use mj_core::workspace::WorkspaceRecord;
35
36use crate::controller::config_only_controller;
37use crate::daemon::{
38    CreateSessionControl, CreateSessionRequest, ResumeSessionRequest, RuntimeState,
39};
40use crate::pollers::{
41    CredentialSyncNotices, CredentialSyncSignalTracker, QUOTA_STALE_AFTER, QuotaRefreshBatch,
42    QuotaUpdate, apply_worker_record_update, credential_sync_targets, dashboard_worker_targets,
43    projected_queued_prompts, queued_prompt_projection, quota_refresh_profiles,
44    schedule_due_credential_syncs, spawn_quota_refresher,
45};
46
47#[derive(Debug, Clone)]
48pub struct ServerArgs {
49    bind: String,
50    tailscale_detect: bool,
51    tls_cert: Option<PathBuf>,
52    tls_key: Option<PathBuf>,
53}
54
55impl From<&PhoneConfig> for ServerArgs {
56    fn from(config: &PhoneConfig) -> Self {
57        Self {
58            bind: config.bind.clone(),
59            tailscale_detect: config.tailscale_detect,
60            tls_cert: config.tls_cert.clone(),
61            tls_key: config.tls_key.clone(),
62        }
63    }
64}
65
66const TAILSCALE_COMMAND_TIMEOUT: Duration = Duration::from_secs(120);
67const TAILSCALE_RENEW_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60);
68
69/// Transcript projection parses every stored ACP content chunk. Keep that work
70/// off the control task, and allow only a small number of projections to use
71/// the blocking pool at once so a burst of active sessions cannot turn the
72/// pool into an unbounded queue.
73const MAX_CONCURRENT_CONVERSATION_PROJECTIONS: usize = 2;
74const CONVERSATION_PROJECTION_CHANNEL_CAPACITY: usize = 16;
75
76#[derive(Debug, Clone, PartialEq, Eq)]
77struct ConversationProjectionKey {
78    ordinal: u64,
79    digest: String,
80}
81
82impl ConversationProjectionKey {
83    fn of(materialized: &MaterializedSession) -> Self {
84        Self {
85            ordinal: materialized.applied_event_ordinal,
86            digest: materialized.applied_event_digest.clone(),
87        }
88    }
89
90    /// Event ordinals are monotonic. A changed digest at one ordinal is also a
91    /// new projection, which preserves the integrity-repair path without
92    /// relying on string ordering for digests.
93    fn is_newer_than(&self, other: &Self) -> bool {
94        self.ordinal > other.ordinal
95            || (self.ordinal == other.ordinal && self.digest != other.digest)
96    }
97}
98
99struct ConversationProjectionRequest {
100    materialized: MaterializedSession,
101    key: ConversationProjectionKey,
102    generation: u64,
103}
104
105struct ConversationProjectionResult {
106    session_id: String,
107    key: ConversationProjectionKey,
108    generation: u64,
109    result: std::result::Result<BrowserTranscript, String>,
110}
111
112/// Owns the one-at-a-time/latest-state scheduling for each session. The
113/// control loop remains the sole owner of these maps; background tasks only
114/// return completed browser projections through `results`.
115struct ConversationProjectionDispatcher {
116    in_flight: std::collections::BTreeMap<String, (ConversationProjectionKey, u64)>,
117    pending: std::collections::BTreeMap<String, ConversationProjectionRequest>,
118    completed: std::collections::BTreeMap<String, ConversationProjectionKey>,
119    generations: std::collections::BTreeMap<String, u64>,
120    permits: Arc<tokio::sync::Semaphore>,
121    results: tokio::sync::mpsc::Sender<ConversationProjectionResult>,
122    shutdown: tokio_util::sync::CancellationToken,
123}
124
125impl ConversationProjectionDispatcher {
126    fn new(
127        results: tokio::sync::mpsc::Sender<ConversationProjectionResult>,
128        shutdown: tokio_util::sync::CancellationToken,
129    ) -> Self {
130        Self {
131            in_flight: std::collections::BTreeMap::new(),
132            pending: std::collections::BTreeMap::new(),
133            completed: std::collections::BTreeMap::new(),
134            generations: std::collections::BTreeMap::new(),
135            permits: Arc::new(tokio::sync::Semaphore::new(
136                MAX_CONCURRENT_CONVERSATION_PROJECTIONS,
137            )),
138            results,
139            shutdown,
140        }
141    }
142
143    #[cfg(test)]
144    fn with_permits(
145        results: tokio::sync::mpsc::Sender<ConversationProjectionResult>,
146        shutdown: tokio_util::sync::CancellationToken,
147        permits: usize,
148    ) -> Self {
149        let mut dispatcher = Self::new(results, shutdown);
150        dispatcher.permits = Arc::new(tokio::sync::Semaphore::new(permits));
151        dispatcher
152    }
153
154    /// Queue a session's newest durable view. At most one request is running
155    /// and one newer request is retained for any given session.
156    fn enqueue(&mut self, materialized: MaterializedSession) {
157        let session_id = materialized.session_id.clone();
158        let key = ConversationProjectionKey::of(&materialized);
159        if self
160            .completed
161            .get(&session_id)
162            .is_some_and(|completed| !key.is_newer_than(completed))
163        {
164            return;
165        }
166        let generation = *self.generations.entry(session_id.clone()).or_default();
167        if let Some((in_flight, in_flight_generation)) = self.in_flight.get(&session_id) {
168            if generation != *in_flight_generation || key.is_newer_than(in_flight) {
169                let replace = self.pending.get(&session_id).is_none_or(|pending| {
170                    pending.generation != generation || key.is_newer_than(&pending.key)
171                });
172                if replace {
173                    self.pending.insert(
174                        session_id,
175                        ConversationProjectionRequest {
176                            materialized,
177                            key,
178                            generation,
179                        },
180                    );
181                }
182            }
183            return;
184        }
185        self.in_flight
186            .insert(session_id.clone(), (key.clone(), generation));
187        self.start(ConversationProjectionRequest {
188            materialized,
189            key,
190            generation,
191        });
192    }
193
194    /// Finish one task and, when the session is still active, immediately
195    /// launch the newest coalesced request. Returning `None` means the task
196    /// failed or no longer belongs to the current in-flight request.
197    fn finish(
198        &mut self,
199        result: ConversationProjectionResult,
200        session_active: bool,
201    ) -> Option<(String, ConversationProjectionKey, BrowserTranscript)> {
202        let expected = self.in_flight.remove(&result.session_id);
203        if expected.as_ref() != Some(&(result.key.clone(), result.generation)) {
204            tracing::warn!(
205                session_id = %result.session_id,
206                "discarding an out-of-date browser transcript projection"
207            );
208            return None;
209        }
210        let session_id = result.session_id;
211        let key = result.key;
212        let current_generation = self
213            .generations
214            .get(&session_id)
215            .copied()
216            .unwrap_or_default();
217        let current = result.generation == current_generation;
218        let projected = match result.result {
219            Ok(transcript) if session_active && current => {
220                self.completed.insert(session_id.clone(), key.clone());
221                Some((session_id.clone(), key, transcript))
222            }
223            Ok(_) => {
224                // An inactive session, or a result from an earlier lifecycle
225                // generation, must not resurrect a conversation. Its next
226                // active update gets a fresh generation.
227                self.completed.remove(&session_id);
228                None
229            }
230            Err(error) => {
231                tracing::warn!(
232                    session_id = %session_id,
233                    "browser transcript projection failed: {error}"
234                );
235                None
236            }
237        };
238        if session_active {
239            if let Some(pending) = self.pending.remove(&session_id) {
240                self.enqueue(pending.materialized);
241            }
242        } else {
243            self.pending.remove(&session_id);
244            self.completed.remove(&session_id);
245        }
246        projected
247    }
248
249    /// Drop queued/completed state after a controller reload removes a
250    /// session. An in-flight task is allowed to finish; `finish` receives the
251    /// current active-state guard and discards its result.
252    fn forget(&mut self, session_id: &str) {
253        self.pending.remove(session_id);
254        self.completed.remove(session_id);
255        let generation = self.generations.entry(session_id.to_owned()).or_default();
256        *generation = generation.wrapping_add(1);
257    }
258
259    fn session_ids(&self) -> std::collections::BTreeSet<String> {
260        self.in_flight
261            .keys()
262            .chain(self.pending.keys())
263            .chain(self.completed.keys())
264            .cloned()
265            .collect()
266    }
267
268    fn start(&self, request: ConversationProjectionRequest) {
269        let session_id = request.materialized.session_id.clone();
270        let key = request.key;
271        let generation = request.generation;
272        let permits = Arc::clone(&self.permits);
273        let results = self.results.clone();
274        let shutdown = self.shutdown.clone();
275        tokio::spawn(async move {
276            let result = match tokio::select! {
277                _ = shutdown.cancelled() => return,
278                result = permits.acquire_owned() => result,
279            } {
280                Ok(permit) => {
281                    let projection = tokio::task::spawn_blocking(move || {
282                        mj_client::transcript::materialized_browser_transcript(
283                            &request.materialized,
284                        )
285                    })
286                    .await;
287                    drop(permit);
288                    projection
289                        .map_err(|error| format!("transcript projection task failed: {error}"))
290                }
291                Err(error) => Err(format!("transcript projection worker stopped: {error}")),
292            };
293            let message = ConversationProjectionResult {
294                session_id,
295                key,
296                generation,
297                result,
298            };
299            tokio::select! {
300                _ = shutdown.cancelled() => {}
301                result = results.send(message) => {
302                    if let Err(error) = result {
303                        tracing::debug!(%error, "browser transcript projection result dropped after server shutdown");
304                    }
305                }
306            }
307        });
308    }
309}
310
311struct ResolvedServerArgs {
312    bind: SocketAddr,
313    viewer_url: String,
314    tls_files: Option<(PathBuf, PathBuf)>,
315    tailscale: Option<TailscaleTls>,
316    fallback_reason: Option<String>,
317}
318
319async fn resolve_server_args(
320    args: ServerArgs,
321    termination: tokio_util::sync::CancellationToken,
322) -> Result<ResolvedServerArgs> {
323    let configured_bind: SocketAddr = args.bind.parse().context("parse web viewer bind address")?;
324    match (args.tls_cert, args.tls_key) {
325        (Some(cert), Some(key)) => {
326            let scheme = "https";
327            return Ok(ResolvedServerArgs {
328                bind: configured_bind,
329                viewer_url: format!("{scheme}://{configured_bind}"),
330                tls_files: Some((cert, key)),
331                tailscale: None,
332                fallback_reason: None,
333            });
334        }
335        (None, None) => {}
336        _ => bail!("web viewer TLS requires both a certificate and private key"),
337    }
338
339    if !args.tailscale_detect {
340        return Ok(loopback_server_args(
341            configured_bind,
342            Some("automatic Tailscale detection is disabled".into()),
343        ));
344    }
345
346    let tls_root = mj_core::config::data_dir().join("viewer");
347    let prepared = run_tailscale_blocking(termination.clone(), move |executor| {
348        crate::tailscale::prepare_tailscale_tls(&tls_root, executor)
349    })
350    .await;
351    match prepared {
352        Ok(tailscale) => {
353            let bind = tailscale_bind(configured_bind);
354            let viewer_url = format!(
355                "https://{}:{}",
356                tailscale.cert_domain(),
357                configured_bind.port()
358            );
359            Ok(ResolvedServerArgs {
360                bind,
361                viewer_url,
362                tls_files: Some((
363                    tailscale.cert_path().to_owned(),
364                    tailscale.key_path().to_owned(),
365                )),
366                tailscale: Some(tailscale),
367                fallback_reason: None,
368            })
369        }
370        Err(error) if termination.is_cancelled() => Err(error),
371        Err(error) => {
372            let reason = format!("{error:#}");
373            tracing::debug!(error = reason, "Tailscale HTTPS unavailable for web viewer");
374            Ok(loopback_server_args(configured_bind, Some(reason)))
375        }
376    }
377}
378
379fn tailscale_bind(configured_bind: SocketAddr) -> SocketAddr {
380    SocketAddr::from((Ipv4Addr::UNSPECIFIED, configured_bind.port()))
381}
382
383fn loopback_server_args(bind: SocketAddr, fallback_reason: Option<String>) -> ResolvedServerArgs {
384    ResolvedServerArgs {
385        bind,
386        viewer_url: format!("http://{bind}"),
387        tls_files: None,
388        tailscale: None,
389        fallback_reason,
390    }
391}
392
393async fn run_tailscale_blocking<T>(
394    termination: tokio_util::sync::CancellationToken,
395    operation: impl FnOnce(&CancellableProcessExecutor) -> Result<T> + Send + 'static,
396) -> Result<T>
397where
398    T: Send + 'static,
399{
400    let cancelled = Arc::new(AtomicBool::new(false));
401    let executor_cancelled = cancelled.clone();
402    let mut task = tokio::task::spawn_blocking(move || {
403        let executor = CancellableProcessExecutor::new(executor_cancelled)
404            .with_deadline(TAILSCALE_COMMAND_TIMEOUT);
405        operation(&executor)
406    });
407    tokio::select! {
408        result = &mut task => result.context("Tailscale background task panicked")?,
409        _ = termination.cancelled() => {
410            cancelled.store(true, Ordering::Release);
411            let _ = task.await;
412            bail!("Tailscale operation cancelled during web viewer shutdown")
413        }
414    }
415}
416
417fn spawn_tailscale_cert_renewer(
418    tailscale: TailscaleTls,
419    rustls: axum_server::tls_rustls::RustlsConfig,
420    termination: tokio_util::sync::CancellationToken,
421) -> tokio::task::JoinHandle<()> {
422    tokio::spawn(async move {
423        let mut interval = tokio::time::interval(TAILSCALE_RENEW_INTERVAL);
424        interval.tick().await;
425        loop {
426            tokio::select! {
427                _ = termination.cancelled() => return,
428                _ = interval.tick() => {}
429            }
430            let renewing = tailscale.clone();
431            let result = run_tailscale_blocking(termination.clone(), move |executor| {
432                renewing.renew(executor)
433            })
434            .await;
435            if let Err(error) = result {
436                if !termination.is_cancelled() {
437                    tracing::warn!(
438                        error = format!("{error:#}"),
439                        "Tailscale certificate renewal failed"
440                    );
441                }
442                continue;
443            }
444            if let Err(error) = rustls
445                .reload_from_pem_file(tailscale.cert_path(), tailscale.key_path())
446                .await
447            {
448                tracing::warn!(%error, "could not activate renewed Tailscale certificate");
449            }
450        }
451    })
452}
453
454const MAX_CONCURRENT_PHONE_ACTIONS: usize = 4;
455const MAX_CONCURRENT_BUNDLE_CREATIONS: usize = 4;
456const MAX_CONCURRENT_PREFLIGHTS: usize = 4;
457
458struct PhoneActionStarted {
459    action_id: u64,
460    session: SessionRecord,
461    published: tokio::sync::oneshot::Sender<std::result::Result<(), String>>,
462}
463
464/// The phone replies the control loop still owes.
465///
466/// A phone is answered as soon as its action is admitted, because provisioning,
467/// resume and close run for minutes and a request held open that long dies on a
468/// mobile network. `new` is the one action whose acceptance means more than
469/// admission: the phone has no session id until the provisional session is
470/// published, so its reply is parked here until the loop publishes it — or
471/// until the action ends without ever getting that far.
472#[derive(Default)]
473struct PendingActionReplies(
474    std::collections::BTreeMap<u64, tokio::sync::oneshot::Sender<ActionOutcome>>,
475);
476
477impl PendingActionReplies {
478    fn accept(
479        &mut self,
480        action_id: u64,
481        action: &ControllerAction,
482        reply: tokio::sync::oneshot::Sender<ActionOutcome>,
483    ) {
484        if matches!(action, ControllerAction::New { .. }) {
485            self.0.insert(action_id, reply);
486        } else {
487            if reply.send(ActionOutcome::accepted()).is_err() {
488                tracing::debug!(
489                    action_id,
490                    "phone action acceptance reply dropped after client disconnect"
491                );
492            }
493        }
494    }
495
496    fn resolve(&mut self, action_id: u64, outcome: ActionOutcome) {
497        if let Some(reply) = self.0.remove(&action_id)
498            && reply.send(outcome).is_err()
499        {
500            tracing::debug!(
501                action_id,
502                "phone action completion reply dropped after client disconnect"
503            );
504        }
505    }
506}
507
508/// Admission control for one phone action, run before any work starts so the
509/// answer to the phone never waits on the operation itself. Reports the session
510/// the action occupies, or the outcome that refuses it.
511fn admit_phone_action(
512    action: &ControllerAction,
513    running_actions: usize,
514    active_sessions: &mut std::collections::BTreeSet<String>,
515) -> std::result::Result<Option<String>, ActionOutcome> {
516    let closing = matches!(
517        action,
518        ControllerAction::Close { .. } | ControllerAction::ForceClose { .. }
519    );
520    if !closing && !phone_action_capacity_available(running_actions) {
521        return Err(ActionOutcome::Busy);
522    }
523    let session_id = controller_action_session_id(action);
524    if let Some(session_id) = &session_id
525        && !active_sessions.insert(session_id.clone())
526        && !closing
527    {
528        return Err(ActionOutcome::SessionBusy);
529    }
530    Ok(session_id)
531}
532
533struct ReadReceiptPersisted {
534    session_id: String,
535    result: std::result::Result<u64, String>,
536    reply: tokio::sync::oneshot::Sender<std::result::Result<(), String>>,
537}
538
539struct ControllerReloaded {
540    result: std::result::Result<Controller, String>,
541}
542
543struct BundleCreated {
544    result: std::result::Result<
545        crate::controller::QuickBundleCreation,
546        crate::controller::QuickBundleFailure,
547    >,
548    reply: tokio::sync::oneshot::Sender<std::result::Result<String, crate::server::BundleFailure>>,
549}
550
551struct MovePrepared {
552    result: std::result::Result<mj_core::state::MovePreparation, String>,
553    reply:
554        tokio::sync::oneshot::Sender<std::result::Result<mj_core::state::MovePreparation, String>>,
555}
556
557/// Loads durable controller state without occupying the phone control loop.
558/// The outer task observes blocking-task panics and reports a closed result
559/// channel instead of silently abandoning the refresh.
560fn spawn_controller_reload(completed: tokio::sync::mpsc::UnboundedSender<ControllerReloaded>) {
561    spawn_controller_reload_with(completed, Controller::load);
562}
563
564fn spawn_controller_reload_with(
565    completed: tokio::sync::mpsc::UnboundedSender<ControllerReloaded>,
566    load: impl FnOnce() -> Result<Controller> + Send + 'static,
567) {
568    tokio::spawn(async move {
569        let result = match tokio::task::spawn_blocking(load).await {
570            Ok(result) => result.map_err(|error| format!("{error:#}")),
571            Err(error) => Err(format!("controller reload task failed: {error}")),
572        };
573        if completed.send(ControllerReloaded { result }).is_err() {
574            tracing::debug!("controller reload completed after the phone control loop stopped");
575        }
576    });
577}
578
579fn request_controller_reload(
580    in_flight: &mut bool,
581    requested: &mut bool,
582    completed: &tokio::sync::mpsc::UnboundedSender<ControllerReloaded>,
583) {
584    if *in_flight {
585        *requested = true;
586    } else {
587        *in_flight = true;
588        spawn_controller_reload(completed.clone());
589    }
590}
591
592fn request_daemon_controller_reload(daemon_runtime: Arc<RuntimeState>, reason: &'static str) {
593    tokio::spawn(async move {
594        if let Err(error) = daemon_runtime.reload_controller().await {
595            tracing::warn!(
596                error = format!("{error:#}"),
597                reason,
598                "phone operation could not refresh dashboard controller state"
599            );
600        }
601    });
602}
603
604/// What one phone read receipt actually needs.
605#[derive(Debug, PartialEq, Eq)]
606#[cfg(test)]
607enum ReadReceiptPlan {
608    UnknownSession,
609    /// The cursor has not advanced, so the receipt needs no work at all.
610    AlreadyRead,
611    /// The cursor advanced: persist it, then refresh the snapshot.
612    Persist,
613}
614
615#[cfg(test)]
616fn plan_read_receipt(state: &State, session_id: &str, through: u64) -> ReadReceiptPlan {
617    let Some(session) = state.sessions.get(session_id) else {
618        return ReadReceiptPlan::UnknownSession;
619    };
620    if through > session.viewed_through_event_ordinal {
621        ReadReceiptPlan::Persist
622    } else {
623        ReadReceiptPlan::AlreadyRead
624    }
625}
626
627/// Record a persisted receipt in the in-memory projection, reporting whether
628/// the cursor moved. That is exactly when the snapshot revision has to move,
629/// so surfaces showing unread state refresh and nothing else does.
630#[cfg(test)]
631fn apply_read_receipt(state: &mut State, session_id: &str, receipt: u64) -> bool {
632    let Some(session) = state.sessions.get_mut(session_id) else {
633        return false;
634    };
635    if receipt <= session.viewed_through_event_ordinal {
636        return false;
637    }
638    session.viewed_through_event_ordinal = receipt;
639    true
640}
641
642#[derive(Clone)]
643struct PhoneActionControl {
644    cancelled: Arc<AtomicBool>,
645    create: Option<CreateSessionControl>,
646}
647
648impl PhoneActionControl {
649    fn for_action(action: &ControllerAction) -> Self {
650        let create =
651            matches!(action, ControllerAction::New { .. }).then(CreateSessionControl::default);
652        let cancelled = create.as_ref().map_or_else(
653            || Arc::new(AtomicBool::new(false)),
654            |control| control.cancelled.clone(),
655        );
656        Self { cancelled, create }
657    }
658
659    fn request_cancel(&self) -> bool {
660        let accepted = self.create.as_ref().map_or_else(
661            || {
662                self.cancelled
663                    .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
664                    .is_ok()
665            },
666            |control| control.request_cancel(),
667        );
668        if accepted {
669            self.cancelled.store(true, Ordering::Release);
670        }
671        accepted
672    }
673
674    #[cfg(test)]
675    fn grant_new_commit(&self) -> bool {
676        self.create
677            .as_ref()
678            .is_some_and(|control| control.grant_commit())
679    }
680}
681
682pub async fn run_server(
683    args: ServerArgs,
684    termination: tokio_util::sync::CancellationToken,
685    worker: SessionManagerChannels,
686    daemon_runtime: Arc<RuntimeState>,
687    mut workspace_updates: tokio::sync::watch::Receiver<Vec<WorkspaceRecord>>,
688) -> Result<()> {
689    let resolved = resolve_server_args(args, termination.clone()).await?;
690    let bind = resolved.bind;
691    let mut controller = Controller::load()?;
692    let mut daemon_revisions = daemon_runtime.revisions();
693    daemon_revisions.borrow_and_update();
694    let mut phone_workspaces = workspace_updates.borrow_and_update().clone();
695    let mut quotas = std::collections::BTreeMap::new();
696    let subagent_quota_reports = Arc::new(std::sync::Mutex::new(quotas.clone()));
697    let (quota_profiles_tx, mut quota_updates_rx) = spawn_quota_refresher();
698    let mut quota_batch = QuotaRefreshBatch::default();
699    let mut published_quota_profiles = std::collections::BTreeMap::new();
700    republish_quota_profiles(
701        &controller,
702        &mut published_quota_profiles,
703        &mut quota_batch,
704        &quota_profiles_tx,
705    );
706    let mut revision = daemon_runtime.allocate_revision();
707    let mut conversations = std::collections::BTreeMap::new();
708    let mut queued_prompts = projected_queued_prompts(&controller)?;
709    let mut active_user_shells = std::collections::BTreeMap::new();
710    let mut pending_elicitations = std::collections::BTreeMap::new();
711    let mut prompt_images = std::collections::BTreeSet::new();
712    let mut operational = std::collections::BTreeMap::new();
713    let mut materialized_activity = load_materialized_activity(&controller).await?;
714    let mut project_sources = PhoneProjectSources::default();
715    let (records, lifecycles) = daemon_runtime.session_projection();
716    controller.state.sessions = records;
717    let mut operations = lifecycles
718        .iter()
719        .map(|view| (view.session_id.clone(), viewer_operation(view)))
720        .collect::<std::collections::BTreeMap<_, _>>();
721    let mut move_recoveries = ViewerMoveRecoveries::new();
722    let (move_recovery_tx, mut move_recovery_rx) =
723        tokio::sync::mpsc::unbounded_channel::<Result<ViewerMoveRecoveries, String>>();
724    let mut move_recovery_load_in_flight = false;
725    let mut launch_failures = Vec::new();
726    // What the capacity poller last said, per probe target. The projection is
727    // built from this on every publish rather than being accumulated, so a
728    // target that disappears from the configuration disappears from the page.
729    let mut capacity_state: std::collections::BTreeMap<String, PhoneCapacity> =
730        std::collections::BTreeMap::new();
731    let (capacity_targets_tx, capacity_triggers_tx, mut capacity_updates_rx) =
732        crate::pollers::spawn_dashboard_capacity_poller();
733    let (snapshot_tx, snapshot_rx) = tokio::sync::watch::channel(viewer_snapshot(
734        &controller,
735        &phone_workspaces,
736        &quotas,
737        &PhoneSessionViews {
738            conversations: &conversations,
739            queued_prompts: &queued_prompts,
740            active_user_shells: &active_user_shells,
741            pending_elicitations: &pending_elicitations,
742            prompt_images: &prompt_images,
743            operational: &operational,
744            materialized_activity: &materialized_activity,
745            project_sources: &project_sources,
746            operations: &operations,
747            move_recoveries: &move_recoveries,
748            capacity: &viewer_capacity(&capacity_state),
749            launch_failures: &launch_failures,
750            reviews: &review_views(&daemon_runtime),
751        },
752        revision,
753    ));
754    let (conversation_tx, conversation_rx) = tokio::sync::watch::channel(conversations.clone());
755    let (action_tx, mut action_rx) = tokio::sync::mpsc::channel(32);
756    let (bundle_tx, mut bundle_rx) = tokio::sync::mpsc::channel(16);
757    let (receipt_tx, mut receipt_rx) = tokio::sync::mpsc::channel(32);
758    let (preflight_tx, mut preflight_rx) = tokio::sync::mpsc::channel(32);
759    let (move_preparation_tx, mut move_preparation_rx) = tokio::sync::mpsc::channel(32);
760    let (client_state_tx, mut client_state_rx) = tokio::sync::mpsc::channel(64);
761    let (dictation_tx, mut dictation_rx) =
762        tokio::sync::mpsc::channel::<crate::dictation::DictationRequest>(8);
763    let (background_task_stop_tx, mut background_task_stop_rx) =
764        tokio::sync::mpsc::channel::<BackgroundTaskStopRequest>(32);
765    let SessionManagerChannels {
766        targets: worker_targets_tx,
767        control: worker_commands_tx,
768        updates: mut worker_updates_rx,
769        shutdown: worker_shutdown,
770    } = worker;
771    worker_targets_tx.send_replace(dashboard_worker_targets(&controller));
772    publish_capacity_targets(&controller, &capacity_targets_tx, &mut capacity_state);
773    let mut credential_sync = CredentialSyncCoordinator::spawn();
774    let credential_sync_handle = credential_sync.handle();
775    credential_sync_handle.set_targets(credential_sync_targets(&controller));
776    let mut credential_sync_signals = CredentialSyncSignalTracker::default();
777    let mut credential_sync_notices = CredentialSyncNotices::default();
778    // Captured before `options` is moved into the server.
779    let options_session_ttl = crate::server::default_session_ttl();
780    let activity_snapshots = snapshot_rx.clone();
781    let mut options = ServerOptions::new(
782        bind,
783        snapshot_rx,
784        conversation_rx,
785        crate::server::ServerRequests {
786            action_tx,
787            bundle_tx,
788            receipt_tx,
789            preflight_tx,
790            move_preparation_tx,
791            client_state_tx,
792            dictation_tx,
793        },
794    )?;
795    options.set_background_task_stop_tx(background_task_stop_tx);
796    options.shutdown = termination.clone();
797    // Session cookies are stateless, so a per-process key would sign every
798    // phone out on every restart. Delete the key file to sign them out on
799    // purpose.
800    let cookie_key_path = crate::server::cookie_key_path();
801    options.set_cookie_key(crate::server::load_or_create_cookie_key(&cookie_key_path)?)?;
802    // The documented `/api/v1` surface authenticates with a persisted bearer
803    // token and drives sessions through the daemon-side backend.
804    options.set_api_token(crate::server::load_or_create_api_token(
805        &crate::server::api_token_path(),
806    )?);
807    let api_runtime = daemon_runtime.clone();
808    let api_backend = Arc::new(
809        api::ApiBackend::new(
810            worker_commands_tx.client(),
811            Arc::new(move |session_id: &str| api_runtime.session_state(session_id)),
812            daemon_runtime.clone(),
813        )
814        .with_quota_reports(subagent_quota_reports.clone()),
815    );
816    options.set_subagent_backend(api_backend.clone());
817    let renewal_cancellation = termination.child_token();
818    let mut renewal_task = None;
819    if let Some((cert, key)) = resolved.tls_files {
820        let rustls = axum_server::tls_rustls::RustlsConfig::from_pem_file(cert, key)
821            .await
822            .context("load web viewer TLS certificate")?;
823        options.set_tls_config(rustls.clone());
824        if let Some(tailscale) = resolved.tailscale {
825            renewal_task = Some(spawn_tailscale_cert_renewer(
826                tailscale,
827                rustls,
828                renewal_cancellation.clone(),
829            ));
830        }
831    } else if bind.ip().is_loopback() {
832        options.secure_cookie = false;
833    } else {
834        anyhow::bail!("non-loopback web viewer requires TLS");
835    }
836    let fallback_reason = resolved.fallback_reason;
837    let qr_login_url = if fallback_reason.is_none() && resolved.viewer_url.starts_with("https://") {
838        let encoded = url::form_urlencoded::byte_serialize(options.login_token().as_bytes())
839            .collect::<String>();
840        Some(format!(
841            "{}/auth/login?token={encoded}",
842            resolved.viewer_url.trim_end_matches('/')
843        ))
844    } else {
845        None
846    };
847    let ready = crate::server::WebViewerAccess::Ready {
848        viewer_url: resolved.viewer_url,
849        viewer_code: options.viewer_code().to_owned(),
850        qr_login_url,
851        fallback_reason,
852    };
853
854    let serve = crate::web_viewer::serve(options, ready, &daemon_runtime.web_viewer, |access| {
855        daemon_runtime.publish_web_access(access);
856    });
857    let conversation_projection_shutdown = termination.child_token();
858    let control = async {
859        let mut credential_tick = tokio::time::interval(Duration::from_millis(250));
860        // Stored viewer state expires with the authentication that created it.
861        // The sweep is hourly rather than on every request, because it is
862        // housekeeping and nothing waits for it.
863        let mut prune_tick = tokio::time::interval(Duration::from_secs(60 * 60));
864        let client_state_retention = options_session_ttl;
865        let (action_done_tx, mut action_done_rx) = tokio::sync::mpsc::unbounded_channel::<(
866            u64,
867            Option<String>,
868            std::result::Result<(), String>,
869        )>();
870        let (action_started_tx, mut action_started_rx) =
871            tokio::sync::mpsc::unbounded_channel::<PhoneActionStarted>();
872        let (receipt_done_tx, mut receipt_done_rx) =
873            tokio::sync::mpsc::unbounded_channel::<ReadReceiptPersisted>();
874        let (controller_reload_tx, mut controller_reload_rx) =
875            tokio::sync::mpsc::unbounded_channel::<ControllerReloaded>();
876        let (bundle_done_tx, mut bundle_done_rx) =
877            tokio::sync::mpsc::unbounded_channel::<BundleCreated>();
878        let (move_prepared_tx, mut move_prepared_rx) =
879            tokio::sync::mpsc::unbounded_channel::<MovePrepared>();
880        let mut dictation_jobs = tokio::task::JoinSet::new();
881        let mut bundle_jobs = tokio::task::JoinSet::new();
882        let mut preflight_jobs = tokio::task::JoinSet::new();
883        let mut move_preparation_jobs = tokio::task::JoinSet::new();
884        let mut move_recovery_jobs = tokio::task::JoinSet::new();
885        let mut background_task_stop_jobs = tokio::task::JoinSet::new();
886        let mut background_task_stop_open = true;
887        let mut controller_reload_in_flight = false;
888        let mut controller_reload_requested = false;
889        let mut controller_reload_invalidated = false;
890        let mut pending_action_errors = std::collections::BTreeMap::<String, String>::new();
891        let mut active_actions = std::collections::BTreeSet::new();
892        let mut closing_actions = std::collections::BTreeMap::<String, u64>::new();
893        let mut next_action_id = 0_u64;
894        let mut action_cancellations = std::collections::BTreeMap::<u64, PhoneActionControl>::new();
895        let mut action_sessions = std::collections::BTreeMap::<u64, String>::new();
896        let mut action_replies = PendingActionReplies::default();
897        let mut launch_workspaces = std::collections::BTreeMap::new();
898        let mut subagent_jobs = tokio::task::JoinSet::new();
899        let mut subagent_completion_jobs = tokio::task::JoinSet::new();
900        let mut active_subagent_requests = std::collections::BTreeSet::new();
901        let (conversation_projection_tx, mut conversation_projection_rx) =
902            tokio::sync::mpsc::channel(CONVERSATION_PROJECTION_CHANNEL_CAPACITY);
903        let mut conversation_projections = ConversationProjectionDispatcher::new(
904            conversation_projection_tx,
905            conversation_projection_shutdown.clone(),
906        );
907        let mut quota_updates_open = true;
908        // A feed that ends is not a reason to exit quietly: the phone server
909        // exists to follow sessions, so losing that feed is a named failure
910        // rather than a silent success.
911        let mut failure: Option<anyhow::Error> = None;
912        request_move_recovery_reload(
913            &move_recovery_tx,
914            &mut move_recovery_load_in_flight,
915            &mut move_recovery_jobs,
916        );
917        macro_rules! publish_snapshot {
918            ($revision:expr) => {
919                let (records, lifecycles) = daemon_runtime.session_projection();
920                controller.state.sessions = records;
921                for (session_id, error) in &pending_action_errors {
922                    if let Some(session) = controller.state.sessions.get_mut(session_id)
923                        && session.last_error.is_none()
924                    {
925                        session.last_error = Some(error.clone());
926                    }
927                }
928                operations = lifecycles.iter()
929                    .map(|view| (view.session_id.clone(), viewer_operation(view)))
930                    .collect();
931                if let Err(error) = snapshot_tx.send(viewer_snapshot(
932                    &controller,
933                    &phone_workspaces,
934                    &quotas,
935                    &PhoneSessionViews {
936                        conversations: &conversations,
937                        queued_prompts: &queued_prompts,
938                        active_user_shells: &active_user_shells,
939                        pending_elicitations: &pending_elicitations,
940                        prompt_images: &prompt_images,
941                        operational: &operational,
942                        materialized_activity: &materialized_activity,
943                        project_sources: &project_sources,
944                        operations: &operations,
945                        move_recoveries: &move_recoveries,
946                        capacity: &viewer_capacity(&capacity_state),
947                        launch_failures: &launch_failures,
948                        reviews: &review_views(&daemon_runtime),
949                    },
950                    $revision,
951                )) {
952                    tracing::debug!(revision = $revision, %error, "phone snapshot delivery failed; no viewer is subscribed");
953                }
954            };
955        }
956        loop {
957            project_sources.synchronize(&controller);
958            tokio::select! {
959                _ = termination.cancelled() => break,
960                request = background_task_stop_rx.recv(), if background_task_stop_open => {
961                    let Some(request) = request else {
962                        background_task_stop_open = false;
963                        tracing::warn!("background-task stop request feed closed while the phone server was running");
964                        continue;
965                    };
966                    let session_control = worker_commands_tx.clone();
967                    background_task_stop_jobs.spawn(async move {
968                        let result = match session_control.session(&request.session_id).await {
969                            Ok(session) => session
970                                .client()
971                                .stop_background_task(request.background_task_id.clone())
972                                .await
973                                .map_err(|error| {
974                                    tracing::warn!(
975                                        session_id = %request.session_id,
976                                        background_task_id = %request.background_task_id,
977                                        %error,
978                                        "provider rejected background-task stop"
979                                    );
980                                    BackgroundTaskStopFailure::Provider
981                                }),
982                            Err(error) => {
983                                tracing::warn!(
984                                    session_id = %request.session_id,
985                                    %error,
986                                    "could not resolve live session for background-task stop"
987                                );
988                                Err(BackgroundTaskStopFailure::SessionUnavailable)
989                            }
990                        };
991                        if request.reply.send(result).is_err() {
992                            tracing::debug!(
993                                session_id = %request.session_id,
994                                background_task_id = %request.background_task_id,
995                                "background-task stop result dropped after viewer disconnected"
996                            );
997                        }
998                    });
999                }
1000                completed = background_task_stop_jobs.join_next(), if !background_task_stop_jobs.is_empty() => {
1001                    if let Some(Err(error)) = completed {
1002                        tracing::error!(%error, "background-task stop task failed unexpectedly");
1003                    }
1004                }
1005                move_reloaded = move_recovery_rx.recv() => {
1006                    let Some(result) = move_reloaded else {
1007                        failure = feed_stopped(
1008                            termination.is_cancelled(),
1009                            "the Move recovery projection stopped while the phone server was running",
1010                        );
1011                        break;
1012                    };
1013                    move_recovery_load_in_flight = false;
1014                    match result {
1015                        Ok(recoveries) => {
1016                            move_recoveries = recoveries;
1017                            revision = daemon_runtime.allocate_revision();
1018                            publish_snapshot!(revision);
1019                        }
1020                        Err(error) => tracing::warn!(%error, "could not refresh Move recovery projection"),
1021                    }
1022                }
1023                resolved = project_sources.jobs.join_next(), if !project_sources.jobs.is_empty() => {
1024                    match resolved {
1025                        Some(Ok(resolved)) => project_sources.complete(resolved),
1026                        Some(Err(error)) => {
1027                            failure = Some(anyhow::anyhow!("web project source task failed: {error}"));
1028                            break;
1029                        }
1030                        None => unreachable!("project source jobs were not empty"),
1031                    }
1032                    revision = daemon_runtime.allocate_revision();
1033                    publish_snapshot!(revision);
1034                }
1035                changed = daemon_revisions.changed() => {
1036                    if changed.is_err() {
1037                        failure = feed_stopped(
1038                            termination.is_cancelled(),
1039                            "the daemon stopped publishing runtime revisions to the phone server",
1040                        );
1041                        break;
1042                    }
1043                    daemon_revisions.borrow_and_update();
1044                    revision = daemon_runtime.allocate_revision();
1045                    publish_snapshot!(revision);
1046                    request_controller_reload(
1047                        &mut controller_reload_in_flight,
1048                        &mut controller_reload_requested,
1049                        &controller_reload_tx,
1050                    );
1051                }
1052                changed = workspace_updates.changed() => {
1053                    if changed.is_err() {
1054                        failure = feed_stopped(
1055                            termination.is_cancelled(),
1056                            "the daemon stopped publishing workspaces to the phone server",
1057                        );
1058                        break;
1059                    }
1060                    phone_workspaces = workspace_updates.borrow_and_update().clone();
1061                    revision = daemon_runtime.allocate_revision();
1062                    publish_snapshot!(revision);
1063                }
1064                update = capacity_updates_rx.recv() => {
1065                    let Some(update) = update else {
1066                        failure = feed_stopped(termination.is_cancelled(), "the capacity poller stopped while the phone server was running");
1067                        break;
1068                    };
1069                    if let Some(entry) = capacity_state.get_mut(&update.target_id) {
1070                        entry.refreshing = false;
1071                        entry.sampled_at_epoch_seconds = Some(update.sampled_at_epoch_seconds);
1072                        match update.result {
1073                            Ok(usage) => {
1074                                // A fleet with nothing running reports no
1075                                // figures, and that is an answer rather than a
1076                                // failure.
1077                                entry.on_demand = usage.is_none();
1078                                entry.usage = usage;
1079                                entry.failed = false;
1080                            }
1081                            // The last good reading stays on screen beside the
1082                            // failure: one failed probe is not a reason to
1083                            // forget what the machine was doing.
1084                            Err(_) => entry.failed = true,
1085                        }
1086                    }
1087                    revision = daemon_runtime.allocate_revision();
1088                    publish_snapshot!(revision);
1089                }
1090                update = quota_updates_rx.recv(), if quota_updates_open => {
1091                    match update {
1092                        Some(QuotaUpdate::Report(outcome)) => {
1093                            if outcome.credentials_changed {
1094                                credential_sync_handle
1095                                    .sync_profile_now(&outcome.report.profile_id, None);
1096                            }
1097                            quotas.insert(outcome.report.profile_id.clone(), outcome.report.clone());
1098                            subagent_quota_reports
1099                                .lock()
1100                                .expect("sub-agent quota reports lock poisoned")
1101                                .insert(outcome.report.profile_id.clone(), outcome.report);
1102                            revision = daemon_runtime.allocate_revision();
1103                            publish_snapshot!(revision);
1104                        }
1105                        Some(QuotaUpdate::Refreshing { .. } | QuotaUpdate::Finished { .. }) => {}
1106                        None => {
1107                            quota_updates_open = false;
1108                            tracing::warn!("quota refresher stopped while the phone server is running");
1109                        }
1110                    }
1111                }
1112                projected = conversation_projection_rx.recv() => {
1113                    let Some(projected) = projected else {
1114                        failure = feed_stopped(
1115                            termination.is_cancelled(),
1116                            "the browser transcript projection feed stopped",
1117                        );
1118                        break;
1119                    };
1120                    let session_id = projected.session_id.clone();
1121                    let session_active = controller
1122                        .state
1123                        .sessions
1124                        .get(&session_id)
1125                        .is_some_and(|session| session.state.is_active());
1126                    if !session_active {
1127                        // Invalidate a late result before it can be applied or
1128                        // launch another queued projection. A session that is
1129                        // resumed later gets a new generation from its next
1130                        // worker snapshot.
1131                        conversation_projections.forget(&session_id);
1132                    }
1133                    if let Some((session_id, _key, transcript)) =
1134                        conversation_projections.finish(projected, session_active)
1135                    {
1136                        conversations.insert(session_id, transcript);
1137                        revision = daemon_runtime.allocate_revision();
1138                        conversation_tx.send_replace(conversations.clone());
1139                        publish_snapshot!(revision);
1140                    } else if !session_active && conversations.remove(&session_id).is_some() {
1141                        // Controller reload normally removes inactive rows
1142                        // first, but this also covers a worker result racing
1143                        // that reload and keeps the viewer from seeing a
1144                        // conversation for a dead session.
1145                        revision = daemon_runtime.allocate_revision();
1146                        conversation_tx.send_replace(conversations.clone());
1147                        publish_snapshot!(revision);
1148                    }
1149                    // SessionManagerUpdates has a synchronous pending fast
1150                    // path. Yield after each completion so a hot stream of
1151                    // updates cannot monopolize this runtime worker.
1152                    tokio::task::yield_now().await;
1153                }
1154                update = worker_updates_rx.recv() => {
1155                    let Some(update) = update else {
1156                        failure = feed_stopped(termination.is_cancelled(), "the session manager stopped; the phone server can no longer follow sessions");
1157                        break;
1158                    };
1159                    if let Some(snapshot) = update.view.snapshot.as_ref()
1160                        && let Some(session) = controller.state.sessions.get(&update.session_id)
1161                        && let Some(signal) = snapshot.latest_credential_sync_signal.clone()
1162                    {
1163                        credential_sync_signals.observe(
1164                            &update.session_id,
1165                            &session.last_profile,
1166                            signal,
1167                        );
1168                    }
1169                    schedule_due_credential_syncs(
1170                        &mut credential_sync_signals,
1171                        &credential_sync_handle,
1172                        Instant::now(),
1173                    );
1174                    apply_worker_record_update(&mut controller, &update);
1175                    if let Some(snapshot) = update.view.snapshot {
1176                        for request in snapshot.subagent_requests.iter().cloned() {
1177                            let identity = (update.session_id.clone(), request.request_id.clone());
1178                            if !active_subagent_requests.insert(identity.clone()) {
1179                                continue;
1180                            }
1181                            let backend = api_backend.clone();
1182                            let runtime = daemon_runtime.clone();
1183                            let parent_session_id = update.session_id.clone();
1184                            subagent_jobs.spawn(async move {
1185                                let result = backend
1186                                    .execute_subagent_tool(parent_session_id.clone(), request)
1187                                    .await;
1188                                let outcome = async {
1189                                    // The result reaches the model as the tool
1190                                    // call's own answer: completing the request
1191                                    // unblocks the worker socket the harness is
1192                                    // waiting on. It is not injected as a turn.
1193                                    let handle = runtime
1194                                        .workspace_session_handle(&parent_session_id)
1195                                        .await?;
1196                                    let mut lease = handle.lease_connection().await?;
1197                                    lease
1198                                        .connection_mut()
1199                                        .complete_subagent_request(result)
1200                                        .await?;
1201                                    lease.release();
1202                                    anyhow::Ok(())
1203                                }
1204                                .await;
1205                                (identity, outcome)
1206                            });
1207                        }
1208                        if let Some(relation) = controller.state.subagents.get_mut(&update.session_id)
1209                            && matches!(snapshot.materialized.execution, mj_core::state::MaterializedExecutionState::Idle)
1210                            && let Some(outcome) = snapshot.materialized.last_turn_outcome.as_ref()
1211                            && relation.delivered_turn != Some(outcome.completed_ordinal)
1212                        {
1213                            let turn = outcome.completed_ordinal;
1214                            let output = snapshot
1215                                .materialized
1216                                .transcript
1217                                .iter()
1218                                .rev()
1219                                .find_map(|item| match &item.body {
1220                                    mj_core::transcript::TranscriptBody::Agent { chunks, .. }
1221                                        if item.position >= outcome.turn_start_position.unwrap_or(0) =>
1222                                    {
1223                                        Some(mj_core::transcript::materialized_chunks_text(chunks))
1224                                    }
1225                                    _ => None,
1226                                })
1227                                .unwrap_or_else(|| "The sub-agent completed without a final text response.".to_owned());
1228                            let child_id = relation.child_session_id.clone();
1229                            let parent_id = relation.parent_session_id.clone();
1230                            let task_name = relation.task_name.clone();
1231                            let outcome_name = format!("{:?}", outcome.outcome).to_lowercase();
1232                            relation.delivered_turn = Some(turn);
1233                            let backend = api_backend.clone();
1234                            subagent_completion_jobs.spawn(async move {
1235                                let result = async {
1236                                    backend
1237                                        .deliver_subagent_completion(
1238                                            parent_id,
1239                                            &child_id,
1240                                            &task_name,
1241                                            turn,
1242                                            &outcome_name,
1243                                            &output,
1244                                        )
1245                                        .await?;
1246                                    tokio::task::spawn_blocking({
1247                                        let child_id = child_id.clone();
1248                                        move || crate::database::mark_subagent_turn_delivered(&child_id, turn)
1249                                    })
1250                                    .await??;
1251                                    anyhow::Ok(())
1252                                }
1253                                .await;
1254                                (child_id, turn, result)
1255                            });
1256                        }
1257                        if snapshot.operational.native_session_is_ready()
1258                            && operational.get(&update.session_id).is_none_or(|old: &mj_core::relay::RelayOperationalState| old.config_options != snapshot.operational.config_options)
1259                            && let Some(session) = controller.state.sessions.get(&update.session_id)
1260                            && matches!(session.target, Some(mj_core::state::TargetLocator::LocalBare { .. } | mj_core::state::TargetLocator::SshBare { .. } | mj_core::state::TargetLocator::AwsEc2 { .. }))
1261                            && let Some(build) = snapshot.worker_build.clone()
1262                        {
1263                            let profile = session.last_profile.clone();
1264                            let state = snapshot.operational.clone();
1265                            tokio::spawn(async move {
1266                                if let Err(error) = crate::controller::profile_config::observe(profile, build, state).await {
1267                                    tracing::warn!(%error, "could not cache observed profile choices");
1268                                }
1269                            });
1270                        }
1271                        let materialized = snapshot.materialized;
1272                        let operational_state = snapshot.operational;
1273                        materialized_activity.insert(
1274                            update.session_id.clone(),
1275                            materialized.last_activity_at_ms,
1276                        );
1277                        let queued = queued_prompt_projection(&materialized);
1278                        let pending = materialized.pending_elicitations.clone();
1279                        let active_shells = operational_state.active_user_shells.clone();
1280                        let prompt_images_supported =
1281                            agent_accepts_prompt_images(&operational_state);
1282                        active_user_shells.insert(
1283                            update.session_id.clone(),
1284                            active_shells,
1285                        );
1286                        conversation_projections.enqueue(materialized);
1287                        queued_prompts.insert(
1288                            update.session_id.clone(),
1289                            queued,
1290                        );
1291                        pending_elicitations.insert(
1292                            update.session_id.clone(),
1293                            pending,
1294                        );
1295                        if prompt_images_supported {
1296                            prompt_images.insert(update.session_id.clone());
1297                        } else {
1298                            prompt_images.remove(&update.session_id);
1299                        }
1300                        operational.insert(
1301                            update.session_id.clone(),
1302                            operational_state,
1303                        );
1304                        revision = daemon_runtime.allocate_revision();
1305                        conversation_tx.send_replace(conversations.clone());
1306                        publish_snapshot!(revision);
1307                    }
1308                    // The session update receiver can return pending entries
1309                    // without touching Tokio's budgeted receive operation.
1310                    // Give HTTP/TLS tasks a scheduling opportunity after each
1311                    // update even when the worker is publishing continuously.
1312                    tokio::task::yield_now().await;
1313                }
1314                completed = subagent_jobs.join_next(), if !subagent_jobs.is_empty() => {
1315                    match completed {
1316                        Some(Ok((identity, Ok(())))) => {
1317                            active_subagent_requests.remove(&identity);
1318                        }
1319                        Some(Ok((identity, Err(error)))) => {
1320                            active_subagent_requests.remove(&identity);
1321                            tracing::warn!(
1322                                parent_session_id = %identity.0,
1323                                request_id = %identity.1,
1324                                error = %format!("{error:#}"),
1325                                "sub-agent tool request failed"
1326                            );
1327                        }
1328                        Some(Err(error)) => tracing::warn!(%error, "sub-agent tool task panicked"),
1329                        None => {}
1330                    }
1331                }
1332                completed = subagent_completion_jobs.join_next(), if !subagent_completion_jobs.is_empty() => {
1333                    match completed {
1334                        Some(Ok((_, _, Ok(())))) => {}
1335                        Some(Ok((child_id, turn, Err(error)))) => {
1336                            if let Some(relation) = controller.state.subagents.get_mut(&child_id)
1337                                && relation.delivered_turn == Some(turn)
1338                            {
1339                                relation.delivered_turn = None;
1340                            }
1341                            tracing::warn!(%child_id, turn, error = %format!("{error:#}"), "could not deliver sub-agent completion");
1342                        }
1343                        Some(Err(error)) => tracing::warn!(%error, "sub-agent completion task panicked"),
1344                        None => {}
1345                    }
1346                }
1347                _ = prune_tick.tick() => {
1348                    // Only rows whose client id names a phone are considered:
1349                    // a terminal client's place in a conversation is not the
1350                    // phone's to expire.
1351                    tokio::spawn(async move {
1352                        let pruned = tokio::task::spawn_blocking(move || {
1353                            crate::database::prune_phone_client_state(client_state_retention)
1354                        })
1355                        .await;
1356                        match pruned {
1357                            Ok(Ok(0)) => {}
1358                            Ok(Ok(rows)) => tracing::debug!(rows, "pruned expired phone viewer state"),
1359                            Ok(Err(error)) => tracing::warn!(%error, "could not prune phone viewer state"),
1360                            Err(error) => tracing::warn!(%error, "phone viewer state pruning task failed"),
1361                        }
1362                    });
1363                }
1364                _ = credential_tick.tick() => {
1365                    schedule_due_credential_syncs(
1366                        &mut credential_sync_signals,
1367                        &credential_sync_handle,
1368                        Instant::now(),
1369                    );
1370                    while let Some(result) = credential_sync.try_result() {
1371                        crate::pollers::log_credential_sync_actions(&result);
1372                        let harness = controller
1373                            .config
1374                            .profiles
1375                            .get(&result.profile_id)
1376                            .map(|profile| profile.kind);
1377                        if let Some(notice) = credential_sync_notices.notice(&result, harness) {
1378                            eprintln!("Mjolnir: {notice}");
1379                        }
1380                    }
1381                }
1382                request = dictation_rx.recv() => {
1383                    let Some(request) = request else {
1384                        failure = feed_stopped(termination.is_cancelled(), "the phone HTTP server stopped delivering dictation requests");
1385                        break;
1386                    };
1387                    let paths = controller.state.sessions.get(&request.session_id).map(|session| {
1388                        crate::dictation::auth_paths(&controller.config, &session.last_profile)
1389                    });
1390                    dictation_jobs.spawn(crate::dictation::execute(
1391                        request, paths, termination.clone(),
1392                    ));
1393                }
1394                job = dictation_jobs.join_next(), if !dictation_jobs.is_empty() => {
1395                    if let Some(Err(error)) = job {
1396                        tracing::warn!(%error, "web dictation task failed");
1397                    }
1398                }
1399                stored = client_state_rx.recv() => {
1400                    let Some(stored) = stored else {
1401                        failure = feed_stopped(termination.is_cancelled(), "the phone HTTP server stopped delivering viewer state requests");
1402                        break;
1403                    };
1404                    // Every one of these touches SQLite, so each runs on its
1405                    // own task. A composer autosaving on a debounce must never
1406                    // be able to stall the loop that follows sessions.
1407                    let workspace_of = |session_id: &str| {
1408                        controller
1409                            .state
1410                            .sessions
1411                            .get(session_id)
1412                            .map(|session| session.workspace_id.clone())
1413                    };
1414                    let bundle_of = |session_id: &str| {
1415                        controller
1416                            .state
1417                            .sessions
1418                            .get(session_id)
1419                            .map(|session| session.bundle_id.clone())
1420                    };
1421                    match stored {
1422                        crate::server::ClientStateRequest::Read { client_id, session_id, reply } => {
1423                            let workspace = workspace_of(&session_id);
1424                            tokio::spawn(async move {
1425                                let answer = tokio::task::spawn_blocking(move || {
1426                                    let workspace = workspace.context("unknown session")?;
1427                                    let state = crate::database::client_session_state(
1428                                        &client_id, &workspace, &session_id,
1429                                    )?;
1430                                    anyhow::Ok(crate::server::ViewerClientState {
1431                                        draft: state.draft,
1432                                        through_event_ordinal: state.through_event_ordinal,
1433                                    })
1434                                })
1435                                .await;
1436                                reply.send(flatten_stored(answer)).ok();
1437                            });
1438                        }
1439                        crate::server::ClientStateRequest::SaveDraft { client_id, session_id, draft, reply } => {
1440                            let workspace = workspace_of(&session_id);
1441                            tokio::spawn(async move {
1442                                let answer = tokio::task::spawn_blocking(move || {
1443                                    let workspace = workspace.context("unknown session")?;
1444                                    crate::database::persist_client_draft(
1445                                        &client_id, &workspace, &session_id, &draft,
1446                                    )
1447                                })
1448                                .await;
1449                                reply.send(flatten_stored(answer)).ok();
1450                            });
1451                        }
1452                        crate::server::ClientStateRequest::MarkWorkspaceRead { client_id, workspace_id, reply } => {
1453                            let sessions = controller
1454                                .state
1455                                .sessions
1456                                .values()
1457                                .filter(|session| session.workspace_id == workspace_id)
1458                                .map(|session| (session.id.clone(), session.viewed_through_event_ordinal))
1459                                .collect::<Vec<_>>();
1460                            tokio::spawn(async move {
1461                                let answer = tokio::task::spawn_blocking(move || {
1462                                    for (session_id, through) in sessions {
1463                                        // A receipt that would move backwards
1464                                        // is not an error; it is a session this
1465                                        // viewer had already read past.
1466                                        crate::database::persist_read_receipt(
1467                                            &client_id, &workspace_id, &session_id, through,
1468                                        )
1469                                        .ok();
1470                                    }
1471                                    anyhow::Ok(())
1472                                })
1473                                .await;
1474                                reply.send(flatten_stored(answer)).ok();
1475                            });
1476                        }
1477                        crate::server::ClientStateRequest::History { session_id, query, scope, reply } => {
1478                            let bundle = bundle_of(&session_id);
1479                            tokio::spawn(async move {
1480                                let answer = tokio::task::spawn_blocking(move || {
1481                                    let bundle = bundle.context("unknown session")?;
1482                                    let scope = match scope.as_str() {
1483                                        "session" => crate::database::HistoryScope::Session,
1484                                        "all" => crate::database::HistoryScope::All,
1485                                        _ => crate::database::HistoryScope::Project,
1486                                    };
1487                                    let found = crate::database::search_prompts_bounded(
1488                                        &session_id,
1489                                        &bundle,
1490                                        scope,
1491                                        &query,
1492                                        crate::server::MAX_HISTORY_MATCHES,
1493                                    )?;
1494                                    anyhow::Ok(crate::server::ViewerPromptHistory {
1495                                        entries: found
1496                                            .entries
1497                                            .into_iter()
1498                                            .map(|entry| entry.text)
1499                                            .collect(),
1500                                        truncated: found.truncated,
1501                                    })
1502                                })
1503                                .await;
1504                                reply.send(flatten_stored(answer)).ok();
1505                            });
1506                        }
1507                    }
1508                }
1509                bundle = bundle_rx.recv(), if bundle_jobs.len() < MAX_CONCURRENT_BUNDLE_CREATIONS => {
1510                    let Some(crate::server::BundleRequest { source, reply }) = bundle else {
1511                        failure = feed_stopped(termination.is_cancelled(), "the phone HTTP server stopped delivering bundle requests");
1512                        break;
1513                    };
1514                    // Repository canonicalization and config persistence both
1515                    // touch the filesystem. Keep them off this loop, and
1516                    // report a panic as a failed request rather than dropping
1517                    // the browser's reply.
1518                    let done = bundle_done_tx.clone();
1519                    let daemon_runtime = daemon_runtime.clone();
1520                    bundle_jobs.spawn(async move {
1521                        let result = daemon_runtime
1522                            .create_quick_bundle(source)
1523                            .await;
1524                        if let Err(error) = done.send(BundleCreated { result, reply }) {
1525                            tracing::debug!(%error, "bundle creation finished after the server stopped");
1526                        }
1527                    });
1528                }
1529                bundle_done = bundle_done_rx.recv() => {
1530                    let Some(BundleCreated { result, reply }) = bundle_done else {
1531                        failure = feed_stopped(termination.is_cancelled(), "the bundle creation pipeline stopped while the phone server was running");
1532                        break;
1533                    };
1534                    match result {
1535                        Ok(created) => {
1536                            let bundle_id = created.bundle_id;
1537                            // Other config sections may have changed while
1538                            // this request was in flight. Publish only the
1539                            // bundle this transaction created; a full fresh
1540                            // config is requested below and must not make a
1541                            // later completion hide another completed bundle.
1542                            let Some(bundle) = created.config.bundles.get(&bundle_id) else {
1543                                tracing::error!(%bundle_id, "bundle creation returned a config without its bundle");
1544                                if reply.send(Err(crate::server::BundleFailure::Controller)).is_err() {
1545                                    tracing::debug!("bundle creation failure reply dropped after client disconnect");
1546                                }
1547                                continue;
1548                            };
1549                            controller
1550                                .config
1551                                .bundles
1552                                .insert(bundle_id.clone(), bundle.clone());
1553                            // A reload started before this save may still be
1554                            // queued. It must not hide a bundle after we have
1555                            // acknowledged it as available to the browser.
1556                            controller_reload_invalidated |= controller_reload_in_flight;
1557                            revision = daemon_runtime.allocate_revision();
1558                            publish_snapshot!(revision);
1559                            request_daemon_controller_reload(
1560                                daemon_runtime.clone(),
1561                                "new bundle publication",
1562                            );
1563                            if reply.send(Ok(bundle_id)).is_err() {
1564                                tracing::debug!("bundle creation reply dropped after client disconnect");
1565                            }
1566                        }
1567                        Err(error) => {
1568                            let failure = match error {
1569                                crate::controller::QuickBundleFailure::InvalidSource(
1570                                    detail,
1571                                ) => {
1572                                    tracing::debug!(error = %detail, "phone bundle source was invalid");
1573                                    crate::server::BundleFailure::InvalidSource
1574                                }
1575                                crate::controller::QuickBundleFailure::Persistence(
1576                                    detail,
1577                                ) => {
1578                                    tracing::warn!(error = %detail, "phone bundle creation failed");
1579                                    crate::server::BundleFailure::Controller
1580                                }
1581                            };
1582                            if reply.send(Err(failure)).is_err() {
1583                                tracing::debug!("bundle creation failure reply dropped after client disconnect");
1584                            }
1585                        }
1586                    }
1587                }
1588                bundle_job = bundle_jobs.join_next(), if !bundle_jobs.is_empty() => {
1589                    if let Some(Err(error)) = bundle_job {
1590                        tracing::warn!(%error, "bundle creation task panicked");
1591                    }
1592                }
1593                preflight = preflight_rx.recv(), if preflight_jobs.len() < MAX_CONCURRENT_PREFLIGHTS => {
1594                    let Some(preflight) = preflight else {
1595                        failure = feed_stopped(termination.is_cancelled(), "the phone HTTP server stopped delivering preflight requests");
1596                        break;
1597                    };
1598                    // A resume preflight asks a different question about the
1599                    // same disk, so it runs on the same supervised task set
1600                    // and under the same cap as a new-session preflight.
1601                    let crate::server::NewPreflightRequest {
1602                        bundle_id,
1603                        target_id,
1604                        project_directory,
1605                        mut reply,
1606                        remote_repairs,
1607                    } = match preflight {
1608                        crate::server::PreflightRequest::New(request) => request,
1609                        crate::server::PreflightRequest::Resume(request) => {
1610                            spawn_resume_preflight(
1611                                &mut preflight_jobs,
1612                                &controller,
1613                                request,
1614                                &termination,
1615                            );
1616                            continue;
1617                        }
1618                    };
1619                    // Reading a working tree's status or validating a project
1620                    // directory touches the disk, so it runs on its own task
1621                    // rather than on the loop that has to stay responsive to
1622                    // every other feed.
1623                    let config = controller.config.clone();
1624                    let project_validation = project_directory.is_some();
1625                    let task_termination = termination.clone();
1626                    preflight_jobs.spawn(async move {
1627                        let cancelled = Arc::new(AtomicBool::new(false));
1628                        let cancellation_guard = ProcessCancellationGuard(cancelled.clone());
1629                        let mut blocking = tokio::task::spawn_blocking(move || {
1630                            run_new_preflight_with_cancellation(
1631                                config,
1632                                bundle_id,
1633                                target_id,
1634                                project_directory,
1635                                cancelled,
1636                                remote_repairs,
1637                            )
1638                        });
1639                        let answer = tokio::select! {
1640                            biased;
1641                            _ = task_termination.cancelled() => None,
1642                            _ = reply.closed() => None,
1643                            answer = &mut blocking => Some(answer),
1644                        };
1645                        let Some(answer) = answer else {
1646                            drop(cancellation_guard);
1647                            match blocking.await {
1648                                Err(error) => tracing::warn!(%error, "cancelled phone preflight task failed"),
1649                                Ok(Err(error)) => tracing::debug!(%error, "phone preflight cancelled"),
1650                                Ok(Ok(_)) => {}
1651                            }
1652                            return;
1653                        };
1654                        let answer = match answer {
1655                            Ok(Ok(answer)) => Ok(answer),
1656                            Ok(Err(error)) => {
1657                                tracing::debug!(
1658                                    error = %error,
1659                                    project_validation,
1660                                    "phone preflight check failed"
1661                                );
1662                                Err(if project_validation {
1663                                    PreflightFailure::Validation
1664                                } else {
1665                                    PreflightFailure::InvalidRepository(format!("{error:#}"))
1666                                })
1667                            }
1668                            Err(error) => {
1669                                tracing::warn!(%error, "phone preflight task failed");
1670                                Err(PreflightFailure::Controller(format!(
1671                                    "preflight task failed: {error}"
1672                                )))
1673                            }
1674                        };
1675                        if reply.send(answer).is_err() {
1676                            tracing::debug!("phone preflight reply dropped after client disconnect");
1677                        }
1678                    });
1679                }
1680                preflight_job = preflight_jobs.join_next(), if !preflight_jobs.is_empty() => {
1681                    if let Some(Err(error)) = preflight_job {
1682                        tracing::warn!(%error, "phone preflight task panicked");
1683                    }
1684                }
1685                preparation = move_preparation_rx.recv() => {
1686                    let Some(MovePreparationRequest { selection, reply }) = preparation else {
1687                        failure = feed_stopped(termination.is_cancelled(), "the phone HTTP server stopped delivering move preparation requests");
1688                        break;
1689                    };
1690                    // Preparation can inspect archives, target prerequisites,
1691                    // and harness capabilities. Keep it supervised and away
1692                    // from this feed loop so another browser can still read
1693                    // snapshots while a move form is open.
1694                    let done = move_prepared_tx.clone();
1695                    let daemon_runtime = daemon_runtime.clone();
1696                    move_preparation_jobs.spawn(async move {
1697                        let result = daemon_runtime
1698                            .prepare_move_session(selection)
1699                            .await
1700                            .map_err(|error| format!("{error:#}"));
1701                        if let Err(error) = done.send(MovePrepared { result, reply }) {
1702                            tracing::debug!(%error, "move preparation finished after the server stopped");
1703                        }
1704                    });
1705                }
1706                prepared = move_prepared_rx.recv() => {
1707                    let Some(MovePrepared { result, reply }) = prepared else {
1708                        failure = feed_stopped(termination.is_cancelled(), "the move preparation pipeline stopped while the phone server was running");
1709                        break;
1710                    };
1711                    if reply.send(result).is_err() {
1712                        tracing::debug!("move preparation reply dropped after client disconnect");
1713                    }
1714                }
1715                move_preparation_job = move_preparation_jobs.join_next(), if !move_preparation_jobs.is_empty() => {
1716                    if let Some(Err(error)) = move_preparation_job {
1717                        tracing::warn!(%error, "move preparation task failed");
1718                    }
1719                }
1720                move_recovery_job = move_recovery_jobs.join_next(), if !move_recovery_jobs.is_empty() => {
1721                    if let Some(Err(error)) = move_recovery_job {
1722                        move_recovery_load_in_flight = false;
1723                        tracing::warn!(%error, "Move recovery projection task failed");
1724                    }
1725                }
1726                receipt = receipt_rx.recv() => {
1727                    let Some(ReadReceiptRequest { client_id, session_id, through, reply }) = receipt else {
1728                        failure = feed_stopped(termination.is_cancelled(), "the phone HTTP server stopped delivering read receipts");
1729                        break;
1730                    };
1731                    match controller.state.sessions.get(&session_id) {
1732                        None => {
1733                            if reply.send(Err("unknown session".into())).is_err() {
1734                                tracing::debug!(%session_id, "unknown-session read receipt reply dropped after client disconnect");
1735                            }
1736                        }
1737                        Some(session) => {
1738                            let workspace_id = session.workspace_id.clone();
1739                            let done = receipt_done_tx.clone();
1740                            let persisted_session_id = session_id.clone();
1741                            tokio::spawn(async move {
1742                                let joined = tokio::task::spawn_blocking(move || {
1743                                    crate::database::persist_read_receipt(
1744                                        &client_id,
1745                                        &workspace_id,
1746                                        &persisted_session_id,
1747                                        through,
1748                                    )
1749                                })
1750                                .await;
1751                                let result = match joined {
1752                                    Ok(result) => result.map_err(|error| format!("{error:#}")),
1753                                    Err(error) => Err(format!("phone read receipt task failed: {error}")),
1754                                };
1755                                if let Err(error) = done.send(ReadReceiptPersisted { session_id, result, reply }) {
1756                                    tracing::debug!(%error, "phone read receipt finished after the server stopped");
1757                                }
1758                            });
1759                        }
1760                    }
1761                }
1762                persisted = receipt_done_rx.recv() => {
1763                    let Some(ReadReceiptPersisted { session_id, result, reply }) = persisted else { continue };
1764                    match result {
1765                        Ok(receipt) => {
1766                            let _ = receipt;
1767                            if reply.send(Ok(())).is_err() {
1768                                tracing::debug!(%session_id, "phone read receipt reply dropped after client disconnect");
1769                            }
1770                        }
1771                        Err(error) => {
1772                            tracing::warn!(%session_id, "could not persist a phone read receipt: {error}");
1773                            if reply.send(Err(error)).is_err() {
1774                                tracing::debug!(%session_id, "failed phone read receipt reply dropped after client disconnect");
1775                            }
1776                        }
1777                    }
1778                }
1779                action = action_rx.recv() => {
1780                    let Some(request) = action else {
1781                        failure = feed_stopped(termination.is_cancelled(), "the phone HTTP server stopped delivering actions");
1782                        break;
1783                    };
1784                    // A refresh nudges a poller this loop owns. It takes no
1785                    // session slot and starts no lifecycle work, so it is
1786                    // answered here rather than admitted as an action.
1787                    match &request.action {
1788                        ControllerAction::RefreshCapacity { target_id } => {
1789                            let known = capacity_state.contains_key(target_id);
1790                            // One queued nudge refreshes every target. Do not
1791                            // block the consumer while readings wait for it.
1792                            let accepted = known && match capacity_triggers_tx.try_send(()) {
1793                                Ok(()) | Err(tokio::sync::mpsc::error::TrySendError::Full(())) => true,
1794                                Err(tokio::sync::mpsc::error::TrySendError::Closed(())) => {
1795                                    tracing::warn!("phone capacity refresh rejected: poller stopped");
1796                                    false
1797                                }
1798                            };
1799                            if known {
1800                                if let Some(entry) = capacity_state.get_mut(target_id) {
1801                                    entry.refreshing = accepted;
1802                                    if !accepted {
1803                                        entry.failed = true;
1804                                    }
1805                                }
1806                                revision = daemon_runtime.allocate_revision();
1807                                publish_snapshot!(revision);
1808                            }
1809                            let outcome = if accepted {
1810                                ActionOutcome::accepted()
1811                            } else {
1812                                ActionOutcome::Failed
1813                            };
1814                            if request.reply.send(outcome).is_err() {
1815                                tracing::debug!(%target_id, "phone capacity refresh reply dropped after client disconnect");
1816                            }
1817                            tokio::task::yield_now().await;
1818                            continue;
1819                        }
1820                        ControllerAction::RefreshQuota { profile_id } => {
1821                            let known = controller.config.enabled_profile(profile_id).is_some();
1822                            if known {
1823                                // The refresher works from a generation-stamped
1824                                // batch, so a new generation is how one is asked
1825                                // for again rather than a per-profile trigger.
1826                                quota_batch.generation = quota_batch.generation.saturating_add(1);
1827                                quota_batch.profiles = quota_refresh_profiles(&controller);
1828                                quota_profiles_tx.send_replace(quota_batch.clone());
1829                            }
1830                            let outcome = if known {
1831                                ActionOutcome::accepted()
1832                            } else {
1833                                ActionOutcome::Failed
1834                            };
1835                            if request.reply.send(outcome).is_err() {
1836                                tracing::debug!(%profile_id, "phone quota refresh reply dropped after client disconnect");
1837                            }
1838                            tokio::task::yield_now().await;
1839                            continue;
1840                        }
1841                        _ => {}
1842                    }
1843                    if let ControllerAction::Cancel { session_id } = &request.action {
1844                        let outcome = if request_phone_action_cancellation(
1845                            session_id,
1846                            &action_sessions,
1847                            &action_cancellations,
1848                        ) {
1849                            daemon_runtime.cancel_lifecycle_if_active(session_id);
1850                            ActionOutcome::accepted()
1851                        } else {
1852                            ActionOutcome::NotCancellable
1853                        };
1854                        if request.reply.send(outcome).is_err() {
1855                            tracing::debug!(%session_id, "phone cancellation reply dropped after client disconnect");
1856                        }
1857                        tokio::task::yield_now().await;
1858                        continue;
1859                    }
1860                    if let ControllerAction::Close { session_id } = &request.action {
1861                        if closing_actions.contains_key(session_id) {
1862                            if request.reply.send(ActionOutcome::accepted()).is_err() { tracing::debug!(%session_id, "repeated close reply dropped"); }
1863                            continue;
1864                        }
1865                        request_phone_action_cancellation(session_id, &action_sessions, &action_cancellations);
1866                        daemon_runtime.request_close(session_id);
1867                    }
1868                    // A force close runs even while a graceful close for the
1869                    // same session is still in flight; that stuck close is
1870                    // exactly what it is meant to take over.
1871                    if let ControllerAction::ForceClose { session_id } = &request.action {
1872                        request_phone_action_cancellation(session_id, &action_sessions, &action_cancellations);
1873                        daemon_runtime.request_close(session_id);
1874                    }
1875                    let session_id = match admit_phone_action(
1876                        &request.action,
1877                        action_cancellations.len(),
1878                        &mut active_actions,
1879                    ) {
1880                        Ok(session_id) => session_id,
1881                        Err(refusal) => {
1882                            if request.reply.send(refusal).is_err() {
1883                                tracing::debug!("phone action refusal reply dropped after client disconnect");
1884                            }
1885                            tokio::task::yield_now().await;
1886                            continue;
1887                        }
1888                    };
1889                    let ControllerRequest { action, reply } = request;
1890                    let done = action_done_tx.clone();
1891                    let session_control = worker_commands_tx.clone();
1892                    let daemon_runtime = daemon_runtime.clone();
1893                    let started = action_started_tx.clone();
1894                    next_action_id = next_action_id.wrapping_add(1).max(1);
1895                    let action_id = next_action_id;
1896                    if let ControllerAction::Close { session_id } | ControllerAction::ForceClose { session_id } = &action { closing_actions.insert(session_id.clone(), action_id); }
1897                    if let ControllerAction::New { workspace_id, .. } = &action {
1898                        let workspace_id = if workspace_id.is_empty() && phone_workspaces.len() == 1 {
1899                            phone_workspaces[0].id.clone()
1900                        } else {
1901                            workspace_id.clone()
1902                        };
1903                        launch_workspaces.insert(action_id, workspace_id);
1904                    }
1905                    let control = PhoneActionControl::for_action(&action);
1906                    action_cancellations.insert(action_id, control.clone());
1907                    if let Some(session_id) = &session_id {
1908                        action_sessions.insert(action_id, session_id.clone());
1909                    }
1910                    action_replies.accept(action_id, &action, reply);
1911                    let runtime = tokio::runtime::Handle::current();
1912                    tokio::spawn(async move {
1913                        let joined = tokio::task::spawn_blocking(move || {
1914                            let result = (|| -> Result<()> {
1915                                if control.cancelled.load(Ordering::Acquire) {
1916                                    bail!("phone action cancelled");
1917                                }
1918                                let mut operation_controller = Controller::load()?;
1919                                let executor =
1920                                    CancellableProcessExecutor::new(control.cancelled.clone());
1921                                runtime.block_on(apply_phone_action(
1922                                    &mut operation_controller,
1923                                    PhoneActionServices {
1924                                        sessions: &session_control,
1925                                        daemon_runtime: &daemon_runtime,
1926                                    },
1927                                    action,
1928                                    &executor,
1929                                    action_id,
1930                                    &started,
1931                                    &control,
1932                                ))
1933                            })();
1934                            result.map_err(|error| format!("{error:#}"))
1935                        })
1936                        .await;
1937                        let result = match joined {
1938                            Ok(result) => result,
1939                            Err(error) => Err(format!("phone action task failed: {error}")),
1940                        };
1941                        if let Err(error) = done.send((action_id, session_id, result)) {
1942                            tracing::debug!(action_id, %error, "phone action finished after the server stopped");
1943                        }
1944                    });
1945                }
1946                started = action_started_rx.recv() => {
1947                    let Some(started) = started else {
1948                        tokio::task::yield_now().await;
1949                        continue;
1950                    };
1951                    let started_session_id = started.session.id.clone();
1952                    let publication = if !action_cancellations.contains_key(&started.action_id) {
1953                        Err("phone action completed before its provisional session was published".into())
1954                    } else {
1955                        track_started_phone_session(
1956                            &mut controller.state,
1957                            &mut active_actions,
1958                            &mut action_sessions,
1959                            started.action_id,
1960                            started.session,
1961                        )
1962                    };
1963                    if publication.is_ok() {
1964                        revision = daemon_runtime.allocate_revision();
1965                        publish_snapshot!(revision);
1966                        request_daemon_controller_reload(
1967                            daemon_runtime.clone(),
1968                            "new session publication",
1969                        );
1970                    };
1971                    if publication.is_err()
1972                        && let Some(control) = action_cancellations.get(&started.action_id)
1973                    {
1974                        control.request_cancel();
1975                    }
1976                    // The phone asked for a session, and now there is one to
1977                    // point at: that is what its request was waiting for.
1978                    action_replies.resolve(
1979                        started.action_id,
1980                        if publication.is_ok() {
1981                            ActionOutcome::Accepted {
1982                                session_id: Some(started_session_id),
1983                            }
1984                        } else {
1985                            ActionOutcome::Failed
1986                        },
1987                    );
1988                    if started.published.send(publication).is_err() {
1989                        tracing::debug!(action_id = started.action_id, "phone new-session publication reply dropped after client disconnect");
1990                    }
1991                }
1992                completed = action_done_rx.recv() => {
1993                    let Some((action_id, session_id, result)) = completed else {
1994                        failure = feed_stopped(termination.is_cancelled(), "the phone action pipeline stopped reporting completions");
1995                        break;
1996                    };
1997                    action_cancellations.remove(&action_id);
1998                    let session_id = action_sessions.remove(&action_id).or(session_id);
1999                    if closing_actions.values().any(|closing_id| *closing_id == action_id) && let Some(id) = &session_id { daemon_runtime.clear_close_request(id); }
2000                    closing_actions.retain(|_, closing_id| *closing_id != action_id);
2001                    if let Some(session_id) = &session_id && !action_sessions.values().any(|active| active == session_id) {
2002                        active_actions.remove(session_id);
2003                    }
2004                    // A `new` that failed before publishing a session never
2005                    // reached the arm that answers it, so its phone is still
2006                    // waiting for a reply it can act on.
2007                    action_replies.resolve(action_id, ActionOutcome::Failed);
2008                    if let Some(workspace_id) = launch_workspaces.remove(&action_id)
2009                        && result.is_err()
2010                        && !session_id.as_ref().is_some_and(|id| closing_actions.contains_key(id))
2011                    {
2012                        record_launch_failure(
2013                            &mut launch_failures,
2014                            action_id,
2015                            workspace_id,
2016                            session_id.clone(),
2017                            result.as_ref().err().cloned(),
2018                        );
2019                        revision = daemon_runtime.allocate_revision();
2020                        publish_snapshot!(revision);
2021                    }
2022                    if let Err(error) = &result {
2023                        tracing::warn!(
2024                            action_id,
2025                            session_id = session_id.as_deref(),
2026                            %error,
2027                            "phone action failed"
2028                        );
2029                    }
2030                    record_action_result(
2031                        &mut pending_action_errors,
2032                        session_id.as_deref(),
2033                        &result,
2034                    );
2035                    request_controller_reload(
2036                        &mut controller_reload_in_flight,
2037                        &mut controller_reload_requested,
2038                        &controller_reload_tx,
2039                    );
2040                    request_move_recovery_reload(
2041                        &move_recovery_tx,
2042                        &mut move_recovery_load_in_flight,
2043                        &mut move_recovery_jobs,
2044                    );
2045                    request_daemon_controller_reload(
2046                        daemon_runtime.clone(),
2047                        "phone action completion",
2048                    );
2049                }
2050                reloaded = controller_reload_rx.recv() => {
2051                    let Some(ControllerReloaded { result }) = reloaded else {
2052                        failure = feed_stopped(
2053                            termination.is_cancelled(),
2054                            "the controller reload pipeline stopped while the phone server was running",
2055                        );
2056                        break;
2057                    };
2058                    controller_reload_in_flight = false;
2059                    if std::mem::take(&mut controller_reload_invalidated) {
2060                        if let Err(error) = &result {
2061                            tracing::warn!(%error, "superseded controller reload failed");
2062                        }
2063                        controller_reload_requested = false;
2064                        request_controller_reload(
2065                            &mut controller_reload_in_flight,
2066                            &mut controller_reload_requested,
2067                            &controller_reload_tx,
2068                        );
2069                        continue;
2070                    }
2071                    match result {
2072                        Ok(mut reloaded) => {
2073                            for (session_id, error) in &pending_action_errors {
2074                                if let Some(session) = reloaded.state.sessions.get_mut(session_id)
2075                                    && session.last_error.is_none()
2076                                {
2077                                    session.last_error = Some(error.clone());
2078                                }
2079                            }
2080                            controller = reloaded;
2081                            quotas.retain(|id, _| controller.config.enabled_profile(id).is_some());
2082                            subagent_quota_reports
2083                                .lock()
2084                                .expect("sub-agent quota reports lock poisoned")
2085                                .retain(|id, _| controller.config.enabled_profile(id).is_some());
2086                            worker_targets_tx.send_replace(dashboard_worker_targets(&controller));
2087                            publish_capacity_targets(
2088                                &controller,
2089                                &capacity_targets_tx,
2090                                &mut capacity_state,
2091                            );
2092                            credential_sync_handle.set_targets(credential_sync_targets(&controller));
2093                            republish_quota_profiles(
2094                                &controller,
2095                                &mut published_quota_profiles,
2096                                &mut quota_batch,
2097                                &quota_profiles_tx,
2098                            );
2099                            queued_prompts.retain(|session_id, _| {
2100                                controller.state.sessions.contains_key(session_id)
2101                            });
2102                            pending_elicitations.retain(|session_id, _| {
2103                                controller.state.sessions.contains_key(session_id)
2104                            });
2105                            prompt_images.retain(|session_id| {
2106                                controller.state.sessions.contains_key(session_id)
2107                            });
2108                            operational.retain(|session_id, _| {
2109                                controller.state.sessions.contains_key(session_id)
2110                            });
2111                            materialized_activity.retain(|session_id, _| {
2112                                controller.state.sessions.contains_key(session_id)
2113                            });
2114                            request_move_recovery_reload(
2115                                &move_recovery_tx,
2116                                &mut move_recovery_load_in_flight,
2117                                &mut move_recovery_jobs,
2118                            );
2119                            conversations.retain(|id, _| {
2120                                controller.state.sessions.get(id).is_some_and(|session| session.state.is_active())
2121                            });
2122                            for session_id in conversation_projections.session_ids() {
2123                                if !controller
2124                                    .state
2125                                    .sessions
2126                                    .get(&session_id)
2127                                    .is_some_and(|session| session.state.is_active())
2128                                {
2129                                    conversation_projections.forget(&session_id);
2130                                }
2131                            }
2132                            revision = daemon_runtime.allocate_revision();
2133                            conversation_tx.send_replace(conversations.clone());
2134                            publish_snapshot!(revision);
2135                        }
2136                        Err(error) => {
2137                            tracing::warn!(%error, "completed phone operation could not reload controller state");
2138                        }
2139                    }
2140                    if controller_reload_requested {
2141                        controller_reload_requested = false;
2142                        controller_reload_in_flight = true;
2143                        spawn_controller_reload(controller_reload_tx.clone());
2144                    }
2145                }
2146            }
2147        }
2148        // Stop provider requests before the HTTP request channels disappear.
2149        dictation_jobs.shutdown().await;
2150        // Bundle jobs are supervised so shutdown never leaves a detached
2151        // request task behind holding the config mutation lock.
2152        bundle_jobs.shutdown().await;
2153        // Preflight jobs own cancellation guards for their blocking Git
2154        // probes. Aborting them here signals those probes before the server's
2155        // request channels disappear.
2156        preflight_jobs.shutdown().await;
2157        // Preparation tasks may be inspecting an archive or probing a target;
2158        // abort and drain them before the HTTP server's channels disappear.
2159        move_preparation_jobs.shutdown().await;
2160        move_recovery_jobs.shutdown().await;
2161        // Every exit stops in-flight work, whether it was asked for or forced.
2162        crate::controller::profile_config::cancel_all();
2163        for control in action_cancellations.values() {
2164            control.request_cancel();
2165        }
2166        match failure {
2167            Some(failure) => Err(failure),
2168            None => Ok::<(), anyhow::Error>(()),
2169        }
2170    };
2171    let result = tokio::select! {
2172        result = api_activity::record_activity_stream(activity_snapshots) => result.context("native API activity recorder stopped"),
2173        result = serve => result,
2174        result = control => result,
2175    };
2176    conversation_projection_shutdown.cancel();
2177    renewal_cancellation.cancel();
2178    if let Some(task) = renewal_task
2179        && let Err(error) = task.await
2180    {
2181        tracing::warn!(%error, "Tailscale certificate renewal task failed");
2182    }
2183    worker_shutdown
2184        .shutdown()
2185        .await
2186        .context("shut down phone server session manager")?;
2187    result?;
2188    Ok(())
2189}
2190
2191/// Why the control loop is stopping because one of its feeds ended.
2192///
2193/// During shutdown every feed ends, and that is the plan. At any other time it
2194/// means the phone server has lost the machinery it exists to drive, so it
2195/// says which feed and exits non-zero instead of reporting success.
2196fn feed_stopped(shutting_down: bool, reason: &'static str) -> Option<anyhow::Error> {
2197    (!shutting_down).then(|| anyhow::anyhow!(reason))
2198}
2199
2200/// Whether a session's agent said it accepts image content in prompts. An
2201/// agent that has not answered `initialize` yet has advertised nothing, so the
2202/// phone is not offered controls the agent may refuse.
2203fn agent_accepts_prompt_images(operational: &mj_core::relay::RelayOperationalState) -> bool {
2204    operational
2205        .agent_capabilities
2206        .as_ref()
2207        .is_some_and(|capabilities| capabilities.prompt_capabilities.image)
2208}
2209
2210fn controller_action_session_id(action: &ControllerAction) -> Option<String> {
2211    match action {
2212        ControllerAction::New { .. } => None,
2213        ControllerAction::Prompt { session_id, .. }
2214        | ControllerAction::RunShell { session_id, .. }
2215        | ControllerAction::CancelShell { session_id, .. }
2216        | ControllerAction::Close { session_id }
2217        | ControllerAction::ForceClose { session_id }
2218        | ControllerAction::Resume { session_id, .. }
2219        | ControllerAction::Open { session_id }
2220        | ControllerAction::Cancel { session_id }
2221        | ControllerAction::RemoveQueuedPrompt { session_id, .. }
2222        | ControllerAction::RespondElicitation { session_id, .. }
2223        | ControllerAction::Rename { session_id, .. }
2224        | ControllerAction::CancelTurn { session_id }
2225        | ControllerAction::SetConfig { session_id, .. }
2226        | ControllerAction::SetPlanMode { session_id, .. }
2227        | ControllerAction::StartReview { session_id }
2228        | ControllerAction::ResolveReview { session_id, .. } => Some(session_id.clone()),
2229        ControllerAction::Move { request } => {
2230            Some(request.preparation.selection.session_id.clone())
2231        }
2232        // A refresh belongs to a profile or a target rather than a session, so
2233        // it takes no session slot and cannot be refused as session-busy.
2234        ControllerAction::RefreshQuota { .. } | ControllerAction::RefreshCapacity { .. } => None,
2235    }
2236}
2237
2238/// Flatten a joined blocking result into the answer a phone channel carries.
2239fn flatten_stored<T>(
2240    joined: std::result::Result<Result<T>, tokio::task::JoinError>,
2241) -> std::result::Result<T, String> {
2242    match joined {
2243        Ok(Ok(value)) => Ok(value),
2244        Ok(Err(error)) => Err(format!("{error:#}")),
2245        Err(error) => Err(format!("viewer state task failed: {error}")),
2246    }
2247}
2248
2249/// Cancel the shared subprocess executor when its request owner goes away.
2250struct ProcessCancellationGuard(Arc<AtomicBool>);
2251
2252impl Drop for ProcessCancellationGuard {
2253    fn drop(&mut self) {
2254        self.0.store(true, Ordering::Release);
2255    }
2256}
2257
2258/// Perform the preflight for a new session.
2259///
2260/// This runs on the blocking task owned by the phone server. Bare targets
2261/// need the same directory-and-Git-HEAD validation as the dashboard. Isolated
2262/// targets resolve every bundle repository to a network source and report the
2263/// exact fetch branch and publication destinations. Local working-tree
2264/// contents are never copied for this path.
2265#[cfg(test)]
2266fn run_new_preflight(
2267    config: Config,
2268    bundle_id: String,
2269    target_id: String,
2270    project_directory: Option<PathBuf>,
2271) -> Result<crate::server::PreflightNew> {
2272    run_new_preflight_with_cancellation(
2273        config,
2274        bundle_id,
2275        target_id,
2276        project_directory,
2277        Arc::new(AtomicBool::new(false)),
2278        Vec::new(),
2279    )
2280}
2281
2282/// Run one resume preflight on its own task, like a new-session preflight:
2283/// the disk work stays off the feed loop, a disconnected browser cancels it,
2284/// and the task is supervised by the same `JoinSet`.
2285fn spawn_resume_preflight(
2286    jobs: &mut tokio::task::JoinSet<()>,
2287    controller: &Controller,
2288    request: crate::server::ResumePreflightRequest,
2289    termination: &tokio_util::sync::CancellationToken,
2290) {
2291    let crate::server::ResumePreflightRequest {
2292        session_id,
2293        target_id,
2294        mut reply,
2295    } = request;
2296    let config = controller.config.clone();
2297    let session = controller.state.sessions.get(&session_id).cloned();
2298    let termination = termination.clone();
2299    jobs.spawn(async move {
2300        let cancelled = Arc::new(AtomicBool::new(false));
2301        let cancellation_guard = ProcessCancellationGuard(cancelled.clone());
2302        let mut blocking = tokio::task::spawn_blocking(move || {
2303            run_resume_preflight(config, session, &target_id, cancelled)
2304        });
2305        let answer = tokio::select! {
2306            biased;
2307            _ = termination.cancelled() => None,
2308            _ = reply.closed() => None,
2309            answer = &mut blocking => Some(answer),
2310        };
2311        let Some(answer) = answer else {
2312            drop(cancellation_guard);
2313            if let Err(error) = blocking.await {
2314                tracing::warn!(%error, "cancelled phone resume preflight task failed");
2315            }
2316            return;
2317        };
2318        let answer = answer.map_err(|error| {
2319            tracing::warn!(%error, "phone resume preflight task failed");
2320            PreflightFailure::Controller(format!("resume preflight task failed: {error}"))
2321        });
2322        if reply.send(answer).is_err() {
2323            tracing::debug!("phone resume preflight reply dropped after client disconnect");
2324        }
2325    });
2326}
2327
2328/// What resuming this session on this target does to its repository content.
2329///
2330/// Anything but a local-checkout conversion is `Ready` and reads nothing:
2331/// the compatibility gate is a pure function, so a browser may ask about
2332/// every destination it offers. A conversion that cannot be planned reports
2333/// the plan's own message, which is what tells a person to add a remote or
2334/// commit a submodule.
2335fn run_resume_preflight(
2336    config: Config,
2337    session: Option<mj_core::state::SessionRecord>,
2338    target_id: &str,
2339    cancelled: Arc<AtomicBool>,
2340) -> crate::server::PreflightResume {
2341    let Some(session) = session else {
2342        return crate::server::PreflightResume::Unavailable {
2343            detail: "this session is no longer available".to_owned(),
2344        };
2345    };
2346    match crate::controller::resume_compatibility(&session, &config, target_id) {
2347        Err(reason) => crate::server::PreflightResume::Unavailable { detail: reason },
2348        Ok(plan) if plan != crate::controller::ResumePlan::RawToWorkspace => {
2349            crate::server::PreflightResume::Ready
2350        }
2351        Ok(_) => {
2352            let executor =
2353                CancellableProcessExecutor::new(cancelled).with_deadline(Duration::from_secs(30));
2354            match crate::controller::raw_conversion_preview_for(&session, &config, &executor) {
2355                Err(error) => crate::server::PreflightResume::Unavailable {
2356                    detail: format!("{error:#}"),
2357                },
2358                Ok(mut preview) => {
2359                    // Credentials never reach a browser, here as everywhere
2360                    // else a repository URL is published.
2361                    preview.fetch_url = display_url(&preview.fetch_url);
2362                    preview.push_urls = preview
2363                        .push_urls
2364                        .iter()
2365                        .map(|url| display_url(url))
2366                        .collect();
2367                    crate::server::PreflightResume::ConvertingRawCheckout {
2368                        preview: Box::new(preview),
2369                    }
2370                }
2371            }
2372        }
2373    }
2374}
2375
2376fn run_new_preflight_with_cancellation(
2377    config: Config,
2378    bundle_id: String,
2379    target_id: String,
2380    project_directory: Option<PathBuf>,
2381    cancelled: Arc<AtomicBool>,
2382    remote_repairs: Vec<mj_core::local_git::LocalRemoteRepair>,
2383) -> Result<crate::server::PreflightNew> {
2384    let executor =
2385        CancellableProcessExecutor::new(cancelled).with_deadline(Duration::from_secs(30));
2386    if !remote_repairs.is_empty() {
2387        let target = config.targets.get(&target_id).context("unknown target")?;
2388        anyhow::ensure!(
2389            !is_bare_project_target(target) && project_directory.is_none(),
2390            "remote repair requires an isolated target"
2391        );
2392        let bundle = config.bundles.get(&bundle_id).context("unknown bundle")?;
2393        mj_core::local_git::apply_repository_remote_repairs(bundle, &remote_repairs, &executor)?;
2394    }
2395    run_new_preflight_with_executor(config, bundle_id, target_id, project_directory, &executor)
2396}
2397
2398fn run_new_preflight_with_executor(
2399    config: Config,
2400    bundle_id: String,
2401    target_id: String,
2402    project_directory: Option<PathBuf>,
2403    executor: &impl CommandExecutor,
2404) -> Result<crate::server::PreflightNew> {
2405    let target_is_bare = config
2406        .targets
2407        .get(&target_id)
2408        .with_context(|| format!("unknown target template {target_id:?}"))
2409        .map(is_bare_project_target)?;
2410    if target_is_bare {
2411        let directory =
2412            project_directory.context("project directory is required for a bare target")?;
2413        let controller = config_only_controller(config);
2414        let directory = controller.resolve_project_directory(&target_id, &directory, executor)?;
2415        let managed_worktree =
2416            controller.managed_worktree_options(&target_id, &directory, executor)?;
2417        return Ok(crate::server::PreflightNew {
2418            managed_worktree,
2419            project_directory: Some(directory),
2420            remote_repairs: Vec::new(),
2421            dirty_repositories: Vec::new(),
2422            remote_repositories: Vec::new(),
2423            local_changes_excluded: false,
2424        });
2425    }
2426    if project_directory.is_some() {
2427        bail!("project directory is unsupported for this target");
2428    }
2429
2430    let bundle = config.bundles.get(&bundle_id).context("unknown bundle")?;
2431    let repairs = mj_core::local_git::repository_remote_repairs(bundle, executor)?;
2432    if !repairs.is_empty() {
2433        return Ok(crate::server::PreflightNew {
2434            managed_worktree: Default::default(),
2435            project_directory: None,
2436            remote_repairs: repairs,
2437            dirty_repositories: Vec::new(),
2438            remote_repositories: Vec::new(),
2439            local_changes_excluded: true,
2440        });
2441    }
2442    let remote_repositories = bundle
2443        .repositories
2444        .iter()
2445        .map(|repository| {
2446            let source = resolve_repository(repository, executor)
2447                .with_context(|| format!("repository {:?}", repository.id))?;
2448            let default_branch = default_branch(&source, executor)
2449                .with_context(|| format!("repository {:?}", repository.id))?;
2450            Ok(crate::server::PreflightRepository {
2451                id: repository.id.clone(),
2452                fetch_url: display_url(&source.fetch_url),
2453                default_branch,
2454                push_urls: source
2455                    .push_urls
2456                    .iter()
2457                    .map(|url| display_url(url))
2458                    .collect(),
2459            })
2460        })
2461        .collect::<Result<Vec<_>>>()?;
2462    Ok(crate::server::PreflightNew {
2463        managed_worktree: Default::default(),
2464        project_directory: None,
2465        remote_repairs: Vec::new(),
2466        dirty_repositories: Vec::new(),
2467        remote_repositories,
2468        local_changes_excluded: true,
2469    })
2470}
2471
2472fn phone_action_capacity_available(active_actions: usize) -> bool {
2473    active_actions < MAX_CONCURRENT_PHONE_ACTIONS
2474}
2475
2476/// Point the quota refresher at the profiles the configuration currently
2477/// defines, alongside the worker-poll and credential-sync targets that are
2478/// rebuilt from the same reload. A profile added to `config.toml` while the
2479/// server runs otherwise reaches the snapshot but never the refresher, and
2480/// reads "quota unavailable" until the next restart.
2481///
2482/// Sending a batch restarts every profile's refresh, which spawns a harness
2483/// process per profile, so the batch travels only when the profiles changed.
2484/// Reports whether it did.
2485fn republish_quota_profiles(
2486    controller: &Controller,
2487    published: &mut std::collections::BTreeMap<String, HarnessProfile>,
2488    batch: &mut QuotaRefreshBatch,
2489    profiles_tx: &tokio::sync::watch::Sender<QuotaRefreshBatch>,
2490) -> bool {
2491    if *published == controller.config.profiles {
2492        return false;
2493    }
2494    published.clone_from(&controller.config.profiles);
2495    batch.generation = batch.generation.saturating_add(1);
2496    batch.profiles = quota_refresh_profiles(controller);
2497    profiles_tx.send_replace(batch.clone());
2498    true
2499}
2500
2501fn request_phone_action_cancellation(
2502    session_id: &str,
2503    action_sessions: &std::collections::BTreeMap<u64, String>,
2504    action_cancellations: &std::collections::BTreeMap<u64, PhoneActionControl>,
2505) -> bool {
2506    let control = action_sessions
2507        .iter()
2508        .find_map(|(action_id, active_session_id)| {
2509            (active_session_id == session_id)
2510                .then(|| action_cancellations.get(action_id))
2511                .flatten()
2512        });
2513    if let Some(control) = control {
2514        return control.request_cancel();
2515    }
2516    false
2517}
2518
2519fn track_started_phone_session(
2520    state: &mut State,
2521    active_actions: &mut std::collections::BTreeSet<String>,
2522    action_sessions: &mut std::collections::BTreeMap<u64, String>,
2523    action_id: u64,
2524    session: SessionRecord,
2525) -> std::result::Result<(), String> {
2526    let session_id = session.id.clone();
2527    if !active_actions.insert(session_id.clone()) {
2528        return Err("another operation is already running for the new session".into());
2529    }
2530    action_sessions.insert(action_id, session_id.clone());
2531    state.sessions.insert(session_id, session);
2532    Ok(())
2533}
2534
2535/// Carry a finished action's failure into the session projection, and clear it
2536/// once a later action for the same session succeeds.
2537///
2538/// Nothing is waiting on the request any more, so a failure the action itself
2539/// did not record would reach no one but this process's stderr; the overlay
2540/// keeps it visible through every later durable reload, where the snapshot's
2541/// `has_error` takes it to the phone. Clearing on success matters just as
2542/// much: the overlay has no other expiry, so one transient failure would
2543/// otherwise badge the session as errored for the daemon's whole lifetime.
2544fn record_action_result(
2545    pending_action_errors: &mut std::collections::BTreeMap<String, String>,
2546    session_id: Option<&str>,
2547    result: &std::result::Result<(), String>,
2548) {
2549    let Some(session_id) = session_id else {
2550        return;
2551    };
2552    match result {
2553        Err(error) => {
2554            pending_action_errors.insert(session_id.to_owned(), error.clone());
2555        }
2556        Ok(()) => {
2557            pending_action_errors.remove(session_id);
2558        }
2559    }
2560}
2561
2562/// Retain safe notices even if provisioning removed its provisional session.
2563/// This bounded, daemon-lifetime history survives durable controller reloads.
2564fn record_launch_failure(
2565    failures: &mut Vec<crate::server::ViewerLaunchFailure>,
2566    action_id: u64,
2567    workspace_id: String,
2568    session_id: Option<String>,
2569    error: Option<String>,
2570) {
2571    failures.push(crate::server::ViewerLaunchFailure {
2572        id: format!("{}-{action_id}", std::process::id()),
2573        workspace_id,
2574        session_id,
2575        error,
2576    });
2577    if failures.len() > 16 {
2578        failures.remove(0);
2579    }
2580}
2581
2582struct PhoneActionServices<'a> {
2583    sessions: &'a SessionManagerControl,
2584    daemon_runtime: &'a Arc<RuntimeState>,
2585}
2586
2587async fn apply_phone_action(
2588    controller: &mut Controller,
2589    services: PhoneActionServices<'_>,
2590    action: ControllerAction,
2591    _executor: &(impl CommandExecutor + Sync),
2592    action_id: u64,
2593    started: &tokio::sync::mpsc::UnboundedSender<PhoneActionStarted>,
2594    control: &PhoneActionControl,
2595) -> Result<()> {
2596    match action {
2597        ControllerAction::New {
2598            workspace_id,
2599            profile_id,
2600            bundle_id,
2601            target_id,
2602            title,
2603            project_directory,
2604            create_managed_worktree,
2605            mjolnir_subagents,
2606            dirty_ack: _dirty_ack,
2607        } => {
2608            let workspace_id = if workspace_id.is_empty() {
2609                let workspaces = crate::database::list_workspaces()?;
2610                match workspaces.as_slice() {
2611                    [workspace] => workspace.id.clone(),
2612                    [] => bail!("create a workspace before starting a phone session"),
2613                    _ => bail!("phone session creation requires a workspace_id"),
2614                }
2615            } else {
2616                workspace_id
2617            };
2618            // A phone that supplies no title gets the one the terminal would
2619            // have derived, so a session started from either surface reads the
2620            // same way in both.
2621            let title = title.unwrap_or_else(|| {
2622                let project = project_directory
2623                    .as_ref()
2624                    .and_then(|path| path.file_name())
2625                    .map(|name| name.to_string_lossy().into_owned())
2626                    .unwrap_or_else(|| bundle_id.clone());
2627                format!("{project} via {profile_id}")
2628            });
2629            let session_title_override = Some(title.clone());
2630            // Isolated creation always starts from the network default branch;
2631            // the legacy field remains accepted on the wire for old phones but
2632            // cannot opt local commits or dirty files into a new session.
2633            let allow_dirty_local = false;
2634            let (published, publication) = tokio::sync::oneshot::channel();
2635            let registered = services
2636                .daemon_runtime
2637                .start_create_session_controlled(
2638                    CreateSessionRequest {
2639                        create_managed_worktree,
2640                        mjolnir_subagents,
2641                        initial_prompt: None,
2642                        workspace_id,
2643                        profile_id,
2644                        bundle_id,
2645                        project_directory,
2646                        target_template_id: target_id,
2647                        additional_mounts: Vec::new(),
2648                        allow_dirty_local,
2649                        resource_allocation: None,
2650                        title,
2651                        session_title_override,
2652                    },
2653                    control
2654                        .create
2655                        .clone()
2656                        .expect("New action has a daemon create control"),
2657                    publication,
2658                )
2659                .await?;
2660            let registered_session_id = registered.session.id.clone();
2661            started
2662                .send(PhoneActionStarted {
2663                    action_id,
2664                    session: registered.session,
2665                    published,
2666                })
2667                .map_err(|_| anyhow::anyhow!("phone server stopped before publishing session"))?;
2668            services
2669                .daemon_runtime
2670                .wait_create_session(&registered_session_id)
2671                .await
2672        }
2673        ControllerAction::Prompt {
2674            session_id,
2675            text,
2676            images,
2677        } => {
2678            services
2679                .sessions
2680                .wait_for_session(&session_id, Duration::from_secs(5))
2681                .await?
2682                .submit(
2683                    new_command_id("phone-prompt")?,
2684                    RelayCommand::Prompt {
2685                        prompt: phone_prompt_blocks(text, images),
2686                    },
2687                )
2688                .await?;
2689            Ok(())
2690        }
2691        ControllerAction::RunShell {
2692            session_id,
2693            command,
2694        } => {
2695            services
2696                .sessions
2697                .wait_for_session(&session_id, Duration::from_secs(5))
2698                .await?
2699                .submit(
2700                    new_command_id("phone-shell")?,
2701                    RelayCommand::RunUserShell { command },
2702                )
2703                .await?;
2704            Ok(())
2705        }
2706        ControllerAction::CancelShell {
2707            session_id,
2708            shell_command_id,
2709        } => {
2710            services
2711                .sessions
2712                .wait_for_session(&session_id, Duration::from_secs(5))
2713                .await?
2714                .submit(
2715                    new_command_id("phone-cancel-shell")?,
2716                    RelayCommand::CancelUserShell { shell_command_id },
2717                )
2718                .await?;
2719            Ok(())
2720        }
2721        ControllerAction::Close { session_id } => {
2722            services.daemon_runtime.close_session(session_id).await
2723        }
2724        ControllerAction::ForceClose { session_id } => {
2725            services
2726                .daemon_runtime
2727                .force_destroy_session(session_id)
2728                .await
2729        }
2730        ControllerAction::Resume {
2731            session_id,
2732            workspace_id,
2733            profile_id,
2734            target_id,
2735            queue,
2736            additional_mounts,
2737            resource_allocation,
2738        } => services
2739            .daemon_runtime
2740            .resume_session(ResumeSessionRequest {
2741                session_id,
2742                workspace_id,
2743                profile_id,
2744                target_template_id: target_id,
2745                additional_mounts,
2746                resource_allocation,
2747                discard_queue: queue == ResumeQueueDisposition::Discard,
2748                repository_preflight: None,
2749            })
2750            .await
2751            .map(|_| ()),
2752        ControllerAction::Move { request } => {
2753            let outcome = services.daemon_runtime.move_session(request).await?;
2754            match outcome.outcome.as_str() {
2755                "completed" | "unchanged" => Ok(()),
2756                "cancelled" | "failed" => {
2757                    // The daemon keeps detailed diagnostics in its durable
2758                    // operation record. Only its safe recovery guidance is
2759                    // copied into the phone action error, where the normal
2760                    // failed-action path records a visible session error.
2761                    let recovery = outcome.recovery.unwrap_or_else(|| {
2762                        "Inspect the session and retry Move or Resume with previous settings."
2763                            .into()
2764                    });
2765                    bail!("Move {}: {recovery}", outcome.outcome)
2766                }
2767                status => bail!("Move returned an unknown outcome: {status}"),
2768            }
2769        }
2770        ControllerAction::Open { .. } => Ok(()),
2771        ControllerAction::Cancel { .. } => {
2772            bail!("cancel actions must be handled by the phone control loop")
2773        }
2774        ControllerAction::RemoveQueuedPrompt {
2775            session_id,
2776            queue_id,
2777        } => {
2778            services
2779                .sessions
2780                .session(&session_id)
2781                .await?
2782                .submit(
2783                    new_command_id("phone-remove-prompt")?,
2784                    RelayCommand::RemoveQueuedPrompt {
2785                        queued_command_id: queue_id,
2786                    },
2787                )
2788                .await?;
2789            Ok(())
2790        }
2791        ControllerAction::RespondElicitation {
2792            session_id,
2793            elicitation_id,
2794            response,
2795        } => {
2796            services
2797                .sessions
2798                .session(&session_id)
2799                .await?
2800                .respond_elicitation(elicitation_id, response)
2801                .await
2802        }
2803        ControllerAction::Rename { session_id, title } => {
2804            controller.rename_session(&session_id, &title)?;
2805            Ok(())
2806        }
2807        ControllerAction::StartReview { session_id } => {
2808            // The refusal is a sentence for the person holding the phone --
2809            // "prompts are queued", "set [review] profile in config.toml" --
2810            // so it travels as the error text of this action.
2811            services
2812                .daemon_runtime
2813                .review_host()
2814                .start(&session_id, true)
2815                .await
2816                .map_err(|refusal| anyhow::anyhow!("{refusal}"))?;
2817            Ok(())
2818        }
2819        ControllerAction::ResolveReview {
2820            session_id,
2821            resolution,
2822        } => {
2823            let resolution = crate::server::resolution_from_name(&resolution)
2824                .context("a review is resolved by forward, dismiss, or cancel")?;
2825            services
2826                .daemon_runtime
2827                .review_host()
2828                .resolve(&session_id, resolution)
2829                .await
2830                .map_err(|error| anyhow::anyhow!("{error}"))?;
2831            Ok(())
2832        }
2833        ControllerAction::CancelTurn { session_id } => {
2834            services
2835                .sessions
2836                .session(&session_id)
2837                .await?
2838                .submit(new_command_id("phone-cancel-turn")?, RelayCommand::Cancel)
2839                .await?;
2840            Ok(())
2841        }
2842        ControllerAction::SetConfig {
2843            session_id,
2844            key,
2845            value,
2846        } => {
2847            services
2848                .sessions
2849                .session(&session_id)
2850                .await?
2851                .submit(
2852                    new_command_id("phone-set-config")?,
2853                    RelayCommand::SetConfig { key, value },
2854                )
2855                .await?;
2856            Ok(())
2857        }
2858        ControllerAction::SetPlanMode { session_id, active } => {
2859            // Which call turns plan mode on is a fact about the harness, so it
2860            // is asked of the shared decision rather than decided here or, far
2861            // worse, in the browser.
2862            let harness_kind = controller
2863                .state
2864                .sessions
2865                .get(&session_id)
2866                .with_context(|| format!("unknown session {session_id}"))?
2867                .harness_kind;
2868            let handle = services.sessions.session(&session_id).await?;
2869            let operational = handle
2870                .view()
2871                .snapshot
2872                .map(|snapshot| snapshot.operational)
2873                .context("the session has not reported what it supports yet")?;
2874            let facts = mj_core::acp::AcpSessionFacts::from_operational(
2875                harness_kind,
2876                &operational.config,
2877                &operational.config_options,
2878                operational.modes.as_ref(),
2879            );
2880            let command = match facts.plan_control(active) {
2881                Ok(mj_core::acp::PlanControl::SetConfig { key, value }) => {
2882                    RelayCommand::SetConfig { key, value }
2883                }
2884                Ok(mj_core::acp::PlanControl::SetSessionMode { mode_id }) => {
2885                    RelayCommand::SetSessionMode { mode_id }
2886                }
2887                Err(reason) => bail!("{reason}"),
2888            };
2889            handle
2890                .submit(new_command_id("phone-plan-mode")?, command)
2891                .await?;
2892            Ok(())
2893        }
2894        // Refreshes are handled by the phone control loop, which owns the
2895        // pollers they nudge.
2896        ControllerAction::RefreshQuota { .. } | ControllerAction::RefreshCapacity { .. } => {
2897            bail!("refresh actions must be handled by the phone control loop")
2898        }
2899    }
2900}
2901
2902/// Inputs that can change a session's source without changing its ID.
2903#[derive(Clone, PartialEq, Eq)]
2904struct ProjectSourceKey {
2905    directory: Option<PathBuf>,
2906    worktree: Option<mj_core::state::ManagedWorktree>,
2907    target: Option<mj_core::config::TargetTemplate>,
2908    fallback: ProjectSourceIdentity,
2909}
2910
2911impl ProjectSourceKey {
2912    fn of(session: &SessionRecord, config: &Config) -> Self {
2913        Self {
2914            directory: session.project_directory.clone(),
2915            worktree: session.managed_worktree.clone(),
2916            target: config.targets.get(&session.target_template_id).cloned(),
2917            fallback: session.project_source(config),
2918        }
2919    }
2920}
2921
2922struct ProjectSourceEntry {
2923    key: ProjectSourceKey,
2924    source: Option<ProjectSourceIdentity>,
2925    retry_at: Option<Instant>,
2926    cancelled: Arc<AtomicBool>,
2927}
2928
2929struct ProjectSourceResolved {
2930    cancelled: Arc<AtomicBool>,
2931    session_id: String,
2932    key: ProjectSourceKey,
2933    result: Result<ProjectSourceIdentity, String>,
2934}
2935
2936/// Git/SSH probes run independently of snapshot publication and are bounded
2937/// and cancelled when their inputs disappear or the server shuts down.
2938#[derive(Default)]
2939struct PhoneProjectSources {
2940    entries: std::collections::BTreeMap<String, ProjectSourceEntry>,
2941    jobs: tokio::task::JoinSet<ProjectSourceResolved>,
2942}
2943
2944impl PhoneProjectSources {
2945    fn synchronize(&mut self, controller: &Controller) {
2946        self.entries.retain(|id, entry| {
2947            let keep = controller.state.sessions.get(id).is_some_and(|session| {
2948                session.project_directory.is_some()
2949                    && entry.key == ProjectSourceKey::of(session, &controller.config)
2950            });
2951            if !keep {
2952                entry.cancelled.store(true, Ordering::Release);
2953            }
2954            keep
2955        });
2956        for session in controller.state.sessions.values() {
2957            if self.jobs.len() >= 8 {
2958                break;
2959            }
2960            if session.project_directory.is_none()
2961                || self.entries.get(&session.id).is_some_and(|entry| {
2962                    entry
2963                        .retry_at
2964                        .is_none_or(|deadline| Instant::now() < deadline)
2965                })
2966            {
2967                continue;
2968            }
2969            let key = ProjectSourceKey::of(session, &controller.config);
2970            let cancelled = Arc::new(AtomicBool::new(false));
2971            self.entries.insert(
2972                session.id.clone(),
2973                ProjectSourceEntry {
2974                    key: key.clone(),
2975                    source: None,
2976                    retry_at: None,
2977                    cancelled: cancelled.clone(),
2978                },
2979            );
2980            let source_controller = Controller {
2981                config: controller.config.clone(),
2982                state: State {
2983                    sessions: [(session.id.clone(), session.clone())]
2984                        .into_iter()
2985                        .collect(),
2986                    ..State::default()
2987                },
2988            };
2989            let session_id = session.id.clone();
2990            self.jobs.spawn_blocking(move || {
2991                let executor = CancellableProcessExecutor::new(cancelled.clone())
2992                    .with_deadline(Duration::from_secs(8));
2993                let result = source_controller
2994                    .resolve_session_project_source(&session_id, &executor)
2995                    .map_err(|error| format!("{error:#}"));
2996                ProjectSourceResolved {
2997                    cancelled,
2998                    session_id,
2999                    key,
3000                    result,
3001                }
3002            });
3003        }
3004    }
3005
3006    fn complete(&mut self, resolved: ProjectSourceResolved) {
3007        let Some(entry) = self.entries.get_mut(&resolved.session_id) else {
3008            return;
3009        };
3010        if entry.key != resolved.key || !Arc::ptr_eq(&entry.cancelled, &resolved.cancelled) {
3011            return;
3012        }
3013        match resolved.result {
3014            Ok(source) => entry.source = Some(source),
3015            Err(error) => {
3016                tracing::warn!(session_id = %resolved.session_id, %error, "could not resolve web project source");
3017                entry.retry_at = Some(Instant::now() + Duration::from_secs(30));
3018            }
3019        }
3020    }
3021
3022    fn source(&self, session: &SessionRecord, config: &Config) -> Option<&ProjectSourceIdentity> {
3023        self.entries
3024            .get(&session.id)
3025            .filter(|entry| entry.key == ProjectSourceKey::of(session, config))
3026            .and_then(|entry| entry.source.as_ref())
3027    }
3028}
3029
3030impl Drop for PhoneProjectSources {
3031    fn drop(&mut self) {
3032        for entry in self.entries.values() {
3033            entry.cancelled.store(true, Ordering::Release);
3034        }
3035    }
3036}
3037
3038/// The live, per-session projections the phone snapshot layers on top of the
3039/// controller's durable state. They arrive from relay snapshots rather than
3040/// from disk, so they travel together instead of as separate arguments.
3041struct PhoneSessionViews<'a> {
3042    conversations: &'a std::collections::BTreeMap<String, crate::server::BrowserTranscript>,
3043    queued_prompts: &'a std::collections::BTreeMap<String, Vec<mj_core::relay::QueuedPrompt>>,
3044    active_user_shells:
3045        &'a std::collections::BTreeMap<String, Vec<mj_core::relay::ActiveUserShell>>,
3046    pending_elicitations:
3047        &'a std::collections::BTreeMap<String, Vec<mj_core::elicitation::ElicitationRequest>>,
3048    /// Sessions whose agent advertised image support in prompts.
3049    prompt_images: &'a std::collections::BTreeSet<String>,
3050    /// What each managed session's relay last reported. This is where the
3051    /// projection learns what the agent can do, rather than guessing from the
3052    /// durable record, which knows only what was configured.
3053    operational: &'a std::collections::BTreeMap<String, mj_core::relay::RelayOperationalState>,
3054    /// Durable activity watermarks delivered with the materialized worker
3055    /// snapshots. Keeping this in the control-loop cache avoids a database
3056    /// read while rendering each viewer snapshot.
3057    materialized_activity: &'a std::collections::BTreeMap<String, Option<i64>>,
3058    project_sources: &'a PhoneProjectSources,
3059    /// Lifecycle operations running now, keyed by session.
3060    operations: &'a std::collections::BTreeMap<String, crate::server::ViewerOperation>,
3061    /// Durable Move records, projected without diagnostics or checkpoint paths.
3062    move_recoveries: &'a std::collections::BTreeMap<String, ViewerMoveRecovery>,
3063    /// The most recent capacity reading per probe target.
3064    capacity: &'a [crate::server::ViewerTargetCapacity],
3065    launch_failures: &'a [crate::server::ViewerLaunchFailure],
3066    /// Reviews the daemon is running, keyed by session. The phone renders the
3067    /// same review the terminal does, from the same host.
3068    reviews: &'a std::collections::BTreeMap<String, crate::review_host::RuntimeReviewView>,
3069}
3070
3071/// What the phone server remembers about one probe target between readings.
3072///
3073/// The last good reading is kept beside any failure, because one failed probe
3074/// is not a reason to forget what a machine was doing a minute ago; the phone
3075/// is told both, and says so.
3076#[derive(Debug, Clone)]
3077struct PhoneCapacity {
3078    target: crate::targets::DeploymentCapacityTarget,
3079    usage: Option<crate::targets::DeploymentCapacityUsage>,
3080    on_demand: bool,
3081    sampled_at_epoch_seconds: Option<u64>,
3082    refreshing: bool,
3083    failed: bool,
3084}
3085
3086/// How old a reading may be before the page says so.
3087const CAPACITY_STALE_AFTER: Duration = Duration::from_secs(120);
3088
3089/// Tell the poller which targets to probe, and keep the state map in step.
3090fn publish_capacity_targets(
3091    controller: &Controller,
3092    targets_tx: &tokio::sync::watch::Sender<Vec<crate::targets::DeploymentCapacityTarget>>,
3093    state: &mut std::collections::BTreeMap<String, PhoneCapacity>,
3094) {
3095    let targets = controller.deployment_capacity_targets();
3096    state.retain(|id, _| targets.iter().any(|target| target.id == *id));
3097    for target in &targets {
3098        state
3099            .entry(target.id.clone())
3100            .and_modify(|entry| entry.target = target.clone())
3101            .or_insert_with(|| PhoneCapacity {
3102                target: target.clone(),
3103                usage: None,
3104                on_demand: false,
3105                sampled_at_epoch_seconds: None,
3106                // A target with no reading yet is loading, not idle.
3107                refreshing: true,
3108                failed: false,
3109            });
3110    }
3111    if targets_tx.borrow().as_slice() != targets.as_slice() {
3112        targets_tx.send_replace(targets);
3113    }
3114}
3115
3116/// Project the capacity readings for the phone.
3117fn viewer_capacity(
3118    state: &std::collections::BTreeMap<String, PhoneCapacity>,
3119) -> Vec<crate::server::ViewerTargetCapacity> {
3120    let now = std::time::SystemTime::now()
3121        .duration_since(std::time::UNIX_EPOCH)
3122        .unwrap_or_default()
3123        .as_secs();
3124    state
3125        .values()
3126        .map(|entry| {
3127            let usage = entry.usage.as_ref();
3128            crate::server::ViewerTargetCapacity {
3129                id: entry.target.id.clone(),
3130                label: entry.target.host.clone(),
3131                target_ids: entry.target.target_ids.clone(),
3132                cpu_percent: usage.and_then(|usage| usage.cpu_percent),
3133                memory_used_bytes: usage.map(|usage| usage.memory_used_bytes),
3134                memory_total_bytes: usage.map(|usage| usage.memory_total_bytes),
3135                logical_cores: usage.map(|usage| usage.logical_cores),
3136                disk_total_bytes: usage.and_then(|usage| usage.disk_total_bytes),
3137                // A fleet reports how many machines it is running; a plain host
3138                // has no such count and says nothing rather than zero.
3139                virtual_machines: matches!(
3140                    entry.target.kind,
3141                    crate::targets::DeploymentCapacityKind::AwsFleet
3142                )
3143                .then(|| u64::from(!entry.on_demand)),
3144                sampled_at_epoch_seconds: entry.sampled_at_epoch_seconds,
3145                refreshing: entry.refreshing,
3146                stale: entry.sampled_at_epoch_seconds.is_some_and(|sampled| {
3147                    now.saturating_sub(sampled) > CAPACITY_STALE_AFTER.as_secs()
3148                }),
3149                has_error: entry.failed,
3150            }
3151        })
3152        .collect()
3153}
3154
3155/// Turn one lifecycle operation into the projection a phone follows.
3156fn viewer_operation(view: &crate::daemon::RuntimeLifecycleView) -> crate::server::ViewerOperation {
3157    use crate::server::{ViewerOperationKind, ViewerOperationStage};
3158
3159    crate::server::ViewerOperation {
3160        // RuntimeLifecycle owns the identity. A session can have consecutive
3161        // operations, and a client must be able to retire a late response from
3162        // the previous one without mistaking it for the current operation.
3163        id: view.operation_id.clone(),
3164        session_id: view.session_id.clone(),
3165        kind: match view.kind {
3166            crate::daemon::RuntimeLifecycleKind::Create => ViewerOperationKind::Create,
3167            crate::daemon::RuntimeLifecycleKind::Resume => ViewerOperationKind::Resume,
3168            crate::daemon::RuntimeLifecycleKind::Move => ViewerOperationKind::Move,
3169            // Stop, destroy, and retained cleanup remain distinct so a phone
3170            // can describe which part of teardown owns the session.
3171            crate::daemon::RuntimeLifecycleKind::Close
3172            | crate::daemon::RuntimeLifecycleKind::ForceStop => ViewerOperationKind::Stop,
3173            crate::daemon::RuntimeLifecycleKind::DestroyStopped
3174            | crate::daemon::RuntimeLifecycleKind::ForceDestroy => ViewerOperationKind::Destroy,
3175            crate::daemon::RuntimeLifecycleKind::Cleanup => ViewerOperationKind::Cleanup,
3176        },
3177        started_at_epoch_seconds: view.started_at_epoch_seconds,
3178        stages: view
3179            .active_stages
3180            .iter()
3181            .map(|(stage, started_at)| ViewerOperationStage {
3182                label: stage.label(),
3183                started_at_epoch_seconds: *started_at,
3184            })
3185            .collect(),
3186        notice: view.notice.clone(),
3187        cancellable: view.cancellable,
3188    }
3189}
3190
3191/// What the phone may do with one session.
3192///
3193/// Everything here is a fact the controller holds and the browser cannot:
3194/// whether the session manager is driving this session, what the agent said it
3195/// supports, and whether a lifecycle operation already owns it.
3196fn session_capabilities(
3197    session: &crate::server::ViewerSession,
3198    operational: Option<&mj_core::relay::RelayOperationalState>,
3199    operation: Option<&crate::server::ViewerOperation>,
3200    facts: Option<&mj_core::acp::AcpSessionFacts>,
3201) -> crate::server::ViewerSessionCapabilities {
3202    use crate::server::ViewerLifecycleCategory;
3203
3204    let live = session.lifecycle == ViewerLifecycleCategory::Live;
3205    // A session the manager is not driving cannot be talked to, whatever its
3206    // durable state says.
3207    let attached = operational.is_some();
3208    // A failed close/destroy can leave its durable record in an intermediate
3209    // state after the lifecycle owner has gone away. Keep the status card
3210    // selectable, but do not turn the failure into an endless mutation lock:
3211    // the published Stop capability is the recovery action.
3212    let transition_busy = session.transitioning && !session.has_error;
3213    let busy = operation.is_some() || transition_busy;
3214    // A failed move can retain a live destination while queue admission is
3215    // incomplete. The durable move hold owns all mutating session controls in
3216    // that interval; retry Move is the one intentional exception.
3217    let partial_move_queue = session.move_recovery.as_ref().is_some_and(|recovery| {
3218        recovery.queue_admission_started && !recovery.queue_admission_finished
3219    });
3220    let mutation_busy = busy || partial_move_queue;
3221    let idle = operational
3222        .is_some_and(|state| state.execution == mj_core::relay::RelayExecutionState::Idle);
3223    crate::server::ViewerSessionCapabilities {
3224        open: session.conversation_available
3225            && !session.transitioning
3226            && session.lifecycle == ViewerLifecycleCategory::Live,
3227        prompt: live && attached && !mutation_busy,
3228        run_shell: live && attached && !mutation_busy,
3229        cancel_turn: live
3230            && !mutation_busy
3231            && operational.is_some_and(|state| {
3232                state.active_prompt.is_some() || state.capacity_retry.is_some()
3233            }),
3234        cancel_operation: operation.is_some_and(|operation| operation.cancellable),
3235        // Stopping a session that is already stopping asks for something that
3236        // is happening; resuming one that is running asks for a second copy.
3237        stop: session.lifecycle.is_dashboard_visible() && !mutation_busy,
3238        rename: !session.transitioning,
3239        resume: !session.lifecycle.is_dashboard_visible() && !mutation_busy,
3240        move_session: live && !busy,
3241        set_config: live && attached && facts.is_some() && !mutation_busy,
3242        // Plan mode is a turn boundary: the terminal offers it only while the
3243        // agent is idle, and the phone must not be looser.
3244        set_plan_mode: live
3245            && !mutation_busy
3246            && idle
3247            && facts.is_some_and(mj_core::acp::AcpSessionFacts::supports_plan_mode),
3248    }
3249}
3250
3251/// The ACP content blocks one phone prompt becomes: its text, then each
3252/// attached image as the image block the prompt path already carries.
3253fn phone_prompt_blocks(
3254    text: String,
3255    images: Vec<crate::server::ViewerPromptImage>,
3256) -> Vec<agent_client_protocol::schema::v1::ContentBlock> {
3257    use agent_client_protocol::schema::v1::{ContentBlock, ImageContent, TextContent};
3258
3259    let mut prompt = Vec::with_capacity(images.len() + 1);
3260    if !text.is_empty() {
3261        prompt.push(ContentBlock::Text(TextContent::new(text)));
3262    }
3263    prompt.extend(images.into_iter().map(|image| match image.attachment {
3264        Some(reference) => reference.content_block(),
3265        None => ContentBlock::Image(ImageContent::new(image.data_base64, image.mime_type)),
3266    }));
3267    prompt
3268}
3269
3270/// The Mjolnir commands a phone may offer for one session.
3271///
3272/// The list is built here, from what this session can actually do, and
3273/// published: the browser used to keep its own copy, which is how `/review`
3274/// was missing from the phone while the terminal had it.
3275fn phone_commands(
3276    session: &crate::server::ViewerSession,
3277    operational: Option<&mj_core::relay::RelayOperationalState>,
3278) -> Vec<crate::server::ViewerMjCommand> {
3279    use crate::server::ViewerCommandSource;
3280    use agent_client_protocol::schema::v1::AvailableCommandInput;
3281
3282    let command =
3283        |name: &str, description: &str, argument: Option<&str>| crate::server::ViewerMjCommand {
3284            name: name.to_owned(),
3285            description: description.to_owned(),
3286            source: ViewerCommandSource::Mj,
3287            argument: argument.map(str::to_owned),
3288        };
3289    let mut commands = vec![
3290        command("help", "show available Mjolnir and agent commands", None),
3291        command(
3292            "detach",
3293            "leave the conversation without stopping the worker",
3294            None,
3295        ),
3296    ];
3297    let option = |key: &str| {
3298        session
3299            .config_options
3300            .iter()
3301            .any(|option| option.key == key)
3302    };
3303    if option("model") {
3304        commands.push(command("model", "change the active model", Some("value")));
3305        commands.push(command("fast", "toggle Codex Fast mode", None));
3306    }
3307    if option("effort") {
3308        commands.push(command(
3309            "effort",
3310            "change the active reasoning effort",
3311            Some("value"),
3312        ));
3313    }
3314    if session.plan_mode_active.is_some() && session.capabilities.set_plan_mode {
3315        commands.push(command("plan", "toggle plan mode", Some("message")));
3316        commands.push(command(
3317            "implement",
3318            "leave plan mode and implement",
3319            Some("instruction"),
3320        ));
3321    }
3322    if session.capabilities.prompt || session.turn_review.is_some() {
3323        commands.push(command(
3324            "review",
3325            "review the finished turn now, or report how review is configured",
3326            Some("status"),
3327        ));
3328    }
3329    // These names are handled by Mjolnir even when the corresponding control
3330    // is unavailable for this session. An agent cannot claim one and turn a
3331    // locally interpreted slash command into a misleading palette entry.
3332    let reserved = [
3333        "help",
3334        "detach",
3335        "model",
3336        "fast",
3337        "effort",
3338        "plan",
3339        "implement",
3340        "review",
3341    ];
3342    for advertised in operational
3343        .into_iter()
3344        .flat_map(|state| state.available_commands.iter())
3345    {
3346        let name = advertised.name.trim();
3347        if name.is_empty()
3348            || reserved
3349                .iter()
3350                .any(|local| name.eq_ignore_ascii_case(local))
3351            || commands
3352                .iter()
3353                .any(|existing| existing.name.eq_ignore_ascii_case(name))
3354        {
3355            continue;
3356        }
3357        let argument = advertised.input.as_ref().and_then(|input| match input {
3358            AvailableCommandInput::Unstructured(input) => {
3359                let hint = input.hint.trim();
3360                (!hint.is_empty()).then(|| hint.to_owned())
3361            }
3362            _ => None,
3363        });
3364        commands.push(crate::server::ViewerMjCommand {
3365            name: name.to_owned(),
3366            description: advertised.description.trim().to_owned(),
3367            source: ViewerCommandSource::Agent,
3368            argument,
3369        });
3370    }
3371    commands
3372}
3373
3374/// The open reviews, keyed by session, for one snapshot.
3375fn review_views(
3376    daemon_runtime: &Arc<RuntimeState>,
3377) -> std::collections::BTreeMap<String, crate::review_host::RuntimeReviewView> {
3378    daemon_runtime
3379        .review_host()
3380        .views()
3381        .into_iter()
3382        .map(|review| (review.session_id.clone(), review))
3383        .collect()
3384}
3385
3386type ViewerMoveRecoveries = std::collections::BTreeMap<String, crate::server::ViewerMoveRecovery>;
3387
3388/// Refresh durable Move records away from the phone event loop. A Move can
3389/// finish after its initiating request disconnects, so active lifecycle views
3390/// alone are not enough to render Retry move or Resume with previous settings.
3391fn request_move_recovery_reload(
3392    completed: &tokio::sync::mpsc::UnboundedSender<Result<ViewerMoveRecoveries, String>>,
3393    in_flight: &mut bool,
3394    jobs: &mut tokio::task::JoinSet<()>,
3395) {
3396    if *in_flight {
3397        return;
3398    }
3399    *in_flight = true;
3400    let completed = completed.clone();
3401    jobs.spawn(async move {
3402        let result = match tokio::task::spawn_blocking(|| {
3403            crate::database::load_move_operations()
3404                .map(|operations| {
3405                    operations
3406                        .into_iter()
3407                        .filter_map(|operation| {
3408                            crate::server::ViewerMoveRecovery::from_operation(&operation)
3409                                .map(|recovery| (operation.selection.session_id.clone(), recovery))
3410                        })
3411                        .collect()
3412                })
3413                .map_err(|error| format!("{error:#}"))
3414        })
3415        .await
3416        {
3417            Ok(result) => result,
3418            Err(error) => Err(format!("move recovery projection task failed: {error}")),
3419        };
3420        let _ = completed.send(result);
3421    });
3422}
3423
3424fn viewer_snapshot(
3425    controller: &Controller,
3426    workspaces: &[mj_core::workspace::WorkspaceRecord],
3427    quotas: &std::collections::BTreeMap<String, ProfileQuota>,
3428    views: &PhoneSessionViews<'_>,
3429    revision: u64,
3430) -> ViewerSnapshot {
3431    let PhoneSessionViews {
3432        conversations,
3433        reviews,
3434        queued_prompts,
3435        active_user_shells,
3436        pending_elicitations,
3437        prompt_images,
3438        operational,
3439        materialized_activity,
3440        project_sources,
3441        operations,
3442        move_recoveries,
3443        capacity,
3444        launch_failures,
3445    } = views;
3446    let mut snapshot =
3447        ViewerSnapshot::from_config_state(&controller.config, &controller.state, revision);
3448    snapshot.launch_failures = launch_failures.to_vec();
3449    snapshot.workspaces = workspaces
3450        .iter()
3451        .map(|workspace| crate::server::ViewerWorkspace {
3452            id: workspace.id.clone(),
3453            name: workspace.name.clone(),
3454        })
3455        .collect();
3456    let now = std::time::SystemTime::now()
3457        .duration_since(std::time::UNIX_EPOCH)
3458        .unwrap_or_default()
3459        .as_secs();
3460    for profile in &mut snapshot.profiles {
3461        let Some(quota) = quotas.get(&profile.id) else {
3462            continue;
3463        };
3464        profile.quota = Some(ViewerQuota {
3465            summary: quota.compact(),
3466            windows: quota
3467                .windows
3468                .iter()
3469                .map(|window| crate::server::ViewerQuotaWindow {
3470                    label: window.label.clone(),
3471                    // The controller reports headroom; a bar fills as a limit
3472                    // is consumed, so the phone is given the complement.
3473                    percent_used: window
3474                        .remaining_percent
3475                        .map(|left| 100_u8.saturating_sub(left)),
3476                    resets_at: window.resets.clone(),
3477                    projects_exhaustion_before_reset: crate::quota::projects_exhaustion(
3478                        window,
3479                        quota.refreshed_at_epoch_seconds,
3480                    ),
3481                })
3482                .collect(),
3483            resets_at: quota
3484                .windows
3485                .iter()
3486                .find_map(|window| window.resets.clone()),
3487            stale: now.saturating_sub(quota.refreshed_at_epoch_seconds)
3488                > QUOTA_STALE_AFTER.as_secs(),
3489            refreshed_at_epoch_seconds: quota.refreshed_at_epoch_seconds,
3490            has_error: quota.error.is_some(),
3491        });
3492    }
3493    for session in &mut snapshot.sessions {
3494        session.move_recovery = move_recoveries.get(&session.id).cloned();
3495        if let Some(record) = controller.state.sessions.get(&session.id)
3496            && let Some(source) = project_sources.source(record, &controller.config)
3497        {
3498            session.set_project_source(source);
3499        }
3500        session.last_activity_at_ms = materialized_activity.get(&session.id).copied().flatten();
3501        session.queued_prompts = queued_prompts
3502            .get(&session.id)
3503            .into_iter()
3504            .flatten()
3505            .map(|prompt| ViewerQueuedPrompt {
3506                id: prompt.id.clone(),
3507                text: prompt.text.clone(),
3508                created_at: prompt.created_at_ms.to_string(),
3509            })
3510            .collect();
3511        session.active_user_shells = active_user_shells
3512            .get(&session.id)
3513            .into_iter()
3514            .flatten()
3515            .map(|shell| ViewerUserShell {
3516                id: shell.command_id.clone(),
3517                command: shell.command.clone(),
3518                started_at_ms: shell.started_at_ms,
3519            })
3520            .collect();
3521        session.background_tasks = operational
3522            .get(&session.id)
3523            .map(|state| {
3524                state
3525                    .background_commands
3526                    .iter()
3527                    .map(|task| ViewerBackgroundTask {
3528                        id: task.id.clone(),
3529                        command: task.command.clone(),
3530                        started_at_ms: task.started_at_ms,
3531                        can_stop: task.can_stop,
3532                    })
3533                    .collect()
3534            })
3535            .unwrap_or_default();
3536        session.pending_elicitations = pending_elicitations
3537            .get(&session.id)
3538            .cloned()
3539            .unwrap_or_default();
3540        session.prompt_images_supported = prompt_images.contains(&session.id);
3541        session.operation = operations.get(&session.id).cloned();
3542        // Runtime ownership takes precedence over the durable record. Move
3543        // can still look Running while its old conversation is no longer a
3544        // valid destination, and stopped cleanup/destroy operations have no
3545        // live lifecycle category of their own.
3546        if let Some(operation) = session.operation.as_ref()
3547            && operation.kind.transition_kind().is_some()
3548        {
3549            session.transitioning = true;
3550        }
3551        let live = operational.get(&session.id);
3552        let facts = live.map(|state| {
3553            mj_core::acp::AcpSessionFacts::from_operational(
3554                controller
3555                    .state
3556                    .sessions
3557                    .get(&session.id)
3558                    .map_or(mj_core::config::HarnessKind::Codex, |record| {
3559                        record.harness_kind
3560                    }),
3561                &state.config,
3562                &state.config_options,
3563                state.modes.as_ref(),
3564            )
3565        });
3566        if let Some(state) = live {
3567            session.latest_event_ordinal = state.latest_ordinal;
3568            session.chat_phase = match state.execution {
3569                mj_core::relay::RelayExecutionState::Idle => crate::server::ViewerChatPhase::Idle,
3570                mj_core::relay::RelayExecutionState::Running => {
3571                    crate::server::ViewerChatPhase::Running
3572                }
3573                mj_core::relay::RelayExecutionState::Closing => {
3574                    crate::server::ViewerChatPhase::Closing
3575                }
3576                mj_core::relay::RelayExecutionState::Closed => {
3577                    crate::server::ViewerChatPhase::Closed
3578                }
3579            };
3580            session.config_options = crate::server::viewer_config_options(
3581                &state.config_options,
3582                facts
3583                    .as_ref()
3584                    .expect("live operational state always has ACP session facts"),
3585            );
3586            // Share activity classification with the terminal. The browser
3587            // retains its detailed turn/step/background clock presentation.
3588            let turn_started_at_ms = state
3589                .active_prompt
3590                .as_ref()
3591                .map(|prompt| prompt.started_at_ms)
3592                .or_else(|| state.harness_turn.map(|turn| turn.started_at_ms));
3593            let turn_started_at = turn_started_at_ms
3594                .and_then(|started_at_ms| u64::try_from(started_at_ms).ok())
3595                .map(|started_at_ms| started_at_ms / 1_000);
3596            session.capacity_retry = state.capacity_retry.clone();
3597            let activity = mj_client::usage_format::SessionActivity::of(state);
3598            let activity_details =
3599                activity.details(turn_started_at_ms, state.current_step_started_at_ms);
3600            session.activity_details = Some(viewer_activity_details(&activity_details));
3601            session.is_idle = controller
3602                .state
3603                .sessions
3604                .get(&session.id)
3605                .is_some_and(|record| record.state == mj_core::state::SessionState::Running)
3606                && session.operation.is_none()
3607                && activity.is_idle(turn_started_at);
3608            session.activity = mj_client::usage_format::format_activity_columns(
3609                now,
3610                turn_started_at,
3611                state
3612                    .current_step_started_at_ms
3613                    .and_then(|value| u64::try_from(value).ok()),
3614                &activity,
3615            )
3616            .join("  ")
3617            .trim()
3618            .to_owned();
3619        }
3620        session.plan_mode_active = facts
3621            .as_ref()
3622            .filter(|facts| facts.supports_plan_mode())
3623            .map(mj_core::acp::AcpSessionFacts::plan_mode_active);
3624        session.turn_review = reviews
3625            .get(&session.id)
3626            .map(crate::server::ViewerTurnReview::from_runtime);
3627        session.capabilities =
3628            session_capabilities(session, live, operations.get(&session.id), facts.as_ref());
3629        session.available_commands = phone_commands(session, live);
3630        if let Some(transcript) = conversations.get(&session.id) {
3631            session.conversation_available = true;
3632            if !session.transitioning {
3633                let mut lines = transcript
3634                    .entries
3635                    .iter()
3636                    .flat_map(|entry| {
3637                        entry
3638                            .lines
3639                            .iter()
3640                            .enumerate()
3641                            .filter_map(move |(index, line)| {
3642                                let line = line.trim();
3643                                (!line.is_empty()).then(|| {
3644                                    if index == 0 {
3645                                        format!("{}: {line}", entry.label)
3646                                    } else {
3647                                        line.to_owned()
3648                                    }
3649                                })
3650                            })
3651                    })
3652                    .collect::<Vec<_>>();
3653                session.preview = lines.split_off(lines.len().saturating_sub(4));
3654            }
3655        }
3656        // `conversation_available` is only known after the transcript loop
3657        // above, so the capability that depends on it is settled here.
3658        session.capabilities.open = session.conversation_available && !session.transitioning;
3659    }
3660    snapshot.capacity = capacity.to_vec();
3661    snapshot
3662}
3663
3664fn viewer_activity_details(
3665    details: &mj_client::usage_format::SessionActivityDetails,
3666) -> ViewerActivityDetails {
3667    ViewerActivityDetails {
3668        kind: match details.kind {
3669            mj_client::usage_format::SessionActivityKind::Turn => ViewerActivityKind::Turn,
3670            mj_client::usage_format::SessionActivityKind::Step => ViewerActivityKind::Step,
3671            mj_client::usage_format::SessionActivityKind::Background => {
3672                ViewerActivityKind::Background
3673            }
3674            mj_client::usage_format::SessionActivityKind::Idle => ViewerActivityKind::Idle,
3675            mj_client::usage_format::SessionActivityKind::Lifecycle
3676            | mj_client::usage_format::SessionActivityKind::Goal => ViewerActivityKind::Lifecycle,
3677        },
3678        turn_started_at_ms: details.turn_started_at_ms,
3679        step_started_at_ms: details.step_started_at_ms,
3680        background_started_at_ms: details.background_started_at_ms,
3681        idle_since_ms: details.idle_since_ms,
3682        label: details.label.clone(),
3683    }
3684}
3685
3686/// Seed the viewer's in-memory activity cache from the durable projection
3687/// before publishing its first snapshot. Later worker snapshots update this
3688/// cache without adding a database read to the render path.
3689async fn load_materialized_activity(
3690    controller: &Controller,
3691) -> Result<std::collections::BTreeMap<String, Option<i64>>> {
3692    let session_ids = controller
3693        .state
3694        .sessions
3695        .keys()
3696        .cloned()
3697        .collect::<Vec<_>>();
3698    tokio::task::spawn_blocking(move || {
3699        session_ids
3700            .into_iter()
3701            .map(|session_id| {
3702                crate::database::load_materialized_session_summary(&session_id).map(|summary| {
3703                    (
3704                        session_id,
3705                        summary.and_then(|summary| summary.last_activity_at_ms),
3706                    )
3707                })
3708            })
3709            .collect()
3710    })
3711    .await
3712    .context("materialized activity startup task failed")?
3713}
3714
3715#[cfg(test)]
3716mod tests {
3717    use super::*;
3718    use crate::pollers::QUOTA_REFRESH_INTERVAL;
3719    use std::collections::BTreeMap;
3720
3721    use agent_client_protocol::schema::v1::{
3722        SessionConfigOption, SessionConfigOptionCategory, SessionConfigSelectOption,
3723        SessionConfigSelectOptions,
3724    };
3725    use mj_core::config::{
3726        CONFIG_VERSION, Config, HarnessKind, ProjectBundle, ProjectRepository, TargetTemplate,
3727    };
3728    use mj_core::state::SessionState;
3729
3730    #[test]
3731    fn viewer_config_options_publish_current_advertised_values() {
3732        let make_options = |model, effort| {
3733            vec![
3734                SessionConfigOption::select(
3735                    "model_selector",
3736                    "Model",
3737                    model,
3738                    SessionConfigSelectOptions::Ungrouped(vec![
3739                        SessionConfigSelectOption::new("sonnet", "Claude Sonnet"),
3740                        SessionConfigSelectOption::new("opus", "Claude Opus"),
3741                    ]),
3742                )
3743                .category(SessionConfigOptionCategory::Model),
3744                SessionConfigOption::select(
3745                    "reasoning_effort",
3746                    "Effort",
3747                    effort,
3748                    SessionConfigSelectOptions::Ungrouped(vec![
3749                        SessionConfigSelectOption::new("high", "High"),
3750                        SessionConfigSelectOption::new("max", "Maximum"),
3751                    ]),
3752                ),
3753            ]
3754        };
3755        let options = make_options("sonnet", "high");
3756        let defaults = mj_core::acp::AcpSessionFacts::from_operational(
3757            HarnessKind::Claude,
3758            &BTreeMap::new(),
3759            &options,
3760            None,
3761        );
3762        let projected = crate::server::viewer_config_options(&options, &defaults);
3763        assert_eq!(
3764            projected
3765                .iter()
3766                .map(|option| (option.key.as_str(), option.current.as_deref()))
3767                .collect::<Vec<_>>(),
3768            [("model", Some("sonnet")), ("effort", Some("high"))]
3769        );
3770
3771        let options = make_options("opus", "max");
3772        let updated = mj_core::acp::AcpSessionFacts::from_operational(
3773            HarnessKind::Claude,
3774            &BTreeMap::new(),
3775            &options,
3776            None,
3777        );
3778        let projected = crate::server::viewer_config_options(&options, &updated);
3779        assert_eq!(projected[0].current.as_deref(), Some("opus"));
3780        assert_eq!(projected[1].current.as_deref(), Some("max"));
3781    }
3782
3783    #[tokio::test]
3784    async fn explicit_tls_takes_precedence_over_tailscale_detection() {
3785        let resolved = resolve_server_args(
3786            ServerArgs {
3787                bind: "0.0.0.0:4443".into(),
3788                tailscale_detect: true,
3789                tls_cert: Some(PathBuf::from("configured-cert.pem")),
3790                tls_key: Some(PathBuf::from("configured-key.pem")),
3791            },
3792            tokio_util::sync::CancellationToken::new(),
3793        )
3794        .await
3795        .unwrap();
3796
3797        assert_eq!(resolved.bind, "0.0.0.0:4443".parse().unwrap());
3798        assert_eq!(resolved.viewer_url, "https://0.0.0.0:4443");
3799        assert_eq!(
3800            resolved.tls_files,
3801            Some((
3802                PathBuf::from("configured-cert.pem"),
3803                PathBuf::from("configured-key.pem")
3804            ))
3805        );
3806        assert!(resolved.tailscale.is_none());
3807        assert!(resolved.fallback_reason.is_none());
3808    }
3809
3810    #[tokio::test]
3811    async fn disabling_detection_keeps_the_viewer_on_loopback() {
3812        let resolved = resolve_server_args(
3813            ServerArgs {
3814                bind: "127.0.0.1:4765".into(),
3815                tailscale_detect: false,
3816                tls_cert: None,
3817                tls_key: None,
3818            },
3819            tokio_util::sync::CancellationToken::new(),
3820        )
3821        .await
3822        .unwrap();
3823
3824        assert_eq!(resolved.bind, "127.0.0.1:4765".parse().unwrap());
3825        assert_eq!(resolved.viewer_url, "http://127.0.0.1:4765");
3826        assert!(
3827            resolved
3828                .fallback_reason
3829                .unwrap()
3830                .contains("detection is disabled")
3831        );
3832    }
3833
3834    fn bare_preflight_config() -> Config {
3835        let mut config = Config::default();
3836        config
3837            .targets
3838            .insert("raw".into(), TargetTemplate::LocalBare);
3839        config
3840    }
3841
3842    #[test]
3843    fn move_recovery_projection_exposes_safe_retry_settings_only() {
3844        let operation = mj_core::state::MoveOperation {
3845            source_checkpoint_only: false,
3846            operation_id: "move-1".into(),
3847            selection: mj_core::state::MoveSelection {
3848                clear_resource_allocation: true,
3849                session_id: "session-1".into(),
3850                profile_id: Some("destination-profile".into()),
3851                target_template_id: Some("destination-target".into()),
3852                additional_mounts: Some(vec![crate::targets::AdditionalMount {
3853                    source: "/destination/source".into(),
3854                    destination: "/destination/target".into(),
3855                    read_only: true,
3856                }]),
3857                resource_allocation: None,
3858            },
3859            source_profile_id: "source-profile".into(),
3860            source_target_template_id: "source-target".into(),
3861            source_target: None,
3862            source_native_session_id: Some("private-native-id".into()),
3863            source_additional_mounts: vec![crate::targets::AdditionalMount {
3864                source: "/source/source".into(),
3865                destination: "/source/target".into(),
3866                read_only: false,
3867            }],
3868            source_resource_allocation: Some(
3869                mj_core::state::SessionResourceAllocation::Container {
3870                    cpus: 2,
3871                    memory_bytes: 4096,
3872                },
3873            ),
3874            destination_target: None,
3875            destination_native_session_id: None,
3876            destination_store_id: None,
3877            configuration_fingerprint: "private-fingerprint".into(),
3878            checkpoint: None,
3879            recovery_session: None,
3880            queue: mj_core::state::ResumeQueueDisposition::Start,
3881            phase: mj_core::state::MovePhase::Cancelled,
3882            queue_admission_started: false,
3883            queue_admission_finished: false,
3884            cancellation_requested: true,
3885            created_at: "now".into(),
3886            updated_at: "now".into(),
3887            error: Some("private path and token".into()),
3888        };
3889        let recovery = ViewerMoveRecovery::from_operation(&operation).unwrap();
3890        assert_eq!(recovery.phase, "cancelled");
3891        assert_eq!(recovery.source_profile_id, "source-profile");
3892        assert!(recovery.clear_resource_allocation);
3893        assert_eq!(recovery.source_additional_mounts.len(), 1);
3894        assert!(recovery.source_resource_allocation.is_some());
3895        assert_eq!(recovery.destination_additional_mounts.len(), 1);
3896        assert!(recovery.destination_resource_allocation.is_none());
3897        let json = serde_json::to_string(&recovery).unwrap();
3898        assert!(!json.contains("private-native-id"));
3899        assert!(!json.contains("private-fingerprint"));
3900        assert!(!json.contains("private path and token"));
3901    }
3902
3903    #[test]
3904    fn new_preflight_rejects_a_bare_project_without_a_git_head() {
3905        let error = run_new_preflight(
3906            bare_preflight_config(),
3907            "hel".into(),
3908            "raw".into(),
3909            Some(PathBuf::from("/definitely/not/a/project")),
3910        )
3911        .expect_err("a missing project directory must fail preflight");
3912
3913        assert!(
3914            error
3915                .to_string()
3916                .contains("project directory does not exist or is not a directory")
3917        );
3918    }
3919
3920    #[test]
3921    fn new_preflight_accepts_a_git_project_for_a_bare_target() {
3922        let directory = std::env::current_dir().expect("the test has a working directory");
3923        let answer = run_new_preflight(
3924            bare_preflight_config(),
3925            "hel".into(),
3926            "raw".into(),
3927            Some(directory.clone()),
3928        )
3929        .expect("the repository running the test has a valid Git HEAD");
3930
3931        assert!(answer.dirty_repositories.is_empty());
3932        assert_eq!(answer.project_directory, Some(directory));
3933    }
3934
3935    #[test]
3936    fn new_preflight_requires_network_sources_for_isolated_targets() {
3937        let mut config = Config::default();
3938        config.targets.insert(
3939            "podman".into(),
3940            TargetTemplate::LocalPodman {
3941                container: mj_core::config::ContainerTemplate {
3942                    image: "test-image".into(),
3943                    pull_policy: Default::default(),
3944                    platform: None,
3945                    cpus: None,
3946                    memory: None,
3947                    environment: Default::default(),
3948                    workspace_storage: Default::default(),
3949                },
3950            },
3951        );
3952        config.bundles.insert(
3953            "hel".into(),
3954            ProjectBundle {
3955                primary_repo: "hel".into(),
3956                repositories: vec![ProjectRepository {
3957                    id: "hel".into(),
3958                    github: None,
3959                    local: Some(PathBuf::from("/definitely/not/a/repository")),
3960                    destination: "hel".into(),
3961                    git_ref: None,
3962                }],
3963            },
3964        );
3965        let error = run_new_preflight(config, "hel".into(), "podman".into(), None)
3966            .expect_err("an isolated bundle cannot use a local source");
3967        assert!(error.to_string().contains("repository"));
3968    }
3969
3970    #[test]
3971    fn a_phone_prompt_becomes_its_text_then_its_images() {
3972        use agent_client_protocol::schema::v1::ContentBlock;
3973
3974        let image = |data: &str| crate::server::ViewerPromptImage {
3975            attachment: None,
3976            data_base64: data.into(),
3977            mime_type: "image/png".into(),
3978            width: 32,
3979            height: 24,
3980        };
3981        let blocks = phone_prompt_blocks(
3982            "look at this".into(),
3983            vec![image("aW1hZ2U="), image("c2Vjb25k")],
3984        );
3985        let ContentBlock::Text(text) = &blocks[0] else {
3986            panic!("the prompt leads with its text");
3987        };
3988        assert_eq!(text.text, "look at this");
3989        let ContentBlock::Image(first) = &blocks[1] else {
3990            panic!("each attachment travels as an image block");
3991        };
3992        assert_eq!(first.data, "aW1hZ2U=");
3993        assert_eq!(first.mime_type, "image/png");
3994        assert!(matches!(blocks[2], ContentBlock::Image(_)));
3995        assert_eq!(blocks.len(), 3);
3996
3997        // An image needs no words with it, and an empty text block would be a
3998        // message the user never wrote.
3999        let images_only = phone_prompt_blocks(String::new(), vec![image("aW1hZ2U=")]);
4000        assert_eq!(images_only.len(), 1);
4001        assert!(matches!(images_only[0], ContentBlock::Image(_)));
4002    }
4003
4004    #[test]
4005    fn image_prompts_are_offered_only_after_the_agent_advertises_them() {
4006        use agent_client_protocol::schema::v1::AgentCapabilities;
4007        use mj_core::relay::{RelayExecutionState, RelayOperationalState};
4008
4009        let operational = |agent_capabilities| RelayOperationalState {
4010            goal: Default::default(),
4011            capacity_retry: None,
4012            activity_turn_started_at_ms: None,
4013            session_id: "session-1".into(),
4014            store_id: None,
4015            idle_since_ms: None,
4016            execution: RelayExecutionState::Idle,
4017            latest_ordinal: 0,
4018            latest_digest: String::new(),
4019            acknowledged_through: 0,
4020            acknowledged_digest: String::new(),
4021            recovery_floor_ordinal: 0,
4022            recovery_floor_digest: String::new(),
4023            native_session_id: None,
4024            native_continuity_lost: false,
4025            checkpoint_only: false,
4026            acp_ready: None,
4027            agent_capabilities,
4028            agent_info: None,
4029            steering_supported: None,
4030            config_options: Vec::new(),
4031            modes: None,
4032            available_commands: Vec::new(),
4033            config: std::collections::BTreeMap::new(),
4034            active_prompt: None,
4035            queued_prompts: Vec::new(),
4036            active_user_shells: Vec::new(),
4037            active_agent_terminals: Vec::new(),
4038            checkpoint_barrier: None,
4039            checkpoint_ready: None,
4040            last_acp_activity_at_ms: None,
4041            current_step_started_at_ms: None,
4042            foreground_tool_started_at_ms: None,
4043            harness_turn: None,
4044            last_harness_turn_started_ordinal: None,
4045            background_commands: Vec::new(),
4046            background_work_known: None,
4047        };
4048
4049        // A session whose agent has not answered `initialize` has advertised
4050        // nothing, so the phone is not offered a control the agent may refuse.
4051        assert!(!agent_accepts_prompt_images(&operational(None)));
4052        assert!(!agent_accepts_prompt_images(&operational(Some(Box::new(
4053            AgentCapabilities::default()
4054        )))));
4055        let mut capabilities = AgentCapabilities::default();
4056        capabilities.prompt_capabilities.image = true;
4057        assert!(agent_accepts_prompt_images(&operational(Some(Box::new(
4058            capabilities
4059        )))));
4060    }
4061
4062    #[test]
4063    fn phone_snapshot_projects_capability_gated_and_agent_commands_with_provenance() {
4064        use agent_client_protocol::schema::v1::{
4065            AvailableCommand, AvailableCommandInput, SessionMode, SessionModeState,
4066            UnstructuredCommandInput,
4067        };
4068        use mj_core::relay::{RelayExecutionState, RelayOperationalState};
4069
4070        use crate::server::ViewerCommandSource;
4071
4072        let mut controller = controller_with_profiles(&["claude"]);
4073        let mut record = phone_session("session-1", 0);
4074        record.harness_kind = HarnessKind::Claude;
4075        record.last_profile = "claude".into();
4076        record.state = SessionState::Running;
4077        controller.state.sessions.insert(record.id.clone(), record);
4078        let operational = RelayOperationalState {
4079            goal: Default::default(),
4080            capacity_retry: None,
4081            activity_turn_started_at_ms: None,
4082            session_id: "session-1".into(),
4083            store_id: None,
4084            idle_since_ms: None,
4085            execution: RelayExecutionState::Idle,
4086            latest_ordinal: 0,
4087            latest_digest: String::new(),
4088            acknowledged_through: 0,
4089            acknowledged_digest: String::new(),
4090            recovery_floor_ordinal: 0,
4091            recovery_floor_digest: String::new(),
4092            native_session_id: None,
4093            native_continuity_lost: false,
4094            checkpoint_only: false,
4095            acp_ready: None,
4096            agent_capabilities: None,
4097            agent_info: None,
4098            steering_supported: None,
4099            config_options: Vec::new(),
4100            modes: Some(SessionModeState::new(
4101                "default",
4102                vec![
4103                    SessionMode::new("default", "Default"),
4104                    SessionMode::new("plan", "Plan"),
4105                ],
4106            )),
4107            available_commands: vec![
4108                AvailableCommand::new("inspect", " Inspect the workspace ").input(
4109                    AvailableCommandInput::Unstructured(UnstructuredCommandInput::new(" query ")),
4110                ),
4111                AvailableCommand::new("Review", "agent collision"),
4112                AvailableCommand::new("INSPECT", "duplicate agent command"),
4113            ],
4114            config: std::collections::BTreeMap::new(),
4115            active_prompt: None,
4116            queued_prompts: Vec::new(),
4117            active_user_shells: Vec::new(),
4118            active_agent_terminals: Vec::new(),
4119            checkpoint_barrier: None,
4120            checkpoint_ready: None,
4121            last_acp_activity_at_ms: None,
4122            current_step_started_at_ms: None,
4123            foreground_tool_started_at_ms: None,
4124            harness_turn: None,
4125            last_harness_turn_started_ordinal: None,
4126            background_commands: Vec::new(),
4127            background_work_known: None,
4128        };
4129        let mut operational = std::collections::BTreeMap::from([("session-1".into(), operational)]);
4130        let materialized_activity =
4131            std::collections::BTreeMap::from([("session-1".into(), Some(7_777_i64))]);
4132        let project = |operational: &std::collections::BTreeMap<String, RelayOperationalState>| {
4133            viewer_snapshot(
4134                &controller,
4135                &[],
4136                &std::collections::BTreeMap::new(),
4137                &PhoneSessionViews {
4138                    conversations: &std::collections::BTreeMap::new(),
4139                    queued_prompts: &std::collections::BTreeMap::new(),
4140                    active_user_shells: &std::collections::BTreeMap::new(),
4141                    pending_elicitations: &std::collections::BTreeMap::new(),
4142                    prompt_images: &std::collections::BTreeSet::new(),
4143                    operational,
4144                    materialized_activity: &materialized_activity,
4145                    project_sources: &PhoneProjectSources::default(),
4146                    operations: &std::collections::BTreeMap::new(),
4147                    move_recoveries: &std::collections::BTreeMap::new(),
4148                    capacity: &[],
4149                    launch_failures: &[],
4150                    reviews: &std::collections::BTreeMap::new(),
4151                },
4152                1,
4153            )
4154        };
4155        let snapshot = project(&operational);
4156        let session = &snapshot.sessions[0];
4157
4158        assert_eq!(session.display_location, "podman");
4159        assert_eq!(session.last_activity_at_ms, Some(7_777));
4160        assert_eq!(
4161            session.activity_details,
4162            Some(crate::server::ViewerActivityDetails {
4163                kind: ViewerActivityKind::Idle,
4164                turn_started_at_ms: None,
4165                step_started_at_ms: None,
4166                background_started_at_ms: None,
4167                idle_since_ms: None,
4168                label: None,
4169            })
4170        );
4171
4172        assert!(session.capabilities.prompt);
4173        assert!(session.capabilities.set_plan_mode);
4174        assert_eq!(
4175            session
4176                .available_commands
4177                .iter()
4178                .map(|command| (command.name.as_str(), command.source))
4179                .collect::<Vec<_>>(),
4180            vec![
4181                ("help", ViewerCommandSource::Mj),
4182                ("detach", ViewerCommandSource::Mj),
4183                ("plan", ViewerCommandSource::Mj),
4184                ("implement", ViewerCommandSource::Mj),
4185                ("review", ViewerCommandSource::Mj),
4186                ("inspect", ViewerCommandSource::Agent),
4187            ]
4188        );
4189        let inspect = session.available_commands.last().unwrap();
4190        assert_eq!(inspect.description, "Inspect the workspace");
4191        assert_eq!(inspect.argument.as_deref(), Some("query"));
4192
4193        assert!(session.is_idle);
4194        assert_eq!(session.activity, "[idle]");
4195        let state = operational.get_mut("session-1").unwrap();
4196        state
4197            .background_commands
4198            .push(mj_core::relay::BackgroundCommand {
4199                id: "background-1".into(),
4200                started_at_ms: 1_000,
4201                command: "background check".into(),
4202                can_stop: true,
4203            });
4204        let background = project(&operational);
4205        assert_eq!(
4206            background.sessions[0].background_tasks,
4207            vec![ViewerBackgroundTask {
4208                id: "background-1".into(),
4209                command: "background check".into(),
4210                started_at_ms: 1_000,
4211                can_stop: true,
4212            }]
4213        );
4214        assert!(!background.sessions[0].is_idle);
4215        assert!(background.sessions[0].activity.starts_with("BG "));
4216        let state = operational.get_mut("session-1").unwrap();
4217        state.background_commands.clear();
4218        // A phase flag alone can be stale; a current SDK step proves work.
4219        state.execution = RelayExecutionState::Running;
4220        let stale_running = project(&operational);
4221        assert!(stale_running.sessions[0].is_idle);
4222        operational
4223            .get_mut("session-1")
4224            .unwrap()
4225            .current_step_started_at_ms = Some(1_000);
4226        let running = project(&operational);
4227        assert!(!running.sessions[0].is_idle);
4228        assert_ne!(running.sessions[0].activity, "[idle]");
4229        operational.get_mut("session-1").unwrap().execution = RelayExecutionState::Idle;
4230        let idle_again = project(&operational);
4231        assert!(idle_again.sessions[0].is_idle);
4232        assert_eq!(idle_again.sessions[0].activity, "[idle]");
4233        let unknown = project(&std::collections::BTreeMap::new());
4234        assert!(!unknown.sessions[0].is_idle);
4235        assert!(unknown.sessions[0].activity.is_empty());
4236        assert!(unknown.sessions[0].activity_details.is_none());
4237    }
4238
4239    #[test]
4240    fn tailscale_listener_preserves_the_configured_port() {
4241        assert_eq!(
4242            tailscale_bind("127.0.0.1:4765".parse().unwrap()),
4243            "0.0.0.0:4765".parse().unwrap()
4244        );
4245    }
4246
4247    fn controller_with_profiles(ids: &[&str]) -> Controller {
4248        Controller {
4249            config: Config {
4250                subagents: Default::default(),
4251                version: CONFIG_VERSION,
4252                sessions_side: Default::default(),
4253                advanced: Default::default(),
4254                show_stopped_sessions: false,
4255                newer_config_version: None,
4256                spinner: Default::default(),
4257                theme: Default::default(),
4258                phone: Default::default(),
4259                review: Default::default(),
4260                legacy_startup: (),
4261                profiles: ids
4262                    .iter()
4263                    .map(|id| {
4264                        (
4265                            (*id).to_owned(),
4266                            HarnessProfile {
4267                                enabled: true,
4268                                context_window_bytes: None,
4269                                kind: HarnessKind::Codex,
4270                                home: PathBuf::from("/home/agent").join(id),
4271                                environment: std::collections::BTreeMap::new(),
4272                            },
4273                        )
4274                    })
4275                    .collect(),
4276                bundles: std::collections::BTreeMap::new(),
4277                targets: std::collections::BTreeMap::new(),
4278            },
4279            state: State::default(),
4280        }
4281    }
4282
4283    fn snapshot_with_project_sources(
4284        controller: &Controller,
4285        sources: &PhoneProjectSources,
4286    ) -> ViewerSnapshot {
4287        viewer_snapshot(
4288            controller,
4289            &[],
4290            &Default::default(),
4291            &PhoneSessionViews {
4292                conversations: &Default::default(),
4293                queued_prompts: &Default::default(),
4294                active_user_shells: &Default::default(),
4295                pending_elicitations: &Default::default(),
4296                prompt_images: &Default::default(),
4297                operational: &Default::default(),
4298                materialized_activity: &Default::default(),
4299                project_sources: sources,
4300                operations: &Default::default(),
4301                move_recoveries: &Default::default(),
4302                capacity: &[],
4303                launch_failures: &[],
4304                reviews: &Default::default(),
4305            },
4306            1,
4307        )
4308    }
4309
4310    #[tokio::test]
4311    async fn phone_projects_resolve_origins_and_discard_results_after_location_changes() {
4312        let root = tempfile::tempdir().unwrap();
4313        let mut controller = controller_with_profiles(&["codex"]);
4314        controller
4315            .config
4316            .targets
4317            .insert("local".into(), TargetTemplate::LocalBare);
4318        for (id, origin) in [
4319            ("first-checkout", "git@github.com:BrokkAi/hel.git"),
4320            ("second-checkout", "https://github.com/BrokkAi/hel.git"),
4321        ] {
4322            let directory = root.path().join(id);
4323            std::fs::create_dir(&directory).unwrap();
4324            for args in [vec!["init"], vec!["remote", "add", "origin", origin]] {
4325                let command = crate::targets::CommandSpec::new(
4326                    "git",
4327                    ["-C".to_owned(), directory.to_string_lossy().into_owned()]
4328                        .into_iter()
4329                        .chain(args.into_iter().map(str::to_owned)),
4330                );
4331                assert_eq!(ProcessExecutor.execute(&command).unwrap().status, 0);
4332            }
4333            let mut record = phone_session(id, 0);
4334            record.project_directory = Some(directory);
4335            record.target_template_id = "local".into();
4336            controller.state.sessions.insert(id.into(), record);
4337        }
4338        let mut sources = PhoneProjectSources::default();
4339        sources.synchronize(&controller);
4340        assert_eq!(sources.jobs.len(), 2);
4341        tokio::time::timeout(Duration::from_secs(10), async {
4342            while let Some(result) = sources.jobs.join_next().await {
4343                sources.complete(result.unwrap());
4344            }
4345        })
4346        .await
4347        .unwrap();
4348        let snapshot = snapshot_with_project_sources(&controller, &sources);
4349        assert_eq!(
4350            snapshot.sessions[0].project_key,
4351            snapshot.sessions[1].project_key
4352        );
4353        assert!(
4354            snapshot
4355                .sessions
4356                .iter()
4357                .all(|session| session.project_label == "hel")
4358        );
4359        assert!(
4360            !serde_json::to_string(&snapshot)
4361                .unwrap()
4362                .contains(&root.path().to_string_lossy().to_string())
4363        );
4364        sources.synchronize(&controller);
4365        assert!(
4366            sources.jobs.is_empty(),
4367            "unchanged inputs reuse the resolved origin"
4368        );
4369
4370        let previous = &sources.entries["first-checkout"];
4371        let late = ProjectSourceResolved {
4372            cancelled: previous.cancelled.clone(),
4373            session_id: "first-checkout".into(),
4374            key: previous.key.clone(),
4375            result: Ok(ProjectSourceIdentity::git_remote("old/wrong").unwrap()),
4376        };
4377        controller
4378            .state
4379            .sessions
4380            .get_mut("first-checkout")
4381            .unwrap()
4382            .project_directory = None;
4383        let snapshot = snapshot_with_project_sources(&controller, &sources);
4384        assert_ne!(
4385            snapshot.sessions[0].project_key,
4386            snapshot.sessions[1].project_key
4387        );
4388        sources.synchronize(&controller);
4389        assert!(late.cancelled.load(Ordering::Acquire));
4390        sources.complete(late);
4391        assert!(!sources.entries.contains_key("first-checkout"));
4392        assert_eq!(snapshot.sessions[0].project_label, "project");
4393    }
4394
4395    #[test]
4396    fn capacity_target_publication_skips_unchanged_targets_and_preserves_readings() {
4397        let mut controller = controller_with_profiles(&[]);
4398        controller
4399            .config
4400            .targets
4401            .insert("raw".into(), TargetTemplate::LocalBare);
4402        let (targets_tx, mut targets_rx) = tokio::sync::watch::channel(Vec::new());
4403        let mut state = std::collections::BTreeMap::new();
4404
4405        publish_capacity_targets(&controller, &targets_tx, &mut state);
4406        assert!(targets_rx.has_changed().expect("target sender is alive"));
4407        assert_eq!(targets_rx.borrow_and_update().len(), 1);
4408
4409        let usage = crate::targets::DeploymentCapacityUsage {
4410            cpu_percent: Some(37),
4411            memory_used_bytes: 3,
4412            memory_total_bytes: 4,
4413            logical_cores: 8,
4414            disk_total_bytes: Some(5),
4415        };
4416        let local = state.get_mut("local").expect("local capacity state");
4417        local.usage = Some(usage.clone());
4418        local.on_demand = true;
4419        local.sampled_at_epoch_seconds = Some(42);
4420        local.refreshing = false;
4421
4422        publish_capacity_targets(&controller, &targets_tx, &mut state);
4423        assert!(!targets_rx.has_changed().expect("target sender is alive"));
4424        let local_capacity = viewer_capacity(&state)
4425            .into_iter()
4426            .find(|capacity| capacity.id == "local")
4427            .expect("local viewer capacity");
4428        assert_eq!(local_capacity.cpu_percent, usage.cpu_percent);
4429        assert_eq!(
4430            local_capacity.memory_used_bytes,
4431            Some(usage.memory_used_bytes)
4432        );
4433        assert_eq!(local_capacity.logical_cores, Some(usage.logical_cores));
4434        assert_eq!(local_capacity.sampled_at_epoch_seconds, Some(42));
4435
4436        controller
4437            .config
4438            .targets
4439            .insert("second-local".into(), TargetTemplate::LocalBare);
4440        publish_capacity_targets(&controller, &targets_tx, &mut state);
4441        assert!(targets_rx.has_changed().expect("target sender is alive"));
4442        assert_eq!(targets_rx.borrow_and_update().len(), 1);
4443        assert_eq!(state["local"].usage, Some(usage.clone()));
4444
4445        controller.config.targets.insert(
4446            "fleet".into(),
4447            TargetTemplate::AwsEc2 {
4448                aws_profile: None,
4449                region: "us-east-1".into(),
4450                launch_template: "hel-runson".into(),
4451                launch_template_version: None,
4452                ssh_user: "ubuntu".into(),
4453                address_source: Default::default(),
4454                identity_file: None,
4455                ssh_args: Vec::new(),
4456            },
4457        );
4458        publish_capacity_targets(&controller, &targets_tx, &mut state);
4459        assert!(targets_rx.has_changed().expect("target sender is alive"));
4460        assert_eq!(targets_rx.borrow_and_update().len(), 2);
4461        assert!(state.contains_key("aws:fleet"));
4462        assert_eq!(state["local"].usage, Some(usage.clone()));
4463
4464        controller.config.targets.remove("fleet");
4465        publish_capacity_targets(&controller, &targets_tx, &mut state);
4466        assert!(targets_rx.has_changed().expect("target sender is alive"));
4467        assert_eq!(targets_rx.borrow_and_update().len(), 1);
4468        assert!(!state.contains_key("aws:fleet"));
4469        assert_eq!(state["local"].usage, Some(usage));
4470    }
4471
4472    fn prompt_action() -> ControllerAction {
4473        ControllerAction::Prompt {
4474            session_id: "session-1".into(),
4475            text: "ship it".into(),
4476            images: Vec::new(),
4477        }
4478    }
4479
4480    fn new_action() -> ControllerAction {
4481        ControllerAction::New {
4482            mjolnir_subagents: None,
4483            create_managed_worktree: None,
4484            workspace_id: String::new(),
4485            profile_id: "codex".into(),
4486            bundle_id: "project".into(),
4487            target_id: "podman".into(),
4488            title: Some("Phone launch".into()),
4489            project_directory: None,
4490            dirty_ack: Vec::new(),
4491        }
4492    }
4493
4494    fn phone_session(id: &str, viewed_through_event_ordinal: u64) -> SessionRecord {
4495        SessionRecord {
4496            mjolnir_subagents: None,
4497            create_managed_worktree: None,
4498            workspace_id: mj_core::workspace::DEFAULT_WORKSPACE_ID.to_owned(),
4499            archived: false,
4500            container_cpus: None,
4501            container_memory: None,
4502            id: id.into(),
4503            title: "Phone launch".into(),
4504            harness_kind: mj_core::config::HarnessKind::Codex,
4505            last_profile: "codex".into(),
4506            bundle_id: "project".into(),
4507            project_directory: None,
4508            managed_worktree: None,
4509            target_template_id: "podman".into(),
4510            resource_allocation: None,
4511            additional_mounts: Vec::new(),
4512            state: SessionState::Provisioning,
4513            target: None,
4514            native_session_id: None,
4515            acp_session_title: None,
4516            session_title_override: Some("Phone launch".into()),
4517            created_at: "2026-08-14T00:00:00Z".into(),
4518            updated_at: "2026-08-14T00:00:00Z".into(),
4519            viewed_through_event_ordinal,
4520            draft_input: String::new(),
4521            last_error: None,
4522            last_checkpoint_error: None,
4523            checkpoint: None,
4524        }
4525    }
4526
4527    #[tokio::test]
4528    async fn controller_reload_does_not_block_the_phone_control_loop() {
4529        let (completed_tx, mut completed_rx) = tokio::sync::mpsc::unbounded_channel();
4530        let (started_tx, started_rx) = std::sync::mpsc::channel();
4531        let (release_tx, release_rx) = std::sync::mpsc::channel();
4532        let loaded = controller_with_profiles(&[]);
4533        let release = std::thread::spawn(move || {
4534            started_rx.recv().unwrap();
4535            std::thread::sleep(Duration::from_millis(250));
4536            release_tx.send(()).unwrap();
4537        });
4538
4539        let started = Instant::now();
4540        spawn_controller_reload_with(completed_tx, move || {
4541            started_tx.send(()).unwrap();
4542            release_rx.recv().unwrap();
4543            Ok(loaded)
4544        });
4545        assert!(
4546            started.elapsed() < Duration::from_millis(100),
4547            "scheduling a controller reload occupied the control loop for {:?}",
4548            started.elapsed()
4549        );
4550
4551        let completed = tokio::time::timeout(Duration::from_secs(1), completed_rx.recv())
4552            .await
4553            .expect("background reload timed out")
4554            .expect("background reload channel closed");
4555        assert!(completed.result.is_ok());
4556        release.join().unwrap();
4557    }
4558
4559    #[test]
4560    fn read_receipt_only_persists_and_refreshes_when_the_cursor_advances() {
4561        let session_id = "0123456789abcdef0123456789abcdef";
4562        let mut state = State::default();
4563        state
4564            .sessions
4565            .insert(session_id.into(), phone_session(session_id, 5));
4566
4567        // The viewer re-posts its cursor after every refresh; a repeat must
4568        // not reach the database and must not move the revision.
4569        assert_eq!(
4570            plan_read_receipt(&state, session_id, 5),
4571            ReadReceiptPlan::AlreadyRead
4572        );
4573        assert_eq!(
4574            plan_read_receipt(&state, session_id, 4),
4575            ReadReceiptPlan::AlreadyRead
4576        );
4577        assert_eq!(
4578            plan_read_receipt(&state, "missing", 9),
4579            ReadReceiptPlan::UnknownSession
4580        );
4581        assert_eq!(
4582            plan_read_receipt(&state, session_id, 9),
4583            ReadReceiptPlan::Persist
4584        );
4585
4586        assert!(apply_read_receipt(&mut state, session_id, 9));
4587        assert_eq!(state.sessions[session_id].viewed_through_event_ordinal, 9);
4588        assert!(!apply_read_receipt(&mut state, session_id, 9));
4589        assert!(!apply_read_receipt(&mut state, session_id, 7));
4590        assert!(!apply_read_receipt(&mut state, "missing", 9));
4591        assert_eq!(
4592            plan_read_receipt(&state, session_id, 9),
4593            ReadReceiptPlan::AlreadyRead
4594        );
4595    }
4596
4597    #[tokio::test]
4598    async fn an_admitted_action_answers_its_phone_before_the_work_runs() {
4599        let mut replies = PendingActionReplies::default();
4600        let (reply, answer) = tokio::sync::oneshot::channel();
4601
4602        replies.accept(1, &prompt_action(), reply);
4603
4604        // No completion has been reported, and the phone already has its
4605        // answer: holding it until the action finished is what mobile
4606        // networks time out on.
4607        assert_eq!(answer.await.unwrap(), ActionOutcome::accepted());
4608    }
4609
4610    #[tokio::test]
4611    async fn a_new_action_answers_once_its_provisional_session_is_published() {
4612        let mut replies = PendingActionReplies::default();
4613        let (reply, mut answer) = tokio::sync::oneshot::channel();
4614
4615        replies.accept(7, &new_action(), reply);
4616        assert!(
4617            matches!(
4618                answer.try_recv(),
4619                Err(tokio::sync::oneshot::error::TryRecvError::Empty)
4620            ),
4621            "a new session has no id to report before it is published"
4622        );
4623
4624        replies.resolve(7, ActionOutcome::accepted());
4625        assert_eq!(answer.await.unwrap(), ActionOutcome::accepted());
4626    }
4627
4628    #[tokio::test]
4629    async fn a_new_action_that_never_publishes_still_answers_its_phone() {
4630        let mut replies = PendingActionReplies::default();
4631        let (reply, answer) = tokio::sync::oneshot::channel();
4632        replies.accept(7, &new_action(), reply);
4633
4634        // Registration failed before the session reached the loop, which is
4635        // the completion path rather than the publication path.
4636        replies.resolve(7, ActionOutcome::Failed);
4637
4638        assert_eq!(answer.await.unwrap(), ActionOutcome::Failed);
4639        // A second resolution is a no-op, so a completion after a publication
4640        // cannot overwrite the answer already sent.
4641        replies.resolve(7, ActionOutcome::accepted());
4642    }
4643
4644    #[test]
4645    fn force_close_is_admitted_like_close() {
4646        let mut active = std::collections::BTreeSet::from(["session-1".to_owned()]);
4647        let force_close = ControllerAction::ForceClose {
4648            session_id: "session-1".into(),
4649        };
4650        // A full action pool and a session already busy with a stuck close
4651        // are both exactly when a force close has to get through.
4652        assert_eq!(
4653            admit_phone_action(&force_close, MAX_CONCURRENT_PHONE_ACTIONS, &mut active),
4654            Ok(Some("session-1".into()))
4655        );
4656        assert_eq!(
4657            controller_action_session_id(&force_close),
4658            Some("session-1".to_owned())
4659        );
4660    }
4661
4662    #[test]
4663    fn close_is_admitted_while_provisioning_occupies_a_full_action_pool() {
4664        let mut active = std::collections::BTreeSet::from(["session-1".to_owned()]);
4665        let close = ControllerAction::Close {
4666            session_id: "session-1".into(),
4667        };
4668        assert_eq!(
4669            admit_phone_action(&close, MAX_CONCURRENT_PHONE_ACTIONS, &mut active),
4670            Ok(Some("session-1".into()))
4671        );
4672        assert_eq!(
4673            admit_phone_action(&prompt_action(), 0, &mut active),
4674            Err(ActionOutcome::SessionBusy)
4675        );
4676    }
4677
4678    #[test]
4679    fn a_refused_action_reports_the_reason_the_phone_can_act_on() {
4680        let mut active = std::collections::BTreeSet::new();
4681
4682        assert_eq!(
4683            admit_phone_action(&prompt_action(), 0, &mut active),
4684            Ok(Some("session-1".to_owned()))
4685        );
4686        assert_eq!(
4687            admit_phone_action(&prompt_action(), 1, &mut active),
4688            Err(ActionOutcome::SessionBusy)
4689        );
4690        assert_eq!(
4691            admit_phone_action(&new_action(), MAX_CONCURRENT_PHONE_ACTIONS, &mut active),
4692            Err(ActionOutcome::Busy)
4693        );
4694        // A refusal must not consume the session slot it did not take.
4695        assert_eq!(active.len(), 1);
4696        assert_eq!(admit_phone_action(&new_action(), 1, &mut active), Ok(None));
4697    }
4698
4699    #[test]
4700    fn a_feed_that_ends_outside_shutdown_names_the_failure() {
4701        assert!(feed_stopped(true, "the session manager stopped").is_none());
4702        let failure = feed_stopped(false, "the session manager stopped").expect("named failure");
4703        assert!(failure.to_string().contains("session manager"));
4704    }
4705
4706    #[test]
4707    fn a_profile_added_while_the_server_runs_reaches_the_quota_refresher() {
4708        let (profiles_tx, profiles_rx) = tokio::sync::watch::channel(QuotaRefreshBatch::default());
4709        let mut published = std::collections::BTreeMap::new();
4710        let mut batch = QuotaRefreshBatch::default();
4711        let controller = controller_with_profiles(&["codex"]);
4712
4713        assert!(republish_quota_profiles(
4714            &controller,
4715            &mut published,
4716            &mut batch,
4717            &profiles_tx
4718        ));
4719        assert_eq!(
4720            profiles_rx
4721                .borrow()
4722                .profiles
4723                .iter()
4724                .map(|profile| profile.profile_id.clone())
4725                .collect::<Vec<_>>(),
4726            vec!["codex".to_owned()]
4727        );
4728        let first_generation = profiles_rx.borrow().generation;
4729
4730        // Every finished action reloads the configuration; an unchanged one
4731        // must not restart a harness process per profile.
4732        assert!(!republish_quota_profiles(
4733            &controller,
4734            &mut published,
4735            &mut batch,
4736            &profiles_tx
4737        ));
4738        assert_eq!(profiles_rx.borrow().generation, first_generation);
4739
4740        let grown = controller_with_profiles(&["claude", "codex"]);
4741        assert!(republish_quota_profiles(
4742            &grown,
4743            &mut published,
4744            &mut batch,
4745            &profiles_tx
4746        ));
4747        assert_eq!(
4748            profiles_rx
4749                .borrow()
4750                .profiles
4751                .iter()
4752                .map(|profile| profile.profile_id.clone())
4753                .collect::<Vec<_>>(),
4754            vec!["claude".to_owned(), "codex".to_owned()]
4755        );
4756        assert!(profiles_rx.borrow().generation > first_generation);
4757    }
4758
4759    #[test]
4760    fn a_quota_reads_stale_only_once_its_next_refresh_is_overdue() {
4761        let controller = controller_with_profiles(&["codex"]);
4762        let now = std::time::SystemTime::now()
4763            .duration_since(std::time::UNIX_EPOCH)
4764            .unwrap()
4765            .as_secs();
4766        let quota_refreshed = |age: Duration| {
4767            let quotas = std::collections::BTreeMap::from([(
4768                "codex".to_owned(),
4769                ProfileQuota {
4770                    profile_id: "codex".into(),
4771                    harness: HarnessKind::Codex,
4772                    windows: Vec::new(),
4773                    extra: None,
4774                    error: None,
4775                    refreshed_at_epoch_seconds: now - age.as_secs(),
4776                },
4777            )]);
4778            viewer_snapshot(
4779                &controller,
4780                &[],
4781                &quotas,
4782                &PhoneSessionViews {
4783                    conversations: &std::collections::BTreeMap::new(),
4784                    queued_prompts: &std::collections::BTreeMap::new(),
4785                    active_user_shells: &std::collections::BTreeMap::new(),
4786                    pending_elicitations: &std::collections::BTreeMap::new(),
4787                    prompt_images: &std::collections::BTreeSet::new(),
4788                    operational: &std::collections::BTreeMap::new(),
4789                    materialized_activity: &std::collections::BTreeMap::new(),
4790                    project_sources: &PhoneProjectSources::default(),
4791                    operations: &std::collections::BTreeMap::new(),
4792                    move_recoveries: &std::collections::BTreeMap::new(),
4793                    capacity: &[],
4794                    launch_failures: &[],
4795                    reviews: &std::collections::BTreeMap::new(),
4796                },
4797                1,
4798            )
4799            .profiles[0]
4800                .quota
4801                .as_ref()
4802                .expect("the profile carries its quota")
4803                .stale
4804        };
4805
4806        // A reading taken one refresh interval ago is exactly what a healthy
4807        // refresher produces, so it must not be labelled stale.
4808        assert!(!quota_refreshed(QUOTA_REFRESH_INTERVAL));
4809        assert!(!quota_refreshed(QUOTA_STALE_AFTER));
4810        assert!(quota_refreshed(QUOTA_STALE_AFTER + Duration::from_secs(1)));
4811    }
4812
4813    #[test]
4814    fn phone_action_capacity_is_bounded() {
4815        assert!(phone_action_capacity_available(
4816            MAX_CONCURRENT_PHONE_ACTIONS - 1
4817        ));
4818        assert!(!phone_action_capacity_available(
4819            MAX_CONCURRENT_PHONE_ACTIONS
4820        ));
4821    }
4822
4823    #[test]
4824    fn started_phone_session_is_visible_and_mapped_before_provisioning() {
4825        let session_id = "0123456789abcdef0123456789abcdef";
4826        let session = phone_session(session_id, 0);
4827        let mut state = State::default();
4828        let mut active_actions = std::collections::BTreeSet::new();
4829        let mut action_sessions = std::collections::BTreeMap::new();
4830
4831        track_started_phone_session(
4832            &mut state,
4833            &mut active_actions,
4834            &mut action_sessions,
4835            7,
4836            session,
4837        )
4838        .unwrap();
4839
4840        assert_eq!(state.sessions[session_id].state, SessionState::Provisioning);
4841        assert_eq!(state.sessions[session_id].display_title(), "Phone launch");
4842        assert!(active_actions.contains(session_id));
4843        assert_eq!(
4844            action_sessions.get(&7).map(String::as_str),
4845            Some(session_id)
4846        );
4847    }
4848
4849    #[test]
4850    fn failed_launch_notice_survives_session_rollback_and_history_is_bounded() {
4851        let controller = controller_with_profiles(&["codex"]);
4852        let mut failures = Vec::new();
4853        for index in 0..20 {
4854            record_launch_failure(
4855                &mut failures,
4856                index,
4857                format!("workspace-{index}"),
4858                Some(format!("session-{index}")),
4859                Some(format!("worker bootstrap failed for {index}")),
4860            );
4861        }
4862        let snapshot = viewer_snapshot(
4863            &controller,
4864            &[],
4865            &std::collections::BTreeMap::new(),
4866            &PhoneSessionViews {
4867                conversations: &std::collections::BTreeMap::new(),
4868                queued_prompts: &std::collections::BTreeMap::new(),
4869                active_user_shells: &std::collections::BTreeMap::new(),
4870                pending_elicitations: &std::collections::BTreeMap::new(),
4871                prompt_images: &std::collections::BTreeSet::new(),
4872                operational: &std::collections::BTreeMap::new(),
4873                materialized_activity: &std::collections::BTreeMap::new(),
4874                project_sources: &PhoneProjectSources::default(),
4875                operations: &std::collections::BTreeMap::new(),
4876                move_recoveries: &std::collections::BTreeMap::new(),
4877                capacity: &[],
4878                launch_failures: &failures,
4879                reviews: &std::collections::BTreeMap::new(),
4880            },
4881            1,
4882        );
4883        assert!(snapshot.sessions.is_empty());
4884        assert_eq!(failures.len(), 16);
4885        assert_eq!(failures[0].workspace_id, "workspace-4");
4886        assert_eq!(failures[15].workspace_id, "workspace-19");
4887        assert_ne!(failures[0].id, failures[1].id);
4888        assert_eq!(
4889            failures[15].session_id.as_deref(),
4890            Some("session-19"),
4891            "a wait on that session has to be able to recognize its own launch failure"
4892        );
4893        let json = serde_json::to_value(snapshot).unwrap();
4894        assert_eq!(json["launch_failures"][15]["workspace_id"], "workspace-19");
4895        assert_eq!(
4896            json["launch_failures"][15]["error"], "worker bootstrap failed for 19",
4897            "the recorded failure carries its reason so a client can show it"
4898        );
4899        assert_eq!(json["launch_failures"][15].as_object().unwrap().len(), 4);
4900    }
4901
4902    #[test]
4903    fn a_later_successful_action_clears_a_session_s_recorded_failure() {
4904        let mut pending = std::collections::BTreeMap::new();
4905
4906        record_action_result(&mut pending, Some("session-1"), &Err("relay hiccup".into()));
4907        assert_eq!(
4908            pending.get("session-1").map(String::as_str),
4909            Some("relay hiccup")
4910        );
4911
4912        record_action_result(&mut pending, Some("session-2"), &Ok(()));
4913        record_action_result(&mut pending, Some("session-1"), &Ok(()));
4914        assert!(
4915            pending.is_empty(),
4916            "the overlay has no other expiry, so a stale error would badge the session forever"
4917        );
4918
4919        // A completion with no session cannot clear or record anything.
4920        record_action_result(&mut pending, None, &Err("orphaned".into()));
4921        assert!(pending.is_empty());
4922    }
4923
4924    #[test]
4925    fn phone_cancel_targets_the_matching_background_action() {
4926        let first = PhoneActionControl {
4927            cancelled: Arc::new(AtomicBool::new(false)),
4928            create: None,
4929        };
4930        let second = PhoneActionControl {
4931            cancelled: Arc::new(AtomicBool::new(false)),
4932            create: None,
4933        };
4934        let action_sessions =
4935            std::collections::BTreeMap::from([(1, "session-1".into()), (2, "session-2".into())]);
4936        let cancellations =
4937            std::collections::BTreeMap::from([(1, first.clone()), (2, second.clone())]);
4938
4939        assert!(request_phone_action_cancellation(
4940            "session-2",
4941            &action_sessions,
4942            &cancellations,
4943        ));
4944        assert!(!first.cancelled.load(Ordering::Acquire));
4945        assert!(second.cancelled.load(Ordering::Acquire));
4946        assert!(!request_phone_action_cancellation(
4947            "missing",
4948            &action_sessions,
4949            &cancellations,
4950        ));
4951    }
4952
4953    #[test]
4954    fn phone_new_cancel_and_running_commit_have_one_atomic_winner() {
4955        for _ in 0..100 {
4956            let create = CreateSessionControl::default();
4957            let control = PhoneActionControl {
4958                cancelled: create.cancelled.clone(),
4959                create: Some(create),
4960            };
4961            let cancelling = control.clone();
4962            let committing = control.clone();
4963            let (cancelled, committed) = std::thread::scope(|scope| {
4964                let cancel = scope.spawn(move || cancelling.request_cancel());
4965                let commit = scope.spawn(move || committing.grant_new_commit());
4966                (cancel.join().unwrap(), commit.join().unwrap())
4967            });
4968
4969            assert_ne!(cancelled, committed);
4970            assert_eq!(control.cancelled.load(Ordering::Acquire), cancelled);
4971            assert!(!control.request_cancel());
4972            assert!(!control.grant_new_commit());
4973        }
4974    }
4975
4976    fn materialized_at(ordinal: u64) -> MaterializedSession {
4977        let mut materialized = MaterializedSession::empty("session-1");
4978        materialized.applied_event_ordinal = ordinal;
4979        materialized.applied_event_digest = format!("digest-{ordinal}");
4980        materialized
4981    }
4982
4983    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
4984    async fn browser_projection_enqueue_does_not_wait_for_a_blocking_permit() {
4985        let (results, mut completed) = tokio::sync::mpsc::channel(1);
4986        let shutdown = tokio_util::sync::CancellationToken::new();
4987        let mut dispatcher = ConversationProjectionDispatcher::with_permits(results, shutdown, 0);
4988
4989        dispatcher.enqueue(materialized_at(1));
4990        let (progress, progress_done) = tokio::sync::oneshot::channel();
4991        tokio::spawn(async move {
4992            progress.send(()).expect("control task is still alive");
4993        });
4994
4995        tokio::time::timeout(Duration::from_millis(100), progress_done)
4996            .await
4997            .expect("control progress was starved by transcript projection")
4998            .expect("progress task did not report");
4999        assert!(completed.try_recv().is_err());
5000    }
5001
5002    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
5003    async fn browser_projection_keeps_only_the_newest_pending_snapshot() {
5004        let (results, mut completed) = tokio::sync::mpsc::channel(4);
5005        let shutdown = tokio_util::sync::CancellationToken::new();
5006        let mut dispatcher = ConversationProjectionDispatcher::with_permits(results, shutdown, 0);
5007
5008        dispatcher.enqueue(materialized_at(1));
5009        dispatcher.enqueue(materialized_at(3));
5010        dispatcher.enqueue(materialized_at(2));
5011        assert_eq!(dispatcher.pending["session-1"].key.ordinal, 3);
5012
5013        dispatcher.permits.add_permits(1);
5014        let first = tokio::time::timeout(Duration::from_secs(1), completed.recv())
5015            .await
5016            .expect("first projection did not complete")
5017            .expect("projection result channel closed");
5018        assert_eq!(first.key.ordinal, 1);
5019        assert!(dispatcher.finish(first, true).is_some());
5020
5021        let second = tokio::time::timeout(Duration::from_secs(1), completed.recv())
5022            .await
5023            .expect("coalesced projection did not complete")
5024            .expect("projection result channel closed");
5025        assert_eq!(second.key.ordinal, 3);
5026        assert!(dispatcher.finish(second, true).is_some());
5027        assert!(dispatcher.pending.is_empty());
5028        assert!(dispatcher.in_flight.is_empty());
5029    }
5030
5031    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
5032    async fn forgotten_projection_cannot_repopulate_after_a_same_cursor_resume() {
5033        let (results, mut completed) = tokio::sync::mpsc::channel(4);
5034        let shutdown = tokio_util::sync::CancellationToken::new();
5035        let mut dispatcher = ConversationProjectionDispatcher::with_permits(results, shutdown, 0);
5036        let snapshot = materialized_at(7);
5037
5038        dispatcher.enqueue(snapshot.clone());
5039        dispatcher.forget("session-1");
5040        dispatcher.enqueue(snapshot);
5041        dispatcher.permits.add_permits(1);
5042
5043        let old = tokio::time::timeout(Duration::from_secs(1), completed.recv())
5044            .await
5045            .expect("old projection did not complete")
5046            .expect("projection result channel closed");
5047        assert!(dispatcher.finish(old, true).is_none());
5048
5049        let current = tokio::time::timeout(Duration::from_secs(1), completed.recv())
5050            .await
5051            .expect("resumed projection did not complete")
5052            .expect("projection result channel closed");
5053        assert!(dispatcher.finish(current, true).is_some());
5054        assert!(dispatcher.in_flight.is_empty());
5055    }
5056}